From e8498251d9704f15633a349290c463ec46a17fdf Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 15 Aug 2024 17:25:16 -0600 Subject: [PATCH 001/199] add on_kill override to databricks workflow operator --- .../operators/databricks_workflow.py | 27 +++++++++++++++++++ .../operators/test_databricks_workflow.py | 21 +++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/airflow/providers/databricks/operators/databricks_workflow.py b/airflow/providers/databricks/operators/databricks_workflow.py index 15333dc69118b..e731d3bfbbc87 100644 --- a/airflow/providers/databricks/operators/databricks_workflow.py +++ b/airflow/providers/databricks/operators/databricks_workflow.py @@ -33,6 +33,7 @@ WorkflowJobRunLink, ) from airflow.utils.task_group import TaskGroup +from tests.providers.google.cloud.transfers.test_gcs_to_bigquery import job_id if TYPE_CHECKING: from types import TracebackType @@ -116,6 +117,7 @@ def __init__( self.notebook_params = notebook_params or {} self.tasks_to_convert = tasks_to_convert or [] self.relevant_upstreams = [task_id] + self.workflow_run_metadata: WorkflowRunMetadata | None = None super().__init__(task_id=task_id, **kwargs) def _get_hook(self, caller: str) -> DatabricksHook: @@ -212,12 +214,37 @@ def execute(self, context: Context) -> Any: self._wait_for_job_to_start(run_id) + self.workflow_run_metadata = WorkflowRunMetadata( + self.databricks_conn_id, + job_id, + run_id, + ) + return { "conn_id": self.databricks_conn_id, "job_id": job_id, "run_id": run_id, } + def on_kill(self) -> None: + + if self.workflow_run_metadata: + + run_id = self.workflow_run_metadata.run_id + job_id = self.workflow_run_metadata.job_id + + self._hook.cancel_run(run_id) + self.log.info( + f"Run: {run_id} of job_id: {job_id} was requested to be cancelled." + ) + else: + self.log.error( + f""" + Error: Workflow Run metadata is not populated, so the run was not canceled. This could be due + to the workflow not being started or an error in the workflow creation process. + """ + ) + class DatabricksWorkflowTaskGroup(TaskGroup): """ diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/tests/providers/databricks/operators/test_databricks_workflow.py index 4c3f54b800ae9..f0ecb255295d0 100644 --- a/tests/providers/databricks/operators/test_databricks_workflow.py +++ b/tests/providers/databricks/operators/test_databricks_workflow.py @@ -30,6 +30,7 @@ DatabricksWorkflowTaskGroup, _CreateDatabricksWorkflowOperator, _flatten_node, + WorkflowRunMetadata, ) from airflow.utils import timezone @@ -58,6 +59,10 @@ def mock_task_group(): mock_group.group_id = "test_group" return mock_group +@pytest.fixture +def mock_workflow_run_metadata(): + return MagicMock(spec=WorkflowRunMetadata) + def test_flatten_node(): """Test that _flatten_node returns a flat list of operators.""" @@ -231,3 +236,19 @@ def test_task_group_root_tasks_set_upstream_to_operator(mock_databricks_workflow create_operator_instance = mock_databricks_workflow_operator.return_value task1.set_upstream.assert_called_once_with(create_operator_instance) + + +def test_on_kill(mock_databricks_hook, context, mock_workflow_run_metadata): + """Test that _CreateDatabricksWorkflowOperator.execute runs the task group.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.workflow_run_metadata = mock_workflow_run_metadata + + RUN_ID = 789 + + mock_workflow_run_metadata.conn_id = operator.databricks_conn_id + mock_workflow_run_metadata.job_id = "123" + mock_workflow_run_metadata.run_id = RUN_ID + + operator.on_kill() + + operator._hook.cancel_run.assert_called_once_with(RUN_ID) From 52c00f9f43a4ef36a17a4b0dc509e3ab086aded7 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 30 Aug 2024 18:23:34 -0600 Subject: [PATCH 002/199] on_kill equivalent for DatabricksSqlOperator --- .../databricks/hooks/databricks_sql.py | 36 ++++- .../databricks/hooks/test_databricks_sql.py | 130 ++++++++++++++---- 2 files changed, 142 insertions(+), 24 deletions(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index fed85835d095c..d3fd98c19c769 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -16,10 +16,12 @@ # under the License. from __future__ import annotations +import threading import warnings from collections import namedtuple from contextlib import closing from copy import copy +from datetime import timedelta, datetime from typing import ( TYPE_CHECKING, Any, @@ -49,6 +51,16 @@ T = TypeVar("T") +def create_timeout_thread(cur, execution_timeout: timedelta | None) -> threading.Timer | None: + if execution_timeout is not None: + seconds_to_timeout = execution_timeout.total_seconds() + t = threading.Timer(seconds_to_timeout, cur.connection.cancel) + else: + t = None + + return t + + class DatabricksSqlHook(BaseDatabricksHook, DbApiHook): """ Hook to interact with Databricks SQL. @@ -184,6 +196,7 @@ def run( handler: None = ..., split_statements: bool = ..., return_last: bool = ..., + execution_timeout: timedelta | None = ..., ) -> None: ... @overload @@ -195,12 +208,14 @@ def run( handler: Callable[[Any], T] = ..., split_statements: bool = ..., return_last: bool = ..., + execution_timeout: timedelta | None = ..., ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: ... def run( self, sql: str | Iterable[str], autocommit: bool = False, + execution_timeout: timedelta | None = None, parameters: Iterable | Mapping[str, Any] | None = None, handler: Callable[[Any], T] | None = None, split_statements: bool = True, @@ -222,6 +237,8 @@ def run( :param handler: The result handler which is called with the result of each statement. :param split_statements: Whether to split a single SQL string into statements and run separately :param return_last: Whether to return result for only last statement or for all after split + :param execution_timeout: max time allowed for the execution of + this task instance, if it goes beyond it will raise and fail. :return: return only result of the LAST SQL expression if handler was provided unless return_last is set to False. """ @@ -242,13 +259,30 @@ def run( conn = None results = [] for sql_statement in sql_list: + # when using AAD tokens, it could expire if previous query run longer than token lifetime conn = self.get_conn() with closing(conn.cursor()) as cur: self.set_autocommit(conn, autocommit) with closing(conn.cursor()) as cur: - self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] + + t = create_timeout_thread(cur, execution_timeout) + + #todo; adjust this to make testing easier + try: + self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] + except Exception as e: + if t.is_alive(): + raise AirflowException(f"Error running statement: {sql_statement}. {str(e)}") + else: + raise AirflowException( + f"Timeout threshold exceeded for query: {sql_statement} was cancelled." + ) + finally: + if t is not None: + t.cancel() + if handler is not None: raw_result = handler(cur) if self.return_tuple: diff --git a/tests/providers/databricks/hooks/test_databricks_sql.py b/tests/providers/databricks/hooks/test_databricks_sql.py index a1b318ad00dcd..f87abfe5d4320 100644 --- a/tests/providers/databricks/hooks/test_databricks_sql.py +++ b/tests/providers/databricks/hooks/test_databricks_sql.py @@ -18,18 +18,22 @@ # from __future__ import annotations +import threading +import time from collections import namedtuple +from datetime import timedelta from unittest import mock -from unittest.mock import patch +from unittest.mock import patch, PropertyMock import pytest from databricks.sql.types import Row -from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import fetch_all_handler from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook from airflow.utils.session import provide_session +from airflow.utils.timezone import datetime pytestmark = pytest.mark.db_test @@ -55,23 +59,63 @@ def create_connection(session): def databricks_hook(): return DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=True) +@pytest.fixture +def mock_get_conn(): + # Start the patcher + mock_patch = patch("airflow.providers.databricks.hooks.databricks_sql.DatabricksSqlHook.get_conn") + mock_conn = mock_patch.start() + # Use yield to provide the mock object + yield mock_conn + # Stop the patcher + mock_patch.stop() + + +@pytest.fixture +def mock_get_requests(): + # Start the patcher + mock_patch = patch("airflow.providers.databricks.hooks.databricks_base.requests") + mock_requests = mock_patch.start() + + # Configure the mock object + mock_requests.codes.ok = 200 + mock_requests.get.return_value.json.return_value = { + "endpoints": [ + { + "id": "1264e5078741679a", + "name": "Test", + "odbc_params": { + "hostname": "xx.cloud.databricks.com", + "path": "/sql/1.0/endpoints/1264e5078741679a", + }, + } + ] + } + status_code_mock = PropertyMock(return_value=200) + type(mock_requests.get.return_value).status_code = status_code_mock + + # Yield the mock object + yield mock_requests + + # Stop the patcher after the test + mock_patch.stop() + def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: return [(field,) for field in fields] - # Serializable Row object similar to the one returned by the Hook SerializableRow = namedtuple("Row", ["id", "value"]) # type: ignore[name-match] @pytest.mark.parametrize( - "return_last, split_statements, sql, cursor_calls, return_tuple," + "return_last, split_statements, sql, execution_timeout, cursor_calls, return_tuple," "cursor_descriptions, cursor_results, hook_descriptions, hook_results, ", [ pytest.param( True, False, "select * from test.test", + None, ["select * from test.test"], False, [["id", "value"]], @@ -84,6 +128,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: False, False, "select * from test.test;", + None, ["select * from test.test"], False, [["id", "value"]], @@ -96,6 +141,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: True, True, "select * from test.test;", + None, ["select * from test.test"], False, [["id", "value"]], @@ -108,6 +154,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: False, True, "select * from test.test;", + None, ["select * from test.test"], False, [["id", "value"]], @@ -120,6 +167,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: True, True, "select * from test.test;select * from test.test2;", + None, ["select * from test.test", "select * from test.test2"], False, [["id", "value"], ["id2", "value2"]], @@ -132,6 +180,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: False, True, "select * from test.test;select * from test.test2;", + None, ["select * from test.test", "select * from test.test2"], False, [["id", "value"], ["id2", "value2"]], @@ -147,6 +196,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: True, True, ["select * from test.test;"], + None, ["select * from test.test"], False, [["id", "value"]], @@ -159,6 +209,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: False, True, ["select * from test.test;"], + None, ["select * from test.test"], False, [["id", "value"]], @@ -171,6 +222,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: True, True, "select * from test.test;select * from test.test2;", + None, ["select * from test.test", "select * from test.test2"], False, [["id", "value"], ["id2", "value2"]], @@ -183,6 +235,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: False, True, "select * from test.test;select * from test.test2;", + None, ["select * from test.test", "select * from test.test2"], False, [["id", "value"], ["id2", "value2"]], @@ -198,6 +251,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: True, False, "select * from test.test", + None, ["select * from test.test"], True, [["id", "value"]], @@ -210,6 +264,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: True, False, "select * from test.test", + None, ["select * from test.test"], True, [["id", "value"]], @@ -222,6 +277,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: True, False, "select * from test.test", + None, ["select * from test.test"], True, [["id", "value"]], @@ -236,31 +292,17 @@ def test_query( return_last, split_statements, sql, + execution_timeout, cursor_calls, return_tuple, cursor_descriptions, cursor_results, hook_descriptions, hook_results, + mock_get_conn, + mock_get_requests, ): - with patch( - "airflow.providers.databricks.hooks.databricks_sql.DatabricksSqlHook.get_conn" - ) as mock_conn, patch("airflow.providers.databricks.hooks.databricks_base.requests") as mock_requests: - mock_requests.codes.ok = 200 - mock_requests.get.return_value.json.return_value = { - "endpoints": [ - { - "id": "1264e5078741679a", - "name": "Test", - "odbc_params": { - "hostname": "xx.cloud.databricks.com", - "path": "/sql/1.0/endpoints/1264e5078741679a", - }, - } - ] - } - status_code_mock = mock.PropertyMock(return_value=200) - type(mock_requests.get.return_value).status_code = status_code_mock + connections = [] cursors = [] for index in range(len(cursor_descriptions)): @@ -273,7 +315,7 @@ def test_query( conn.cursor.return_value = cur cursors.append(cur) connections.append(conn) - mock_conn.side_effect = connections + mock_get_conn.side_effect = connections if not return_tuple: with pytest.warns( @@ -329,3 +371,45 @@ def test_incorrect_column_names(row_objects, fields_names): """ result = DatabricksSqlHook(return_tuple=True)._make_common_data_structure(row_objects) assert result._fields == fields_names + + +def test_execution_timeout_exceeded( + mock_get_conn, + mock_get_requests, + sql="select * from test.test", + execution_timeout=timedelta(microseconds=0), + cursor_descriptions=["id", "value"], + cursor_results=[Row(id=1, value=2), Row(id=11, value=12)], +): + with patch( + 'airflow.providers.databricks.hooks.databricks_sql.create_timeout_thread' + ) as mock_create_timeout_thread, patch.object( + DatabricksSqlHook, '_run_command' + ) as mock_run_command: + conn = mock.MagicMock() + cur = mock.MagicMock( + rowcount=len(cursor_results), + description=get_cursor_descriptions(cursor_descriptions), + ) + + # Simulate a timeout + mock_create_timeout_thread.return_value = threading.Timer( + cur, execution_timeout + ) + + mock_run_command.side_effect = Exception('Mocked exception') + + cur.fetchall.return_value = cursor_results + conn.cursor.return_value = cur + mock_get_conn.side_effect = [conn] + + with pytest.raises(AirflowException) as exc_info: + DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=True).run( + sql=sql, + execution_timeout=execution_timeout, + handler=fetch_all_handler, + ) + + assert "Timeout threshold exceeded" in str(exc_info.value) + + From 5f148b2ef18d4cf9ad389af93255dbf76ce6b2ab Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 5 Sep 2024 10:50:34 -0600 Subject: [PATCH 003/199] add tests for create_timeout_thread --- .../databricks/hooks/test_databricks_sql.py | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/providers/databricks/hooks/test_databricks_sql.py b/tests/providers/databricks/hooks/test_databricks_sql.py index f87abfe5d4320..0467ad41eb58c 100644 --- a/tests/providers/databricks/hooks/test_databricks_sql.py +++ b/tests/providers/databricks/hooks/test_databricks_sql.py @@ -31,7 +31,7 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import fetch_all_handler -from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook +from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook, create_timeout_thread from airflow.utils.session import provide_session from airflow.utils.timezone import datetime @@ -99,6 +99,10 @@ def mock_get_requests(): # Stop the patcher after the test mock_patch.stop() +@pytest.fixture +def mock_timer(): + with patch("threading.Timer") as mock_timer: + yield mock_timer def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: return [(field,) for field in fields] @@ -413,3 +417,32 @@ def test_execution_timeout_exceeded( assert "Timeout threshold exceeded" in str(exc_info.value) +def test_create_timeout_thread( + mock_get_conn, + mock_get_requests, + mock_timer, + cursor_descriptions=["id", "value"], +): + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + timeout = timedelta(seconds=1) + thread = create_timeout_thread(cur=cur, execution_timeout=timeout) + mock_timer.assert_called_once_with(timeout.total_seconds(), cur.connection.cancel) + assert thread is not None + + +def test_create_timeout_thread_no_timeout( + mock_get_conn, + mock_get_requests, + mock_timer, + cursor_descriptions=["id", "value"], +): + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + thread = create_timeout_thread(cur=cur, execution_timeout=None) + mock_timer.assert_not_called() + assert thread is None From 9faed81f03ce2ae86ce6f1b77130264b2b7554db Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 6 Sep 2024 10:23:54 -0600 Subject: [PATCH 004/199] add note for on_kill in DatabricksCopyIntoOperator --- airflow/providers/databricks/operators/databricks_sql.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/providers/databricks/operators/databricks_sql.py b/airflow/providers/databricks/operators/databricks_sql.py index 1975b4de2149d..a3851f05e0408 100644 --- a/airflow/providers/databricks/operators/databricks_sql.py +++ b/airflow/providers/databricks/operators/databricks_sql.py @@ -353,3 +353,8 @@ def execute(self, context: Context) -> Any: self.log.info("Executing: %s", sql) hook = self._get_hook() hook.run(sql) + + def on_kill(self) -> None: + #NB: on_kill isn't required for this operator since query cancelling gets + # handled in `DatabricksSqlHook.run()` method which is called in `execute()` + ... From e877d26da5ef996b60e5af13c2a91a3e6c32a161 Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 9 Sep 2024 10:13:38 -0600 Subject: [PATCH 005/199] chore: static checks --- .../databricks/hooks/databricks_sql.py | 12 +- .../databricks/operators/databricks_sql.py | 2 +- .../operators/databricks_workflow.py | 10 +- .../databricks/hooks/test_databricks_sql.py | 131 ++++++++++-------- .../operators/test_databricks_workflow.py | 3 +- 5 files changed, 82 insertions(+), 76 deletions(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index d3fd98c19c769..dc0af83476306 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -21,7 +21,7 @@ from collections import namedtuple from contextlib import closing from copy import copy -from datetime import timedelta, datetime +from datetime import timedelta from typing import ( TYPE_CHECKING, Any, @@ -192,11 +192,11 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., + execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: None = ..., split_statements: bool = ..., return_last: bool = ..., - execution_timeout: timedelta | None = ..., ) -> None: ... @overload @@ -204,11 +204,11 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., + execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: Callable[[Any], T] = ..., split_statements: bool = ..., return_last: bool = ..., - execution_timeout: timedelta | None = ..., ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: ... def run( @@ -259,21 +259,19 @@ def run( conn = None results = [] for sql_statement in sql_list: - # when using AAD tokens, it could expire if previous query run longer than token lifetime conn = self.get_conn() with closing(conn.cursor()) as cur: self.set_autocommit(conn, autocommit) with closing(conn.cursor()) as cur: - t = create_timeout_thread(cur, execution_timeout) - #todo; adjust this to make testing easier + # todo; adjust this to make testing easier try: self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: - if t.is_alive(): + if t is None or t.is_alive(): raise AirflowException(f"Error running statement: {sql_statement}. {str(e)}") else: raise AirflowException( diff --git a/airflow/providers/databricks/operators/databricks_sql.py b/airflow/providers/databricks/operators/databricks_sql.py index a3851f05e0408..62e55c682de04 100644 --- a/airflow/providers/databricks/operators/databricks_sql.py +++ b/airflow/providers/databricks/operators/databricks_sql.py @@ -355,6 +355,6 @@ def execute(self, context: Context) -> Any: hook.run(sql) def on_kill(self) -> None: - #NB: on_kill isn't required for this operator since query cancelling gets + # NB: on_kill isn't required for this operator since query cancelling gets # handled in `DatabricksSqlHook.run()` method which is called in `execute()` ... diff --git a/airflow/providers/databricks/operators/databricks_workflow.py b/airflow/providers/databricks/operators/databricks_workflow.py index e731d3bfbbc87..6df8e2d025cea 100644 --- a/airflow/providers/databricks/operators/databricks_workflow.py +++ b/airflow/providers/databricks/operators/databricks_workflow.py @@ -33,7 +33,6 @@ WorkflowJobRunLink, ) from airflow.utils.task_group import TaskGroup -from tests.providers.google.cloud.transfers.test_gcs_to_bigquery import job_id if TYPE_CHECKING: from types import TracebackType @@ -53,7 +52,7 @@ class WorkflowRunMetadata: """ conn_id: str - job_id: str + job_id: int run_id: int @@ -227,19 +226,18 @@ def execute(self, context: Context) -> Any: } def on_kill(self) -> None: - if self.workflow_run_metadata: - run_id = self.workflow_run_metadata.run_id job_id = self.workflow_run_metadata.job_id self._hook.cancel_run(run_id) self.log.info( - f"Run: {run_id} of job_id: {job_id} was requested to be cancelled." + "Run: %(run_id)s of job_id: %(job_id)s was requested to be cancelled.", + {"run_id": run_id, "job_id": job_id}, ) else: self.log.error( - f""" + """ Error: Workflow Run metadata is not populated, so the run was not canceled. This could be due to the workflow not being started or an error in the workflow creation process. """ diff --git a/tests/providers/databricks/hooks/test_databricks_sql.py b/tests/providers/databricks/hooks/test_databricks_sql.py index 0467ad41eb58c..0fdfba4776d07 100644 --- a/tests/providers/databricks/hooks/test_databricks_sql.py +++ b/tests/providers/databricks/hooks/test_databricks_sql.py @@ -19,11 +19,10 @@ from __future__ import annotations import threading -import time from collections import namedtuple from datetime import timedelta from unittest import mock -from unittest.mock import patch, PropertyMock +from unittest.mock import PropertyMock, patch import pytest from databricks.sql.types import Row @@ -33,7 +32,6 @@ from airflow.providers.common.sql.hooks.sql import fetch_all_handler from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook, create_timeout_thread from airflow.utils.session import provide_session -from airflow.utils.timezone import datetime pytestmark = pytest.mark.db_test @@ -59,6 +57,7 @@ def create_connection(session): def databricks_hook(): return DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=True) + @pytest.fixture def mock_get_conn(): # Start the patcher @@ -99,14 +98,17 @@ def mock_get_requests(): # Stop the patcher after the test mock_patch.stop() + @pytest.fixture def mock_timer(): with patch("threading.Timer") as mock_timer: yield mock_timer + def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: return [(field,) for field in fields] + # Serializable Row object similar to the one returned by the Hook SerializableRow = namedtuple("Row", ["id", "value"]) # type: ignore[name-match] @@ -306,42 +308,41 @@ def test_query( mock_get_conn, mock_get_requests, ): - - connections = [] - cursors = [] - for index in range(len(cursor_descriptions)): - conn = mock.MagicMock() - cur = mock.MagicMock( - rowcount=len(cursor_results[index]), - description=get_cursor_descriptions(cursor_descriptions[index]), - ) - cur.fetchall.return_value = cursor_results[index] - conn.cursor.return_value = cur - cursors.append(cur) - connections.append(conn) - mock_get_conn.side_effect = connections - - if not return_tuple: - with pytest.warns( - AirflowProviderDeprecationWarning, - match="""Returning a raw `databricks.sql.Row` object is deprecated. A namedtuple will be + connections = [] + cursors = [] + for index in range(len(cursor_descriptions)): + conn = mock.MagicMock() + cur = mock.MagicMock( + rowcount=len(cursor_results[index]), + description=get_cursor_descriptions(cursor_descriptions[index]), + ) + cur.fetchall.return_value = cursor_results[index] + conn.cursor.return_value = cur + cursors.append(cur) + connections.append(conn) + mock_get_conn.side_effect = connections + + if not return_tuple: + with pytest.warns( + AirflowProviderDeprecationWarning, + match="""Returning a raw `databricks.sql.Row` object is deprecated. A namedtuple will be returned instead in a future release of the databricks provider. Set `return_tuple=True` to enable this behavior.""", - ): - databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) - else: + ): databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) - results = databricks_hook.run( - sql=sql, handler=fetch_all_handler, return_last=return_last, split_statements=split_statements - ) + else: + databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) + results = databricks_hook.run( + sql=sql, handler=fetch_all_handler, return_last=return_last, split_statements=split_statements + ) - assert databricks_hook.descriptions == hook_descriptions - assert databricks_hook.last_description == hook_descriptions[-1] - assert results == hook_results + assert databricks_hook.descriptions == hook_descriptions + assert databricks_hook.last_description == hook_descriptions[-1] + assert results == hook_results - for index, cur in enumerate(cursors): - cur.execute.assert_has_calls([mock.call(cursor_calls[index])]) - cur.close.assert_called() + for index, cur in enumerate(cursors): + cur.execute.assert_has_calls([mock.call(cursor_calls[index])]) + cur.close.assert_called() @pytest.mark.parametrize( @@ -382,14 +383,18 @@ def test_execution_timeout_exceeded( mock_get_requests, sql="select * from test.test", execution_timeout=timedelta(microseconds=0), - cursor_descriptions=["id", "value"], - cursor_results=[Row(id=1, value=2), Row(id=11, value=12)], + cursor_descriptions=( + "id", + "value", + ), + cursor_results=( + Row(id=1, value=2), + Row(id=11, value=12), + ), ): with patch( - 'airflow.providers.databricks.hooks.databricks_sql.create_timeout_thread' - ) as mock_create_timeout_thread, patch.object( - DatabricksSqlHook, '_run_command' - ) as mock_run_command: + "airflow.providers.databricks.hooks.databricks_sql.create_timeout_thread" + ) as mock_create_timeout_thread, patch.object(DatabricksSqlHook, "_run_command") as mock_run_command: conn = mock.MagicMock() cur = mock.MagicMock( rowcount=len(cursor_results), @@ -397,11 +402,9 @@ def test_execution_timeout_exceeded( ) # Simulate a timeout - mock_create_timeout_thread.return_value = threading.Timer( - cur, execution_timeout - ) + mock_create_timeout_thread.return_value = threading.Timer(cur, execution_timeout) - mock_run_command.side_effect = Exception('Mocked exception') + mock_run_command.side_effect = Exception("Mocked exception") cur.fetchall.return_value = cursor_results conn.cursor.return_value = cur @@ -421,28 +424,34 @@ def test_create_timeout_thread( mock_get_conn, mock_get_requests, mock_timer, - cursor_descriptions=["id", "value"], + cursor_descriptions=( + "id", + "value", + ), ): - cur = mock.MagicMock( - rowcount=1, - description=get_cursor_descriptions(cursor_descriptions), - ) - timeout = timedelta(seconds=1) - thread = create_timeout_thread(cur=cur, execution_timeout=timeout) - mock_timer.assert_called_once_with(timeout.total_seconds(), cur.connection.cancel) - assert thread is not None + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + timeout = timedelta(seconds=1) + thread = create_timeout_thread(cur=cur, execution_timeout=timeout) + mock_timer.assert_called_once_with(timeout.total_seconds(), cur.connection.cancel) + assert thread is not None def test_create_timeout_thread_no_timeout( mock_get_conn, mock_get_requests, mock_timer, - cursor_descriptions=["id", "value"], + cursor_descriptions=( + "id", + "value", + ), ): - cur = mock.MagicMock( - rowcount=1, - description=get_cursor_descriptions(cursor_descriptions), - ) - thread = create_timeout_thread(cur=cur, execution_timeout=None) - mock_timer.assert_not_called() - assert thread is None + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + thread = create_timeout_thread(cur=cur, execution_timeout=None) + mock_timer.assert_not_called() + assert thread is None diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/tests/providers/databricks/operators/test_databricks_workflow.py index f0ecb255295d0..fbc429ed1d9a8 100644 --- a/tests/providers/databricks/operators/test_databricks_workflow.py +++ b/tests/providers/databricks/operators/test_databricks_workflow.py @@ -28,9 +28,9 @@ from airflow.providers.databricks.hooks.databricks import RunLifeCycleState from airflow.providers.databricks.operators.databricks_workflow import ( DatabricksWorkflowTaskGroup, + WorkflowRunMetadata, _CreateDatabricksWorkflowOperator, _flatten_node, - WorkflowRunMetadata, ) from airflow.utils import timezone @@ -59,6 +59,7 @@ def mock_task_group(): mock_group.group_id = "test_group" return mock_group + @pytest.fixture def mock_workflow_run_metadata(): return MagicMock(spec=WorkflowRunMetadata) From c31b23d51930a1bd23f5b40be5c2d08f25dea946 Mon Sep 17 00:00:00 2001 From: Lorin Date: Wed, 2 Oct 2024 08:37:34 -0600 Subject: [PATCH 006/199] remove changes for databricks_workflow.py for PR isolated to databricks_sql --- .../operators/databricks_workflow.py | 27 +----------------- .../operators/test_databricks_workflow.py | 28 ++----------------- 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/airflow/providers/databricks/operators/databricks_workflow.py b/airflow/providers/databricks/operators/databricks_workflow.py index 6df8e2d025cea..15333dc69118b 100644 --- a/airflow/providers/databricks/operators/databricks_workflow.py +++ b/airflow/providers/databricks/operators/databricks_workflow.py @@ -52,7 +52,7 @@ class WorkflowRunMetadata: """ conn_id: str - job_id: int + job_id: str run_id: int @@ -116,7 +116,6 @@ def __init__( self.notebook_params = notebook_params or {} self.tasks_to_convert = tasks_to_convert or [] self.relevant_upstreams = [task_id] - self.workflow_run_metadata: WorkflowRunMetadata | None = None super().__init__(task_id=task_id, **kwargs) def _get_hook(self, caller: str) -> DatabricksHook: @@ -213,36 +212,12 @@ def execute(self, context: Context) -> Any: self._wait_for_job_to_start(run_id) - self.workflow_run_metadata = WorkflowRunMetadata( - self.databricks_conn_id, - job_id, - run_id, - ) - return { "conn_id": self.databricks_conn_id, "job_id": job_id, "run_id": run_id, } - def on_kill(self) -> None: - if self.workflow_run_metadata: - run_id = self.workflow_run_metadata.run_id - job_id = self.workflow_run_metadata.job_id - - self._hook.cancel_run(run_id) - self.log.info( - "Run: %(run_id)s of job_id: %(job_id)s was requested to be cancelled.", - {"run_id": run_id, "job_id": job_id}, - ) - else: - self.log.error( - """ - Error: Workflow Run metadata is not populated, so the run was not canceled. This could be due - to the workflow not being started or an error in the workflow creation process. - """ - ) - class DatabricksWorkflowTaskGroup(TaskGroup): """ diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/tests/providers/databricks/operators/test_databricks_workflow.py index fbc429ed1d9a8..99f1a9d14815d 100644 --- a/tests/providers/databricks/operators/test_databricks_workflow.py +++ b/tests/providers/databricks/operators/test_databricks_workflow.py @@ -28,7 +28,6 @@ from airflow.providers.databricks.hooks.databricks import RunLifeCycleState from airflow.providers.databricks.operators.databricks_workflow import ( DatabricksWorkflowTaskGroup, - WorkflowRunMetadata, _CreateDatabricksWorkflowOperator, _flatten_node, ) @@ -60,11 +59,6 @@ def mock_task_group(): return mock_group -@pytest.fixture -def mock_workflow_run_metadata(): - return MagicMock(spec=WorkflowRunMetadata) - - def test_flatten_node(): """Test that _flatten_node returns a flat list of operators.""" task_group = MagicMock(spec=DatabricksWorkflowTaskGroup) @@ -183,7 +177,7 @@ def mock_databricks_workflow_operator(): def test_task_group_initialization(): """Test that DatabricksWorkflowTaskGroup initializes correctly.""" - with DAG(dag_id="example_databricks_workflow_dag", schedule=None, start_date=DEFAULT_DATE) as example_dag: + with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: with DatabricksWorkflowTaskGroup( group_id="test_databricks_workflow", databricks_conn_id="databricks_conn" ) as task_group: @@ -196,7 +190,7 @@ def test_task_group_initialization(): def test_task_group_exit_creates_operator(mock_databricks_workflow_operator): """Test that DatabricksWorkflowTaskGroup creates a _CreateDatabricksWorkflowOperator on exit.""" - with DAG(dag_id="example_databricks_workflow_dag", schedule=None, start_date=DEFAULT_DATE) as example_dag: + with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: with DatabricksWorkflowTaskGroup( group_id="test_databricks_workflow", databricks_conn_id="databricks_conn", @@ -226,7 +220,7 @@ def test_task_group_exit_creates_operator(mock_databricks_workflow_operator): def test_task_group_root_tasks_set_upstream_to_operator(mock_databricks_workflow_operator): """Test that tasks added to a DatabricksWorkflowTaskGroup are set upstream to the operator.""" - with DAG(dag_id="example_databricks_workflow_dag", schedule=None, start_date=DEFAULT_DATE): + with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE): with DatabricksWorkflowTaskGroup( group_id="test_databricks_workflow1", databricks_conn_id="databricks_conn", @@ -237,19 +231,3 @@ def test_task_group_root_tasks_set_upstream_to_operator(mock_databricks_workflow create_operator_instance = mock_databricks_workflow_operator.return_value task1.set_upstream.assert_called_once_with(create_operator_instance) - - -def test_on_kill(mock_databricks_hook, context, mock_workflow_run_metadata): - """Test that _CreateDatabricksWorkflowOperator.execute runs the task group.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.workflow_run_metadata = mock_workflow_run_metadata - - RUN_ID = 789 - - mock_workflow_run_metadata.conn_id = operator.databricks_conn_id - mock_workflow_run_metadata.job_id = "123" - mock_workflow_run_metadata.run_id = RUN_ID - - operator.on_kill() - - operator._hook.cancel_run.assert_called_once_with(RUN_ID) From 0c52cbc8cbb69f8add6cc0c4e96a5bfec8784054 Mon Sep 17 00:00:00 2001 From: Lorin Date: Wed, 2 Oct 2024 08:49:24 -0600 Subject: [PATCH 007/199] remove file accidently picked up rebasing with main --- .../operators/test_databricks_workflow.py | 233 ------------------ 1 file changed, 233 deletions(-) delete mode 100644 tests/providers/databricks/operators/test_databricks_workflow.py diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/tests/providers/databricks/operators/test_databricks_workflow.py deleted file mode 100644 index 99f1a9d14815d..0000000000000 --- a/tests/providers/databricks/operators/test_databricks_workflow.py +++ /dev/null @@ -1,233 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -import pytest - -from airflow import DAG -from airflow.exceptions import AirflowException -from airflow.models.baseoperator import BaseOperator -from airflow.operators.empty import EmptyOperator -from airflow.providers.databricks.hooks.databricks import RunLifeCycleState -from airflow.providers.databricks.operators.databricks_workflow import ( - DatabricksWorkflowTaskGroup, - _CreateDatabricksWorkflowOperator, - _flatten_node, -) -from airflow.utils import timezone - -pytestmark = pytest.mark.db_test - -DEFAULT_DATE = timezone.datetime(2021, 1, 1) - - -@pytest.fixture -def mock_databricks_hook(): - """Provide a mock DatabricksHook.""" - with patch("airflow.providers.databricks.operators.databricks_workflow.DatabricksHook") as mock_hook: - yield mock_hook - - -@pytest.fixture -def context(): - """Provide a mock context object.""" - return MagicMock() - - -@pytest.fixture -def mock_task_group(): - """Provide a mock DatabricksWorkflowTaskGroup with necessary attributes.""" - mock_group = MagicMock(spec=DatabricksWorkflowTaskGroup) - mock_group.group_id = "test_group" - return mock_group - - -def test_flatten_node(): - """Test that _flatten_node returns a flat list of operators.""" - task_group = MagicMock(spec=DatabricksWorkflowTaskGroup) - base_operator = MagicMock(spec=BaseOperator) - task_group.children = {"task1": base_operator, "task2": base_operator} - - result = _flatten_node(task_group) - assert result == [base_operator, base_operator] - - -def test_create_workflow_json(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator.create_workflow_json returns the expected JSON.""" - operator = _CreateDatabricksWorkflowOperator( - task_id="test_task", - databricks_conn_id="databricks_default", - ) - operator.task_group = mock_task_group - - task = MagicMock(spec=BaseOperator) - task._convert_to_databricks_workflow_task = MagicMock(return_value={}) - operator.add_task(task) - - workflow_json = operator.create_workflow_json(context=context) - - assert ".test_group" in workflow_json["name"] - assert "tasks" in workflow_json - assert workflow_json["format"] == "MULTI_TASK" - assert workflow_json["email_notifications"] == {"no_alert_for_skipped_runs": False} - assert workflow_json["job_clusters"] == [] - assert workflow_json["max_concurrent_runs"] == 1 - assert workflow_json["timeout_seconds"] == 0 - - -def test_create_or_reset_job_existing(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job resets the job if it already exists.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = mock_task_group - operator._hook.list_jobs.return_value = [{"job_id": 123}] - operator._hook.create_job.return_value = 123 - - job_id = operator._create_or_reset_job(context) - assert job_id == 123 - operator._hook.reset_job.assert_called_once() - - -def test_create_or_reset_job_new(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job creates a new job if it does not exist.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = mock_task_group - operator._hook.list_jobs.return_value = [] - operator._hook.create_job.return_value = 456 - - job_id = operator._create_or_reset_job(context) - assert job_id == 456 - operator._hook.create_job.assert_called_once() - - -def test_wait_for_job_to_start(mock_databricks_hook): - """Test that _CreateDatabricksWorkflowOperator._wait_for_job_to_start waits for the job to start.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - mock_hook_instance = mock_databricks_hook.return_value - mock_hook_instance.get_run_state.side_effect = [ - MagicMock(life_cycle_state=RunLifeCycleState.PENDING.value), - MagicMock(life_cycle_state=RunLifeCycleState.RUNNING.value), - ] - - operator._wait_for_job_to_start(123) - mock_hook_instance.get_run_state.assert_called() - - -def test_execute(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator.execute runs the task group.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = mock_task_group - mock_task_group.jar_params = {} - mock_task_group.python_params = {} - mock_task_group.spark_submit_params = {} - - mock_hook_instance = mock_databricks_hook.return_value - mock_hook_instance.run_now.return_value = 789 - mock_hook_instance.list_jobs.return_value = [{"job_id": 123}] - mock_hook_instance.get_run_state.return_value = MagicMock( - life_cycle_state=RunLifeCycleState.RUNNING.value - ) - - task = MagicMock(spec=BaseOperator) - task._convert_to_databricks_workflow_task = MagicMock(return_value={}) - operator.add_task(task) - - result = operator.execute(context) - - assert result == { - "conn_id": "databricks_default", - "job_id": 123, - "run_id": 789, - } - mock_hook_instance.run_now.assert_called_once() - - -def test_execute_invalid_task_group(context): - """Test that _CreateDatabricksWorkflowOperator.execute raises an exception if the task group is invalid.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = MagicMock() # Not a DatabricksWorkflowTaskGroup - - with pytest.raises(AirflowException, match="Task group must be a DatabricksWorkflowTaskGroup"): - operator.execute(context) - - -@pytest.fixture -def mock_databricks_workflow_operator(): - with patch( - "airflow.providers.databricks.operators.databricks_workflow._CreateDatabricksWorkflowOperator" - ) as mock_operator: - yield mock_operator - - -def test_task_group_initialization(): - """Test that DatabricksWorkflowTaskGroup initializes correctly.""" - with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: - with DatabricksWorkflowTaskGroup( - group_id="test_databricks_workflow", databricks_conn_id="databricks_conn" - ) as task_group: - task_1 = EmptyOperator(task_id="task1") - task_1._convert_to_databricks_workflow_task = MagicMock(return_value={}) - assert task_group.group_id == "test_databricks_workflow" - assert task_group.databricks_conn_id == "databricks_conn" - assert task_group.dag == example_dag - - -def test_task_group_exit_creates_operator(mock_databricks_workflow_operator): - """Test that DatabricksWorkflowTaskGroup creates a _CreateDatabricksWorkflowOperator on exit.""" - with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: - with DatabricksWorkflowTaskGroup( - group_id="test_databricks_workflow", - databricks_conn_id="databricks_conn", - ) as task_group: - task1 = MagicMock(task_id="task1") - task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) - task2 = MagicMock(task_id="task2") - task2._convert_to_databricks_workflow_task = MagicMock(return_value={}) - - task_group.add(task1) - task_group.add(task2) - - task1.set_downstream(task2) - - mock_databricks_workflow_operator.assert_called_once_with( - dag=example_dag, - task_group=task_group, - task_id="launch", - databricks_conn_id="databricks_conn", - existing_clusters=[], - extra_job_params={}, - job_clusters=[], - max_concurrent_runs=1, - notebook_params={}, - ) - - -def test_task_group_root_tasks_set_upstream_to_operator(mock_databricks_workflow_operator): - """Test that tasks added to a DatabricksWorkflowTaskGroup are set upstream to the operator.""" - with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE): - with DatabricksWorkflowTaskGroup( - group_id="test_databricks_workflow1", - databricks_conn_id="databricks_conn", - ) as task_group: - task1 = MagicMock(task_id="task1") - task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) - task_group.add(task1) - - create_operator_instance = mock_databricks_workflow_operator.return_value - task1.set_upstream.assert_called_once_with(create_operator_instance) From 25b0ba9afc41f6f106f329143efa3fd702be8a27 Mon Sep 17 00:00:00 2001 From: Lorin Date: Wed, 2 Oct 2024 08:50:14 -0600 Subject: [PATCH 008/199] Revert "remove file accidently picked up rebasing with main" This reverts commit 13287fcc63cfe2f3df548c287bd6386007520167. --- .../operators/test_databricks_workflow.py | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 tests/providers/databricks/operators/test_databricks_workflow.py diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/tests/providers/databricks/operators/test_databricks_workflow.py new file mode 100644 index 0000000000000..99f1a9d14815d --- /dev/null +++ b/tests/providers/databricks/operators/test_databricks_workflow.py @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from airflow import DAG +from airflow.exceptions import AirflowException +from airflow.models.baseoperator import BaseOperator +from airflow.operators.empty import EmptyOperator +from airflow.providers.databricks.hooks.databricks import RunLifeCycleState +from airflow.providers.databricks.operators.databricks_workflow import ( + DatabricksWorkflowTaskGroup, + _CreateDatabricksWorkflowOperator, + _flatten_node, +) +from airflow.utils import timezone + +pytestmark = pytest.mark.db_test + +DEFAULT_DATE = timezone.datetime(2021, 1, 1) + + +@pytest.fixture +def mock_databricks_hook(): + """Provide a mock DatabricksHook.""" + with patch("airflow.providers.databricks.operators.databricks_workflow.DatabricksHook") as mock_hook: + yield mock_hook + + +@pytest.fixture +def context(): + """Provide a mock context object.""" + return MagicMock() + + +@pytest.fixture +def mock_task_group(): + """Provide a mock DatabricksWorkflowTaskGroup with necessary attributes.""" + mock_group = MagicMock(spec=DatabricksWorkflowTaskGroup) + mock_group.group_id = "test_group" + return mock_group + + +def test_flatten_node(): + """Test that _flatten_node returns a flat list of operators.""" + task_group = MagicMock(spec=DatabricksWorkflowTaskGroup) + base_operator = MagicMock(spec=BaseOperator) + task_group.children = {"task1": base_operator, "task2": base_operator} + + result = _flatten_node(task_group) + assert result == [base_operator, base_operator] + + +def test_create_workflow_json(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator.create_workflow_json returns the expected JSON.""" + operator = _CreateDatabricksWorkflowOperator( + task_id="test_task", + databricks_conn_id="databricks_default", + ) + operator.task_group = mock_task_group + + task = MagicMock(spec=BaseOperator) + task._convert_to_databricks_workflow_task = MagicMock(return_value={}) + operator.add_task(task) + + workflow_json = operator.create_workflow_json(context=context) + + assert ".test_group" in workflow_json["name"] + assert "tasks" in workflow_json + assert workflow_json["format"] == "MULTI_TASK" + assert workflow_json["email_notifications"] == {"no_alert_for_skipped_runs": False} + assert workflow_json["job_clusters"] == [] + assert workflow_json["max_concurrent_runs"] == 1 + assert workflow_json["timeout_seconds"] == 0 + + +def test_create_or_reset_job_existing(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job resets the job if it already exists.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = mock_task_group + operator._hook.list_jobs.return_value = [{"job_id": 123}] + operator._hook.create_job.return_value = 123 + + job_id = operator._create_or_reset_job(context) + assert job_id == 123 + operator._hook.reset_job.assert_called_once() + + +def test_create_or_reset_job_new(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job creates a new job if it does not exist.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = mock_task_group + operator._hook.list_jobs.return_value = [] + operator._hook.create_job.return_value = 456 + + job_id = operator._create_or_reset_job(context) + assert job_id == 456 + operator._hook.create_job.assert_called_once() + + +def test_wait_for_job_to_start(mock_databricks_hook): + """Test that _CreateDatabricksWorkflowOperator._wait_for_job_to_start waits for the job to start.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + mock_hook_instance = mock_databricks_hook.return_value + mock_hook_instance.get_run_state.side_effect = [ + MagicMock(life_cycle_state=RunLifeCycleState.PENDING.value), + MagicMock(life_cycle_state=RunLifeCycleState.RUNNING.value), + ] + + operator._wait_for_job_to_start(123) + mock_hook_instance.get_run_state.assert_called() + + +def test_execute(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator.execute runs the task group.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = mock_task_group + mock_task_group.jar_params = {} + mock_task_group.python_params = {} + mock_task_group.spark_submit_params = {} + + mock_hook_instance = mock_databricks_hook.return_value + mock_hook_instance.run_now.return_value = 789 + mock_hook_instance.list_jobs.return_value = [{"job_id": 123}] + mock_hook_instance.get_run_state.return_value = MagicMock( + life_cycle_state=RunLifeCycleState.RUNNING.value + ) + + task = MagicMock(spec=BaseOperator) + task._convert_to_databricks_workflow_task = MagicMock(return_value={}) + operator.add_task(task) + + result = operator.execute(context) + + assert result == { + "conn_id": "databricks_default", + "job_id": 123, + "run_id": 789, + } + mock_hook_instance.run_now.assert_called_once() + + +def test_execute_invalid_task_group(context): + """Test that _CreateDatabricksWorkflowOperator.execute raises an exception if the task group is invalid.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = MagicMock() # Not a DatabricksWorkflowTaskGroup + + with pytest.raises(AirflowException, match="Task group must be a DatabricksWorkflowTaskGroup"): + operator.execute(context) + + +@pytest.fixture +def mock_databricks_workflow_operator(): + with patch( + "airflow.providers.databricks.operators.databricks_workflow._CreateDatabricksWorkflowOperator" + ) as mock_operator: + yield mock_operator + + +def test_task_group_initialization(): + """Test that DatabricksWorkflowTaskGroup initializes correctly.""" + with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: + with DatabricksWorkflowTaskGroup( + group_id="test_databricks_workflow", databricks_conn_id="databricks_conn" + ) as task_group: + task_1 = EmptyOperator(task_id="task1") + task_1._convert_to_databricks_workflow_task = MagicMock(return_value={}) + assert task_group.group_id == "test_databricks_workflow" + assert task_group.databricks_conn_id == "databricks_conn" + assert task_group.dag == example_dag + + +def test_task_group_exit_creates_operator(mock_databricks_workflow_operator): + """Test that DatabricksWorkflowTaskGroup creates a _CreateDatabricksWorkflowOperator on exit.""" + with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: + with DatabricksWorkflowTaskGroup( + group_id="test_databricks_workflow", + databricks_conn_id="databricks_conn", + ) as task_group: + task1 = MagicMock(task_id="task1") + task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) + task2 = MagicMock(task_id="task2") + task2._convert_to_databricks_workflow_task = MagicMock(return_value={}) + + task_group.add(task1) + task_group.add(task2) + + task1.set_downstream(task2) + + mock_databricks_workflow_operator.assert_called_once_with( + dag=example_dag, + task_group=task_group, + task_id="launch", + databricks_conn_id="databricks_conn", + existing_clusters=[], + extra_job_params={}, + job_clusters=[], + max_concurrent_runs=1, + notebook_params={}, + ) + + +def test_task_group_root_tasks_set_upstream_to_operator(mock_databricks_workflow_operator): + """Test that tasks added to a DatabricksWorkflowTaskGroup are set upstream to the operator.""" + with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE): + with DatabricksWorkflowTaskGroup( + group_id="test_databricks_workflow1", + databricks_conn_id="databricks_conn", + ) as task_group: + task1 = MagicMock(task_id="task1") + task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) + task_group.add(task1) + + create_operator_instance = mock_databricks_workflow_operator.return_value + task1.set_upstream.assert_called_once_with(create_operator_instance) From 75f8c057fac72f35dc7a8d78fb1e932c1cc83a11 Mon Sep 17 00:00:00 2001 From: Lorin Date: Wed, 2 Oct 2024 09:03:37 -0600 Subject: [PATCH 009/199] remove file from tracking --- .../operators/test_databricks_workflow.py | 233 ------------------ 1 file changed, 233 deletions(-) delete mode 100644 tests/providers/databricks/operators/test_databricks_workflow.py diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/tests/providers/databricks/operators/test_databricks_workflow.py deleted file mode 100644 index 99f1a9d14815d..0000000000000 --- a/tests/providers/databricks/operators/test_databricks_workflow.py +++ /dev/null @@ -1,233 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -import pytest - -from airflow import DAG -from airflow.exceptions import AirflowException -from airflow.models.baseoperator import BaseOperator -from airflow.operators.empty import EmptyOperator -from airflow.providers.databricks.hooks.databricks import RunLifeCycleState -from airflow.providers.databricks.operators.databricks_workflow import ( - DatabricksWorkflowTaskGroup, - _CreateDatabricksWorkflowOperator, - _flatten_node, -) -from airflow.utils import timezone - -pytestmark = pytest.mark.db_test - -DEFAULT_DATE = timezone.datetime(2021, 1, 1) - - -@pytest.fixture -def mock_databricks_hook(): - """Provide a mock DatabricksHook.""" - with patch("airflow.providers.databricks.operators.databricks_workflow.DatabricksHook") as mock_hook: - yield mock_hook - - -@pytest.fixture -def context(): - """Provide a mock context object.""" - return MagicMock() - - -@pytest.fixture -def mock_task_group(): - """Provide a mock DatabricksWorkflowTaskGroup with necessary attributes.""" - mock_group = MagicMock(spec=DatabricksWorkflowTaskGroup) - mock_group.group_id = "test_group" - return mock_group - - -def test_flatten_node(): - """Test that _flatten_node returns a flat list of operators.""" - task_group = MagicMock(spec=DatabricksWorkflowTaskGroup) - base_operator = MagicMock(spec=BaseOperator) - task_group.children = {"task1": base_operator, "task2": base_operator} - - result = _flatten_node(task_group) - assert result == [base_operator, base_operator] - - -def test_create_workflow_json(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator.create_workflow_json returns the expected JSON.""" - operator = _CreateDatabricksWorkflowOperator( - task_id="test_task", - databricks_conn_id="databricks_default", - ) - operator.task_group = mock_task_group - - task = MagicMock(spec=BaseOperator) - task._convert_to_databricks_workflow_task = MagicMock(return_value={}) - operator.add_task(task) - - workflow_json = operator.create_workflow_json(context=context) - - assert ".test_group" in workflow_json["name"] - assert "tasks" in workflow_json - assert workflow_json["format"] == "MULTI_TASK" - assert workflow_json["email_notifications"] == {"no_alert_for_skipped_runs": False} - assert workflow_json["job_clusters"] == [] - assert workflow_json["max_concurrent_runs"] == 1 - assert workflow_json["timeout_seconds"] == 0 - - -def test_create_or_reset_job_existing(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job resets the job if it already exists.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = mock_task_group - operator._hook.list_jobs.return_value = [{"job_id": 123}] - operator._hook.create_job.return_value = 123 - - job_id = operator._create_or_reset_job(context) - assert job_id == 123 - operator._hook.reset_job.assert_called_once() - - -def test_create_or_reset_job_new(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job creates a new job if it does not exist.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = mock_task_group - operator._hook.list_jobs.return_value = [] - operator._hook.create_job.return_value = 456 - - job_id = operator._create_or_reset_job(context) - assert job_id == 456 - operator._hook.create_job.assert_called_once() - - -def test_wait_for_job_to_start(mock_databricks_hook): - """Test that _CreateDatabricksWorkflowOperator._wait_for_job_to_start waits for the job to start.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - mock_hook_instance = mock_databricks_hook.return_value - mock_hook_instance.get_run_state.side_effect = [ - MagicMock(life_cycle_state=RunLifeCycleState.PENDING.value), - MagicMock(life_cycle_state=RunLifeCycleState.RUNNING.value), - ] - - operator._wait_for_job_to_start(123) - mock_hook_instance.get_run_state.assert_called() - - -def test_execute(mock_databricks_hook, context, mock_task_group): - """Test that _CreateDatabricksWorkflowOperator.execute runs the task group.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = mock_task_group - mock_task_group.jar_params = {} - mock_task_group.python_params = {} - mock_task_group.spark_submit_params = {} - - mock_hook_instance = mock_databricks_hook.return_value - mock_hook_instance.run_now.return_value = 789 - mock_hook_instance.list_jobs.return_value = [{"job_id": 123}] - mock_hook_instance.get_run_state.return_value = MagicMock( - life_cycle_state=RunLifeCycleState.RUNNING.value - ) - - task = MagicMock(spec=BaseOperator) - task._convert_to_databricks_workflow_task = MagicMock(return_value={}) - operator.add_task(task) - - result = operator.execute(context) - - assert result == { - "conn_id": "databricks_default", - "job_id": 123, - "run_id": 789, - } - mock_hook_instance.run_now.assert_called_once() - - -def test_execute_invalid_task_group(context): - """Test that _CreateDatabricksWorkflowOperator.execute raises an exception if the task group is invalid.""" - operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") - operator.task_group = MagicMock() # Not a DatabricksWorkflowTaskGroup - - with pytest.raises(AirflowException, match="Task group must be a DatabricksWorkflowTaskGroup"): - operator.execute(context) - - -@pytest.fixture -def mock_databricks_workflow_operator(): - with patch( - "airflow.providers.databricks.operators.databricks_workflow._CreateDatabricksWorkflowOperator" - ) as mock_operator: - yield mock_operator - - -def test_task_group_initialization(): - """Test that DatabricksWorkflowTaskGroup initializes correctly.""" - with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: - with DatabricksWorkflowTaskGroup( - group_id="test_databricks_workflow", databricks_conn_id="databricks_conn" - ) as task_group: - task_1 = EmptyOperator(task_id="task1") - task_1._convert_to_databricks_workflow_task = MagicMock(return_value={}) - assert task_group.group_id == "test_databricks_workflow" - assert task_group.databricks_conn_id == "databricks_conn" - assert task_group.dag == example_dag - - -def test_task_group_exit_creates_operator(mock_databricks_workflow_operator): - """Test that DatabricksWorkflowTaskGroup creates a _CreateDatabricksWorkflowOperator on exit.""" - with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE) as example_dag: - with DatabricksWorkflowTaskGroup( - group_id="test_databricks_workflow", - databricks_conn_id="databricks_conn", - ) as task_group: - task1 = MagicMock(task_id="task1") - task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) - task2 = MagicMock(task_id="task2") - task2._convert_to_databricks_workflow_task = MagicMock(return_value={}) - - task_group.add(task1) - task_group.add(task2) - - task1.set_downstream(task2) - - mock_databricks_workflow_operator.assert_called_once_with( - dag=example_dag, - task_group=task_group, - task_id="launch", - databricks_conn_id="databricks_conn", - existing_clusters=[], - extra_job_params={}, - job_clusters=[], - max_concurrent_runs=1, - notebook_params={}, - ) - - -def test_task_group_root_tasks_set_upstream_to_operator(mock_databricks_workflow_operator): - """Test that tasks added to a DatabricksWorkflowTaskGroup are set upstream to the operator.""" - with DAG(dag_id="example_databricks_workflow_dag", start_date=DEFAULT_DATE): - with DatabricksWorkflowTaskGroup( - group_id="test_databricks_workflow1", - databricks_conn_id="databricks_conn", - ) as task_group: - task1 = MagicMock(task_id="task1") - task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) - task_group.add(task1) - - create_operator_instance = mock_databricks_workflow_operator.return_value - task1.set_upstream.assert_called_once_with(create_operator_instance) From 74dae27d9f8e3fe3ca4c09862c0ba70c312b3a78 Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 3 Oct 2024 11:12:06 -0600 Subject: [PATCH 010/199] adding updated test_databricks_workflow file --- .../operators/test_databricks_workflow.py | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 tests/providers/databricks/operators/test_databricks_workflow.py diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/tests/providers/databricks/operators/test_databricks_workflow.py new file mode 100644 index 0000000000000..4c3f54b800ae9 --- /dev/null +++ b/tests/providers/databricks/operators/test_databricks_workflow.py @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from airflow import DAG +from airflow.exceptions import AirflowException +from airflow.models.baseoperator import BaseOperator +from airflow.operators.empty import EmptyOperator +from airflow.providers.databricks.hooks.databricks import RunLifeCycleState +from airflow.providers.databricks.operators.databricks_workflow import ( + DatabricksWorkflowTaskGroup, + _CreateDatabricksWorkflowOperator, + _flatten_node, +) +from airflow.utils import timezone + +pytestmark = pytest.mark.db_test + +DEFAULT_DATE = timezone.datetime(2021, 1, 1) + + +@pytest.fixture +def mock_databricks_hook(): + """Provide a mock DatabricksHook.""" + with patch("airflow.providers.databricks.operators.databricks_workflow.DatabricksHook") as mock_hook: + yield mock_hook + + +@pytest.fixture +def context(): + """Provide a mock context object.""" + return MagicMock() + + +@pytest.fixture +def mock_task_group(): + """Provide a mock DatabricksWorkflowTaskGroup with necessary attributes.""" + mock_group = MagicMock(spec=DatabricksWorkflowTaskGroup) + mock_group.group_id = "test_group" + return mock_group + + +def test_flatten_node(): + """Test that _flatten_node returns a flat list of operators.""" + task_group = MagicMock(spec=DatabricksWorkflowTaskGroup) + base_operator = MagicMock(spec=BaseOperator) + task_group.children = {"task1": base_operator, "task2": base_operator} + + result = _flatten_node(task_group) + assert result == [base_operator, base_operator] + + +def test_create_workflow_json(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator.create_workflow_json returns the expected JSON.""" + operator = _CreateDatabricksWorkflowOperator( + task_id="test_task", + databricks_conn_id="databricks_default", + ) + operator.task_group = mock_task_group + + task = MagicMock(spec=BaseOperator) + task._convert_to_databricks_workflow_task = MagicMock(return_value={}) + operator.add_task(task) + + workflow_json = operator.create_workflow_json(context=context) + + assert ".test_group" in workflow_json["name"] + assert "tasks" in workflow_json + assert workflow_json["format"] == "MULTI_TASK" + assert workflow_json["email_notifications"] == {"no_alert_for_skipped_runs": False} + assert workflow_json["job_clusters"] == [] + assert workflow_json["max_concurrent_runs"] == 1 + assert workflow_json["timeout_seconds"] == 0 + + +def test_create_or_reset_job_existing(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job resets the job if it already exists.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = mock_task_group + operator._hook.list_jobs.return_value = [{"job_id": 123}] + operator._hook.create_job.return_value = 123 + + job_id = operator._create_or_reset_job(context) + assert job_id == 123 + operator._hook.reset_job.assert_called_once() + + +def test_create_or_reset_job_new(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator._create_or_reset_job creates a new job if it does not exist.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = mock_task_group + operator._hook.list_jobs.return_value = [] + operator._hook.create_job.return_value = 456 + + job_id = operator._create_or_reset_job(context) + assert job_id == 456 + operator._hook.create_job.assert_called_once() + + +def test_wait_for_job_to_start(mock_databricks_hook): + """Test that _CreateDatabricksWorkflowOperator._wait_for_job_to_start waits for the job to start.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + mock_hook_instance = mock_databricks_hook.return_value + mock_hook_instance.get_run_state.side_effect = [ + MagicMock(life_cycle_state=RunLifeCycleState.PENDING.value), + MagicMock(life_cycle_state=RunLifeCycleState.RUNNING.value), + ] + + operator._wait_for_job_to_start(123) + mock_hook_instance.get_run_state.assert_called() + + +def test_execute(mock_databricks_hook, context, mock_task_group): + """Test that _CreateDatabricksWorkflowOperator.execute runs the task group.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = mock_task_group + mock_task_group.jar_params = {} + mock_task_group.python_params = {} + mock_task_group.spark_submit_params = {} + + mock_hook_instance = mock_databricks_hook.return_value + mock_hook_instance.run_now.return_value = 789 + mock_hook_instance.list_jobs.return_value = [{"job_id": 123}] + mock_hook_instance.get_run_state.return_value = MagicMock( + life_cycle_state=RunLifeCycleState.RUNNING.value + ) + + task = MagicMock(spec=BaseOperator) + task._convert_to_databricks_workflow_task = MagicMock(return_value={}) + operator.add_task(task) + + result = operator.execute(context) + + assert result == { + "conn_id": "databricks_default", + "job_id": 123, + "run_id": 789, + } + mock_hook_instance.run_now.assert_called_once() + + +def test_execute_invalid_task_group(context): + """Test that _CreateDatabricksWorkflowOperator.execute raises an exception if the task group is invalid.""" + operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default") + operator.task_group = MagicMock() # Not a DatabricksWorkflowTaskGroup + + with pytest.raises(AirflowException, match="Task group must be a DatabricksWorkflowTaskGroup"): + operator.execute(context) + + +@pytest.fixture +def mock_databricks_workflow_operator(): + with patch( + "airflow.providers.databricks.operators.databricks_workflow._CreateDatabricksWorkflowOperator" + ) as mock_operator: + yield mock_operator + + +def test_task_group_initialization(): + """Test that DatabricksWorkflowTaskGroup initializes correctly.""" + with DAG(dag_id="example_databricks_workflow_dag", schedule=None, start_date=DEFAULT_DATE) as example_dag: + with DatabricksWorkflowTaskGroup( + group_id="test_databricks_workflow", databricks_conn_id="databricks_conn" + ) as task_group: + task_1 = EmptyOperator(task_id="task1") + task_1._convert_to_databricks_workflow_task = MagicMock(return_value={}) + assert task_group.group_id == "test_databricks_workflow" + assert task_group.databricks_conn_id == "databricks_conn" + assert task_group.dag == example_dag + + +def test_task_group_exit_creates_operator(mock_databricks_workflow_operator): + """Test that DatabricksWorkflowTaskGroup creates a _CreateDatabricksWorkflowOperator on exit.""" + with DAG(dag_id="example_databricks_workflow_dag", schedule=None, start_date=DEFAULT_DATE) as example_dag: + with DatabricksWorkflowTaskGroup( + group_id="test_databricks_workflow", + databricks_conn_id="databricks_conn", + ) as task_group: + task1 = MagicMock(task_id="task1") + task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) + task2 = MagicMock(task_id="task2") + task2._convert_to_databricks_workflow_task = MagicMock(return_value={}) + + task_group.add(task1) + task_group.add(task2) + + task1.set_downstream(task2) + + mock_databricks_workflow_operator.assert_called_once_with( + dag=example_dag, + task_group=task_group, + task_id="launch", + databricks_conn_id="databricks_conn", + existing_clusters=[], + extra_job_params={}, + job_clusters=[], + max_concurrent_runs=1, + notebook_params={}, + ) + + +def test_task_group_root_tasks_set_upstream_to_operator(mock_databricks_workflow_operator): + """Test that tasks added to a DatabricksWorkflowTaskGroup are set upstream to the operator.""" + with DAG(dag_id="example_databricks_workflow_dag", schedule=None, start_date=DEFAULT_DATE): + with DatabricksWorkflowTaskGroup( + group_id="test_databricks_workflow1", + databricks_conn_id="databricks_conn", + ) as task_group: + task1 = MagicMock(task_id="task1") + task1._convert_to_databricks_workflow_task = MagicMock(return_value={}) + task_group.add(task1) + + create_operator_instance = mock_databricks_workflow_operator.return_value + task1.set_upstream.assert_called_once_with(create_operator_instance) From fd5c2741542125ec86b427c392163bb6576d1f2f Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 3 Oct 2024 15:03:36 -0600 Subject: [PATCH 011/199] add indent to execution_timeout param in docstring of DatabricksSqlHook.run --- airflow/providers/databricks/hooks/databricks_sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index dc0af83476306..d959705401d81 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -237,8 +237,8 @@ def run( :param handler: The result handler which is called with the result of each statement. :param split_statements: Whether to split a single SQL string into statements and run separately :param return_last: Whether to return result for only last statement or for all after split - :param execution_timeout: max time allowed for the execution of - this task instance, if it goes beyond it will raise and fail. + :param execution_timeout: max time allowed for the execution of this task instance, if it goes beyond + it will raise and fail. :return: return only result of the LAST SQL expression if handler was provided unless return_last is set to False. """ From 21bbe89329f53d222ba2432a745c8ed705b59efc Mon Sep 17 00:00:00 2001 From: Lorin Dawson <22798188+R7L208@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:36:02 -0600 Subject: [PATCH 012/199] Update airflow/providers/databricks/hooks/databricks_sql.py Co-authored-by: Wei Lee --- airflow/providers/databricks/hooks/databricks_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index d959705401d81..9df20a8166a42 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -267,7 +267,7 @@ def run( with closing(conn.cursor()) as cur: t = create_timeout_thread(cur, execution_timeout) - # todo; adjust this to make testing easier + # TODO: adjust this to make testing easier try: self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: From 619e8f8a44317fefd83717404db00d159c3ba557 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 11:40:45 -0600 Subject: [PATCH 013/199] add new exceptions --- airflow/exceptions.py | 5 +++++ airflow/providers/databricks/hooks/databricks_sql.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 55dd02fdae313..08bf8f8c404ae 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -446,6 +446,11 @@ def __repr__(self) -> str: class TaskDeferralError(AirflowException): """Raised when a task failed during deferral for some reason.""" +class AirflowTaskExecutionError(AirflowException): + """Raised when there is an error in task execution.""" + +class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): + """Raised when a task execution times out.""" # The try/except handling is needed after we moved all k8s classes to cncf.kubernetes provider # These two exceptions are used internally by Kubernetes Executor but also by PodGenerator, so we need diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 9df20a8166a42..a9d9933cba0a8 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -37,7 +37,7 @@ from databricks import sql # type: ignore[attr-defined] -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook @@ -272,9 +272,11 @@ def run( self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: if t is None or t.is_alive(): - raise AirflowException(f"Error running statement: {sql_statement}. {str(e)}") + raise AirflowTaskExecutionError( + f"Error running statement: {sql_statement}. {str(e)}" + ) else: - raise AirflowException( + raise AirflowTaskExecutionTimeout( f"Timeout threshold exceeded for query: {sql_statement} was cancelled." ) finally: From a671901253958df8c1e4737d332d577f6cc8de72 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 12:28:20 -0600 Subject: [PATCH 014/199] add new exceptions --- airflow/exceptions.py | 3 +++ airflow/providers/databricks/hooks/databricks_sql.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 08bf8f8c404ae..628fdac3ce178 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -446,12 +446,15 @@ def __repr__(self) -> str: class TaskDeferralError(AirflowException): """Raised when a task failed during deferral for some reason.""" + class AirflowTaskExecutionError(AirflowException): """Raised when there is an error in task execution.""" + class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): """Raised when a task execution times out.""" + # The try/except handling is needed after we moved all k8s classes to cncf.kubernetes provider # These two exceptions are used internally by Kubernetes Executor but also by PodGenerator, so we need # to leave them here in case older version of cncf.kubernetes provider is used to run KubernetesPodOperator diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index a9d9933cba0a8..0308c307c7b93 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -37,7 +37,12 @@ from databricks import sql # type: ignore[attr-defined] -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowTaskExecutionError, AirflowTaskExecutionTimeout +from airflow.exceptions import ( + AirflowException, + AirflowProviderDeprecationWarning, + AirflowTaskExecutionError, + AirflowTaskExecutionTimeout, +) from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook From 991acf6761c33980e76f2ed19df3ef7b34094cbe Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 12:46:42 -0600 Subject: [PATCH 015/199] fix typing warning --- airflow/providers/databricks/hooks/databricks_sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 0308c307c7b93..13f67a26a14e0 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -33,6 +33,7 @@ TypeVar, cast, overload, + Union, ) from databricks import sql # type: ignore[attr-defined] @@ -56,7 +57,7 @@ T = TypeVar("T") -def create_timeout_thread(cur, execution_timeout: timedelta | None) -> threading.Timer | None: +def create_timeout_thread(cur, execution_timeout: Union[timedelta, None]) -> Union[threading.Timer, None]: if execution_timeout is not None: seconds_to_timeout = execution_timeout.total_seconds() t = threading.Timer(seconds_to_timeout, cur.connection.cancel) From 107392d044939a70d86353a43141d27e969cad09 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 17:02:56 -0600 Subject: [PATCH 016/199] fix ruff error --- airflow/providers/databricks/hooks/databricks_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 13f67a26a14e0..23ef712bd1f6e 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -57,7 +57,7 @@ T = TypeVar("T") -def create_timeout_thread(cur, execution_timeout: Union[timedelta, None]) -> Union[threading.Timer, None]: +def create_timeout_thread(cur, execution_timeout: timedelta | None) -> threading.Timer | None: if execution_timeout is not None: seconds_to_timeout = execution_timeout.total_seconds() t = threading.Timer(seconds_to_timeout, cur.connection.cancel) From 282bc3548273ed02442bfef9c126294b48008fce Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 17:24:36 -0600 Subject: [PATCH 017/199] fix ruff error --- airflow/providers/databricks/hooks/databricks_sql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 23ef712bd1f6e..0308c307c7b93 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -33,7 +33,6 @@ TypeVar, cast, overload, - Union, ) from databricks import sql # type: ignore[attr-defined] From 2bba888f2275412d98e773d5798b7bfdf2aedf0f Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 09:46:53 -0600 Subject: [PATCH 018/199] move new exceptions to Databricks provider --- airflow/exceptions.py | 8 ----- airflow/providers/databricks/exceptions.py | 29 +++++++++++++++++++ .../databricks/hooks/databricks_sql.py | 3 +- 3 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 airflow/providers/databricks/exceptions.py diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 628fdac3ce178..55dd02fdae313 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -447,14 +447,6 @@ class TaskDeferralError(AirflowException): """Raised when a task failed during deferral for some reason.""" -class AirflowTaskExecutionError(AirflowException): - """Raised when there is an error in task execution.""" - - -class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): - """Raised when a task execution times out.""" - - # The try/except handling is needed after we moved all k8s classes to cncf.kubernetes provider # These two exceptions are used internally by Kubernetes Executor but also by PodGenerator, so we need # to leave them here in case older version of cncf.kubernetes provider is used to run KubernetesPodOperator diff --git a/airflow/providers/databricks/exceptions.py b/airflow/providers/databricks/exceptions.py new file mode 100644 index 0000000000000..5dc885aeabf9d --- /dev/null +++ b/airflow/providers/databricks/exceptions.py @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Note: Any AirflowException raised is expected to cause the TaskInstance +# to be marked in an ERROR state +"""Exceptions used by Databricks Provider.""" + +from airflow.exceptions import AirflowException + +class AirflowTaskExecutionError(AirflowException): + """Raised when there is an error in task execution.""" + + +class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): + """Raised when a task execution times out.""" diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 0308c307c7b93..7ebbbe4144ff7 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -40,9 +40,8 @@ from airflow.exceptions import ( AirflowException, AirflowProviderDeprecationWarning, - AirflowTaskExecutionError, - AirflowTaskExecutionTimeout, ) +from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook From c1f64a6b84f62a40987ef4550224ae4b9a69e228 Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 10:19:00 -0600 Subject: [PATCH 019/199] ruff formatting --- airflow/providers/databricks/exceptions.py | 3 +++ airflow/providers/databricks/hooks/databricks_sql.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airflow/providers/databricks/exceptions.py b/airflow/providers/databricks/exceptions.py index 5dc885aeabf9d..aac6556fe113e 100644 --- a/airflow/providers/databricks/exceptions.py +++ b/airflow/providers/databricks/exceptions.py @@ -19,8 +19,11 @@ # to be marked in an ERROR state """Exceptions used by Databricks Provider.""" +from __future__ import annotations + from airflow.exceptions import AirflowException + class AirflowTaskExecutionError(AirflowException): """Raised when there is an error in task execution.""" diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 7ebbbe4144ff7..e093e1d133c2d 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -41,8 +41,8 @@ AirflowException, AirflowProviderDeprecationWarning, ) -from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results +from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook if TYPE_CHECKING: From fecee5287e8eebc340c83f76eb69744b42ef5010 Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 12:22:46 -0600 Subject: [PATCH 020/199] add tests for new exceptions --- tests/providers/databricks/test_exceptions.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/providers/databricks/test_exceptions.py diff --git a/tests/providers/databricks/test_exceptions.py b/tests/providers/databricks/test_exceptions.py new file mode 100644 index 0000000000000..747314e0301e5 --- /dev/null +++ b/tests/providers/databricks/test_exceptions.py @@ -0,0 +1,14 @@ +import pytest +from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout + + +def test_airflow_task_execution_error(): + """Test if AirflowTaskExecutionError can be raised correctly.""" + with pytest.raises(AirflowTaskExecutionError, match="Task execution failed"): + raise AirflowTaskExecutionError("Task execution failed") + + +def test_airflow_task_execution_timeout(): + """Test if AirflowTaskExecutionTimeout can be raised correctly.""" + with pytest.raises(AirflowTaskExecutionTimeout, match="Task execution timed out"): + raise AirflowTaskExecutionTimeout("Task execution timed out") From e6d2e8a838452b082b727aecf780d10cf772c478 Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 14:53:24 -0600 Subject: [PATCH 021/199] chore - static checks --- tests/providers/databricks/test_exceptions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/providers/databricks/test_exceptions.py b/tests/providers/databricks/test_exceptions.py index 747314e0301e5..71b36dff2bb96 100644 --- a/tests/providers/databricks/test_exceptions.py +++ b/tests/providers/databricks/test_exceptions.py @@ -1,4 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + import pytest + from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout From 10e92c479ebed49296d7d33992b8a6bfef6c5f98 Mon Sep 17 00:00:00 2001 From: Lorin Dawson <22798188+R7L208@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:51:21 -0600 Subject: [PATCH 022/199] Update airflow/providers/databricks/hooks/databricks_sql.py Co-authored-by: Wei Lee --- airflow/providers/databricks/hooks/databricks_sql.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index e093e1d133c2d..dca0c7c452231 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -279,10 +279,9 @@ def run( raise AirflowTaskExecutionError( f"Error running statement: {sql_statement}. {str(e)}" ) - else: - raise AirflowTaskExecutionTimeout( - f"Timeout threshold exceeded for query: {sql_statement} was cancelled." - ) + raise AirflowTaskExecutionTimeout( + f"Timeout threshold exceeded for query: {sql_statement} was cancelled." + ) finally: if t is not None: t.cancel() From 09ac2531a6154584436dd6ddfaa1c4651a5f1536 Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 17 Oct 2024 13:35:26 -0600 Subject: [PATCH 023/199] make new exceptions databricks specific --- airflow/providers/databricks/exceptions.py | 8 ++++---- .../providers/databricks/hooks/databricks_sql.py | 10 +++++----- tests/providers/databricks/test_exceptions.py | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/airflow/providers/databricks/exceptions.py b/airflow/providers/databricks/exceptions.py index aac6556fe113e..0488c975ed150 100644 --- a/airflow/providers/databricks/exceptions.py +++ b/airflow/providers/databricks/exceptions.py @@ -24,9 +24,9 @@ from airflow.exceptions import AirflowException -class AirflowTaskExecutionError(AirflowException): - """Raised when there is an error in task execution.""" +class DatabricksSqlExecutionError(AirflowException): + """Raised when there is an error in sql execution.""" -class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): - """Raised when a task execution times out.""" +class DatabricksSqlExecutionTimeout(DatabricksSqlExecutionError): + """Raised when a sql execution times out.""" diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index dca0c7c452231..61975045ac61d 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -42,7 +42,7 @@ AirflowProviderDeprecationWarning, ) from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results -from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout +from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook if TYPE_CHECKING: @@ -276,11 +276,11 @@ def run( self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: if t is None or t.is_alive(): - raise AirflowTaskExecutionError( - f"Error running statement: {sql_statement}. {str(e)}" + raise DatabricksSqlExecutionError( + f"Error running SQL statement: {sql_statement}. {str(e)}" ) - raise AirflowTaskExecutionTimeout( - f"Timeout threshold exceeded for query: {sql_statement} was cancelled." + raise DatabricksSqlExecutionTimeout( + f"Timeout threshold exceeded for SQL statement: {sql_statement} was cancelled." ) finally: if t is not None: diff --git a/tests/providers/databricks/test_exceptions.py b/tests/providers/databricks/test_exceptions.py index 71b36dff2bb96..17c42af474694 100644 --- a/tests/providers/databricks/test_exceptions.py +++ b/tests/providers/databricks/test_exceptions.py @@ -18,16 +18,16 @@ import pytest -from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout +from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout -def test_airflow_task_execution_error(): +def test_databricks_sql_execution_error(): """Test if AirflowTaskExecutionError can be raised correctly.""" - with pytest.raises(AirflowTaskExecutionError, match="Task execution failed"): - raise AirflowTaskExecutionError("Task execution failed") + with pytest.raises(DatabricksSqlExecutionError, match="Task execution failed"): + raise DatabricksSqlExecutionError("Task execution failed") -def test_airflow_task_execution_timeout(): +def test_databricks_sql_execution_timeout(): """Test if AirflowTaskExecutionTimeout can be raised correctly.""" - with pytest.raises(AirflowTaskExecutionTimeout, match="Task execution timed out"): - raise AirflowTaskExecutionTimeout("Task execution timed out") + with pytest.raises(DatabricksSqlExecutionTimeout, match="Task execution timed out"): + raise DatabricksSqlExecutionTimeout("Task execution timed out") From c629c6d1728ee00d75c81d0178d604c83322f90a Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Tue, 8 Oct 2024 22:54:58 +0100 Subject: [PATCH 024/199] Simplify code for recent dbt provider change (#42840) Just simplified redundant code from https://github.com/apache/airflow/pull/42737 --- airflow/providers/dbt/cloud/hooks/dbt.py | 30 +++++++++--------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/airflow/providers/dbt/cloud/hooks/dbt.py b/airflow/providers/dbt/cloud/hooks/dbt.py index b13e1003b9c6f..4007054be69c2 100644 --- a/airflow/providers/dbt/cloud/hooks/dbt.py +++ b/airflow/providers/dbt/cloud/hooks/dbt.py @@ -244,26 +244,18 @@ async def get_job_details( endpoint = f"{account_id}/runs/{run_id}/" headers, tenant = await self.get_headers_tenants_from_connection() url, params = self.get_request_url_params(tenant, endpoint, include_related) - proxies = self._get_proxies(self.connection) + proxies = self._get_proxies(self.connection) or {} + async with aiohttp.ClientSession(headers=headers) as session: - if proxies is not None: - if url.startswith("https"): - proxy = proxies.get("https") - else: - proxy = proxies.get("http") - async with session.get(url, params=params, proxy=proxy) as response: - try: - response.raise_for_status() - return await response.json() - except aiohttp.ClientResponseError as e: - raise AirflowException(f"{e.status}:{e.message}") - else: - async with session.get(url, params=params) as response: - try: - response.raise_for_status() - return await response.json() - except aiohttp.ClientResponseError as e: - raise AirflowException(f"{e.status}:{e.message}") + proxy = proxies.get("https") if proxies and url.startswith("https") else proxies.get("http") + extra_request_args = {} + + if proxy: + extra_request_args["proxy"] = proxy + + async with session.get(url, params=params, **extra_request_args) as response: # type: ignore[arg-type] + response.raise_for_status() + return await response.json() async def get_job_status( self, run_id: int, account_id: int | None = None, include_related: list[str] | None = None From 465553d2108f94e1affdbe11fa10ac0bcaaacd09 Mon Sep 17 00:00:00 2001 From: harjeevan maan Date: Tue, 8 Oct 2024 20:09:26 -0400 Subject: [PATCH 025/199] - Fixes #42432 (#42770) - Added a unit test --- airflow/utils/db.py | 23 +++++++++++++---------- tests/utils/test_db.py | 9 +++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/airflow/utils/db.py b/airflow/utils/db.py index c185c70a9811f..fde641fa9b424 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -1203,19 +1203,22 @@ def resetdb(session: Session = NEW_SESSION, skip_init: bool = False): if not settings.engine: raise RuntimeError("The settings.engine must be set. This is a critical assertion") log.info("Dropping tables that exist") + original_logging_level = logging.root.level + try: + import_all_models() - import_all_models() - - connection = settings.engine.connect() + connection = settings.engine.connect() - with create_global_lock(session=session, lock=DBLocks.MIGRATIONS), connection.begin(): - drop_airflow_models(connection) - drop_airflow_moved_tables(connection) - external_db_manager = RunDBManager() - external_db_manager.drop_tables(session, connection) + with create_global_lock(session=session, lock=DBLocks.MIGRATIONS), connection.begin(): + drop_airflow_models(connection) + drop_airflow_moved_tables(connection) + external_db_manager = RunDBManager() + external_db_manager.drop_tables(session, connection) - if not skip_init: - initdb(session=session) + if not skip_init: + initdb(session=session) + finally: + logging.root.setLevel(original_logging_level) @provide_session diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index 2a197c2e6cf72..ce77f80297fcf 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -18,6 +18,7 @@ from __future__ import annotations import inspect +import logging import os import re from contextlib import redirect_stdout @@ -229,6 +230,14 @@ def test_resetdb( else: mock_init.assert_called_once_with(session=session_mock) + def test_resetdb_logging_level(self): + unset_logging_level = logging.root.level + logging.root.setLevel(logging.DEBUG) + set_logging_level = logging.root.level + resetdb() + assert logging.root.level == set_logging_level + assert logging.root.level != unset_logging_level + def test_alembic_configuration(self): with mock.patch.dict( os.environ, {"AIRFLOW__DATABASE__ALEMBIC_INI_FILE_PATH": "/tmp/alembic.ini"}, clear=True From ea3c1f06ec2cbad700b9c0c99cf86cd380ba7df3 Mon Sep 17 00:00:00 2001 From: Bugra Ozturk Date: Wed, 9 Oct 2024 10:09:40 +0200 Subject: [PATCH 026/199] AIP-84 Migrate views /object/historical_metrics_data to Fast API (#42629) * Include object router and migrate objects/historical_metrics_data to FastAPI * Use pyfixture session * Include provide_session for SQLite session thread issue * Make method async and remove unused session from tests * Include return type to generate proper openapi spec * Object definition to Dashboard definition, make safe date Annotated parameter in parameters.py * Include pydantic model for Dashboard Historical Metric Data and convert response to HistoricalMetricDataResponse * Fix conflict and rerun pre-commit hooks * Rename data variable to historical metrics for consistency * Include object router and migrate objects/historical_metrics_data to FastAPI * Object definition to Dashboard definition, make safe date Annotated parameter in parameters.py * Drop data from method name to prevent double Data in the name of objects * Variable name change to prevent shadow naming with method name * Resolve conflicts and rebase again * Resolve conflicts and rebase --- airflow/api_fastapi/openapi/v1-generated.yaml | 156 ++++++++++++++++++ airflow/api_fastapi/parameters.py | 22 +++ airflow/api_fastapi/serializers/dashboard.py | 63 +++++++ airflow/api_fastapi/views/ui/__init__.py | 2 + airflow/api_fastapi/views/ui/dashboard.py | 100 +++++++++++ airflow/ui/openapi-gen/queries/common.ts | 23 +++ airflow/ui/openapi-gen/queries/prefetch.ts | 27 +++ airflow/ui/openapi-gen/queries/queries.ts | 38 ++++- airflow/ui/openapi-gen/queries/suspense.ts | 34 ++++ .../ui/openapi-gen/requests/schemas.gen.ts | 143 ++++++++++++++++ .../ui/openapi-gen/requests/services.gen.ts | 30 ++++ airflow/ui/openapi-gen/requests/types.gen.ts | 74 +++++++++ airflow/www/views.py | 1 + tests/api_fastapi/views/ui/test_dashboard.py | 151 +++++++++++++++++ 14 files changed, 862 insertions(+), 2 deletions(-) create mode 100644 airflow/api_fastapi/serializers/dashboard.py create mode 100644 airflow/api_fastapi/views/ui/dashboard.py create mode 100644 tests/api_fastapi/views/ui/test_dashboard.py diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index f3bc8612bfa76..28723b800879e 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -34,6 +34,45 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /ui/dashboard/historical_metrics_data: + get: + tags: + - Dashboard + summary: Historical Metrics + description: Return cluster activity historical metrics. + operationId: historical_metrics + parameters: + - name: start_date + in: query + required: true + schema: + type: string + title: Start Date + - name: end_date + in: query + required: true + schema: + type: string + title: End Date + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/HistoricalMetricDataResponse' + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Bad Request + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /public/dags/: get: tags: @@ -964,6 +1003,50 @@ components: - file_token title: DAGResponse description: DAG serializer for responses. + DAGRunStates: + properties: + queued: + type: integer + title: Queued + running: + type: integer + title: Running + success: + type: integer + title: Success + failed: + type: integer + title: Failed + type: object + required: + - queued + - running + - success + - failed + title: DAGRunStates + description: DAG Run States for responses. + DAGRunTypes: + properties: + backfill: + type: integer + title: Backfill + scheduled: + type: integer + title: Scheduled + manual: + type: integer + title: Manual + dataset_triggered: + type: integer + title: Dataset Triggered + type: object + required: + - backfill + - scheduled + - manual + - dataset_triggered + title: DAGRunTypes + description: DAG Run Types for responses. DagRunState: type: string enum: @@ -1016,6 +1099,79 @@ components: title: Detail type: object title: HTTPValidationError + HistoricalMetricDataResponse: + properties: + dag_run_types: + $ref: '#/components/schemas/DAGRunTypes' + dag_run_states: + $ref: '#/components/schemas/DAGRunStates' + task_instance_states: + $ref: '#/components/schemas/TaskInstantState' + type: object + required: + - dag_run_types + - dag_run_states + - task_instance_states + title: HistoricalMetricDataResponse + description: Historical Metric Data serializer for responses. + TaskInstantState: + properties: + no_status: + type: integer + title: No Status + removed: + type: integer + title: Removed + scheduled: + type: integer + title: Scheduled + queued: + type: integer + title: Queued + running: + type: integer + title: Running + success: + type: integer + title: Success + restarting: + type: integer + title: Restarting + failed: + type: integer + title: Failed + up_for_retry: + type: integer + title: Up For Retry + up_for_reschedule: + type: integer + title: Up For Reschedule + upstream_failed: + type: integer + title: Upstream Failed + skipped: + type: integer + title: Skipped + deferred: + type: integer + title: Deferred + type: object + required: + - no_status + - removed + - scheduled + - queued + - running + - success + - restarting + - failed + - up_for_retry + - up_for_reschedule + - upstream_failed + - skipped + - deferred + title: TaskInstantState + description: TaskInstance serializer for responses. ValidationError: properties: loc: diff --git a/airflow/api_fastapi/parameters.py b/airflow/api_fastapi/parameters.py index 504014602f3b5..59d61ad68600d 100644 --- a/airflow/api_fastapi/parameters.py +++ b/airflow/api_fastapi/parameters.py @@ -18,14 +18,18 @@ from __future__ import annotations from abc import ABC, abstractmethod +from datetime import datetime from typing import TYPE_CHECKING, Any, Generic, List, TypeVar from fastapi import Depends, HTTPException, Query +from pendulum.parsing.exceptions import ParserError +from pydantic import AfterValidator from sqlalchemy import case, or_ from typing_extensions import Annotated, Self from airflow.models.dag import DagModel, DagTag from airflow.models.dagrun import DagRun +from airflow.utils import timezone from airflow.utils.state import DagRunState if TYPE_CHECKING: @@ -235,6 +239,24 @@ def depends(self, last_dag_run_state: DagRunState | None = None) -> _LastDagRunS return self.set_value(last_dag_run_state) +def _safe_parse_datetime(date_to_check: str) -> datetime: + """ + Parse datetime and raise error for invalid dates. + + :param date_to_check: the string value to be parsed + """ + if not date_to_check: + raise ValueError(f"{date_to_check} cannot be None.") + try: + return timezone.parse(date_to_check, strict=True) + except (TypeError, ParserError): + raise HTTPException( + 400, f"Invalid datetime: {date_to_check!r}. Please check the date parameter have this value." + ) + + +# Common Safe DateTime +DateTimeQuery = Annotated[str, AfterValidator(_safe_parse_datetime)] # DAG QueryLimit = Annotated[_LimitFilter, Depends(_LimitFilter().depends)] QueryOffset = Annotated[_OffsetFilter, Depends(_OffsetFilter().depends)] diff --git a/airflow/api_fastapi/serializers/dashboard.py b/airflow/api_fastapi/serializers/dashboard.py new file mode 100644 index 0000000000000..f5a38fa22ea87 --- /dev/null +++ b/airflow/api_fastapi/serializers/dashboard.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pydantic import BaseModel + + +class DAGRunTypes(BaseModel): + """DAG Run Types for responses.""" + + backfill: int + scheduled: int + manual: int + dataset_triggered: int + + +class DAGRunStates(BaseModel): + """DAG Run States for responses.""" + + queued: int + running: int + success: int + failed: int + + +class TaskInstantState(BaseModel): + """TaskInstance serializer for responses.""" + + no_status: int + removed: int + scheduled: int + queued: int + running: int + success: int + restarting: int + failed: int + up_for_retry: int + up_for_reschedule: int + upstream_failed: int + skipped: int + deferred: int + + +class HistoricalMetricDataResponse(BaseModel): + """Historical Metric Data serializer for responses.""" + + dag_run_types: DAGRunTypes + dag_run_states: DAGRunStates + task_instance_states: TaskInstantState diff --git a/airflow/api_fastapi/views/ui/__init__.py b/airflow/api_fastapi/views/ui/__init__.py index 8495ac5e5e6a4..f01686cc99ee2 100644 --- a/airflow/api_fastapi/views/ui/__init__.py +++ b/airflow/api_fastapi/views/ui/__init__.py @@ -18,7 +18,9 @@ from airflow.api_fastapi.views.router import AirflowRouter from airflow.api_fastapi.views.ui.assets import assets_router +from airflow.api_fastapi.views.ui.dashboard import dashboard_router ui_router = AirflowRouter(prefix="/ui") ui_router.include_router(assets_router) +ui_router.include_router(dashboard_router) diff --git a/airflow/api_fastapi/views/ui/dashboard.py b/airflow/api_fastapi/views/ui/dashboard.py new file mode 100644 index 0000000000000..0d6b69a1ced5b --- /dev/null +++ b/airflow/api_fastapi/views/ui/dashboard.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fastapi import Depends +from sqlalchemy import func, select +from sqlalchemy.orm import Session +from typing_extensions import Annotated + +from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.parameters import DateTimeQuery +from airflow.api_fastapi.serializers.dashboard import HistoricalMetricDataResponse +from airflow.models.dagrun import DagRun, DagRunType +from airflow.models.taskinstance import TaskInstance +from airflow.utils.state import DagRunState, TaskInstanceState + +if TYPE_CHECKING: + from sqlalchemy.orm import Session +from airflow.api_fastapi.db.common import get_session +from airflow.api_fastapi.views.router import AirflowRouter +from airflow.utils import timezone + +dashboard_router = AirflowRouter(tags=["Dashboard"]) + + +@dashboard_router.get( + "/dashboard/historical_metrics_data", + include_in_schema=False, + responses=create_openapi_http_exception_doc([400]), +) +async def historical_metrics( + start_date: DateTimeQuery, + end_date: DateTimeQuery, + session: Annotated[Session, Depends(get_session)], +) -> HistoricalMetricDataResponse: + """Return cluster activity historical metrics.""" + # DagRuns + dag_run_types = session.execute( + select(DagRun.run_type, func.count(DagRun.run_id)) + .where( + DagRun.start_date >= start_date, + func.coalesce(DagRun.end_date, timezone.utcnow()) <= end_date, + ) + .group_by(DagRun.run_type) + ).all() + + dag_run_states = session.execute( + select(DagRun.state, func.count(DagRun.run_id)) + .where( + DagRun.start_date >= start_date, + func.coalesce(DagRun.end_date, timezone.utcnow()) <= end_date, + ) + .group_by(DagRun.state) + ).all() + + # TaskInstances + task_instance_states = session.execute( + select(TaskInstance.state, func.count(TaskInstance.run_id)) + .join(TaskInstance.dag_run) + .where( + DagRun.start_date >= start_date, + func.coalesce(DagRun.end_date, timezone.utcnow()) <= end_date, + ) + .group_by(TaskInstance.state) + ).all() + + # Combining historical metrics response as dictionary + historical_metrics_response = { + "dag_run_types": { + **{dag_run_type.value: 0 for dag_run_type in DagRunType}, + **dict(dag_run_types), + }, + "dag_run_states": { + **{dag_run_state.value: 0 for dag_run_state in DagRunState}, + **dict(dag_run_states), + }, + "task_instance_states": { + "no_status": 0, + **{ti_state.value: 0 for ti_state in TaskInstanceState}, + **{ti_state or "no_status": sum_value for ti_state, sum_value in task_instance_states}, + }, + } + + return HistoricalMetricDataResponse.model_validate(historical_metrics_response, from_attributes=True) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index afd7afd5bcc06..fbbbac5d60625 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -5,6 +5,7 @@ import { AssetService, ConnectionService, DagService, + DashboardService, VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; @@ -25,6 +26,28 @@ export const UseAssetServiceNextRunAssetsKeyFn = ( }, queryKey?: Array, ) => [useAssetServiceNextRunAssetsKey, ...(queryKey ?? [{ dagId }])]; +export type DashboardServiceHistoricalMetricsDefaultResponse = Awaited< + ReturnType +>; +export type DashboardServiceHistoricalMetricsQueryResult< + TData = DashboardServiceHistoricalMetricsDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useDashboardServiceHistoricalMetricsKey = + "DashboardServiceHistoricalMetrics"; +export const UseDashboardServiceHistoricalMetricsKeyFn = ( + { + endDate, + startDate, + }: { + endDate: string; + startDate: string; + }, + queryKey?: Array, +) => [ + useDashboardServiceHistoricalMetricsKey, + ...(queryKey ?? [{ endDate, startDate }]), +]; export type DagServiceGetDagsDefaultResponse = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index cbb43cca3abb1..7c8555b29d264 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -5,6 +5,7 @@ import { AssetService, ConnectionService, DagService, + DashboardService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; import * as Common from "./common"; @@ -28,6 +29,32 @@ export const prefetchUseAssetServiceNextRunAssets = ( queryKey: Common.UseAssetServiceNextRunAssetsKeyFn({ dagId }), queryFn: () => AssetService.nextRunAssets({ dagId }), }); +/** + * Historical Metrics + * Return cluster activity historical metrics. + * @param data The data for the request. + * @param data.startDate + * @param data.endDate + * @returns HistoricalMetricDataResponse Successful Response + * @throws ApiError + */ +export const prefetchUseDashboardServiceHistoricalMetrics = ( + queryClient: QueryClient, + { + endDate, + startDate, + }: { + endDate: string; + startDate: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseDashboardServiceHistoricalMetricsKeyFn({ + endDate, + startDate, + }), + queryFn: () => DashboardService.historicalMetrics({ endDate, startDate }), + }); /** * Get Dags * Get all DAGs. diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 22d58eadda0cb..9137ea4ed01bc 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -1,15 +1,16 @@ // generated with @7nohe/openapi-react-query-codegen@1.6.0 import { - useMutation, UseMutationOptions, - useQuery, UseQueryOptions, + useMutation, + useQuery, } from "@tanstack/react-query"; import { AssetService, ConnectionService, DagService, + DashboardService, VariableService, } from "../requests/services.gen"; import { DAGPatchBody, DagRunState } from "../requests/types.gen"; @@ -40,6 +41,39 @@ export const useAssetServiceNextRunAssets = < queryFn: () => AssetService.nextRunAssets({ dagId }) as TData, ...options, }); +/** + * Historical Metrics + * Return cluster activity historical metrics. + * @param data The data for the request. + * @param data.startDate + * @param data.endDate + * @returns HistoricalMetricDataResponse Successful Response + * @throws ApiError + */ +export const useDashboardServiceHistoricalMetrics = < + TData = Common.DashboardServiceHistoricalMetricsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + endDate, + startDate, + }: { + endDate: string; + startDate: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseDashboardServiceHistoricalMetricsKeyFn( + { endDate, startDate }, + queryKey, + ), + queryFn: () => + DashboardService.historicalMetrics({ endDate, startDate }) as TData, + ...options, + }); /** * Get Dags * Get all DAGs. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index 04d7eb94b3208..70be4beb0d479 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -5,6 +5,7 @@ import { AssetService, ConnectionService, DagService, + DashboardService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; import * as Common from "./common"; @@ -34,6 +35,39 @@ export const useAssetServiceNextRunAssetsSuspense = < queryFn: () => AssetService.nextRunAssets({ dagId }) as TData, ...options, }); +/** + * Historical Metrics + * Return cluster activity historical metrics. + * @param data The data for the request. + * @param data.startDate + * @param data.endDate + * @returns HistoricalMetricDataResponse Successful Response + * @throws ApiError + */ +export const useDashboardServiceHistoricalMetricsSuspense = < + TData = Common.DashboardServiceHistoricalMetricsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + endDate, + startDate, + }: { + endDate: string; + startDate: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseDashboardServiceHistoricalMetricsKeyFn( + { endDate, startDate }, + queryKey, + ), + queryFn: () => + DashboardService.historicalMetrics({ endDate, startDate }) as TData, + ...options, + }); /** * Get Dags * Get all DAGs. diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 910354423bac3..26fe1180ae7b5 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -784,6 +784,56 @@ export const $DAGResponse = { description: "DAG serializer for responses.", } as const; +export const $DAGRunStates = { + properties: { + queued: { + type: "integer", + title: "Queued", + }, + running: { + type: "integer", + title: "Running", + }, + success: { + type: "integer", + title: "Success", + }, + failed: { + type: "integer", + title: "Failed", + }, + }, + type: "object", + required: ["queued", "running", "success", "failed"], + title: "DAGRunStates", + description: "DAG Run States for responses.", +} as const; + +export const $DAGRunTypes = { + properties: { + backfill: { + type: "integer", + title: "Backfill", + }, + scheduled: { + type: "integer", + title: "Scheduled", + }, + manual: { + type: "integer", + title: "Manual", + }, + dataset_triggered: { + type: "integer", + title: "Dataset Triggered", + }, + }, + type: "object", + required: ["backfill", "scheduled", "manual", "dataset_triggered"], + title: "DAGRunTypes", + description: "DAG Run Types for responses.", +} as const; + export const $DagRunState = { type: "string", enum: ["queued", "running", "success", "failed"], @@ -847,6 +897,99 @@ export const $HTTPValidationError = { title: "HTTPValidationError", } as const; +export const $HistoricalMetricDataResponse = { + properties: { + dag_run_types: { + $ref: "#/components/schemas/DAGRunTypes", + }, + dag_run_states: { + $ref: "#/components/schemas/DAGRunStates", + }, + task_instance_states: { + $ref: "#/components/schemas/TaskInstantState", + }, + }, + type: "object", + required: ["dag_run_types", "dag_run_states", "task_instance_states"], + title: "HistoricalMetricDataResponse", + description: "Historical Metric Data serializer for responses.", +} as const; + +export const $TaskInstantState = { + properties: { + no_status: { + type: "integer", + title: "No Status", + }, + removed: { + type: "integer", + title: "Removed", + }, + scheduled: { + type: "integer", + title: "Scheduled", + }, + queued: { + type: "integer", + title: "Queued", + }, + running: { + type: "integer", + title: "Running", + }, + success: { + type: "integer", + title: "Success", + }, + restarting: { + type: "integer", + title: "Restarting", + }, + failed: { + type: "integer", + title: "Failed", + }, + up_for_retry: { + type: "integer", + title: "Up For Retry", + }, + up_for_reschedule: { + type: "integer", + title: "Up For Reschedule", + }, + upstream_failed: { + type: "integer", + title: "Upstream Failed", + }, + skipped: { + type: "integer", + title: "Skipped", + }, + deferred: { + type: "integer", + title: "Deferred", + }, + }, + type: "object", + required: [ + "no_status", + "removed", + "scheduled", + "queued", + "running", + "success", + "restarting", + "failed", + "up_for_retry", + "up_for_reschedule", + "upstream_failed", + "skipped", + "deferred", + ], + title: "TaskInstantState", + description: "TaskInstance serializer for responses.", +} as const; + export const $ValidationError = { properties: { loc: { diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 72fd2f68f1000..268f636404b16 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -5,6 +5,8 @@ import { request as __request } from "./core/request"; import type { NextRunAssetsData, NextRunAssetsResponse, + HistoricalMetricsData, + HistoricalMetricsResponse, GetDagsData, GetDagsResponse, PatchDagsData, @@ -45,6 +47,34 @@ export class AssetService { } } +export class DashboardService { + /** + * Historical Metrics + * Return cluster activity historical metrics. + * @param data The data for the request. + * @param data.startDate + * @param data.endDate + * @returns HistoricalMetricDataResponse Successful Response + * @throws ApiError + */ + public static historicalMetrics( + data: HistoricalMetricsData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/ui/dashboard/historical_metrics_data", + query: { + start_date: data.startDate, + end_date: data.endDate, + }, + errors: { + 400: "Bad Request", + 422: "Validation Error", + }, + }); + } +} + export class DagService { /** * Get Dags diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index d07d980397c83..268960a596870 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -120,6 +120,26 @@ export type DAGResponse = { readonly file_token: string; }; +/** + * DAG Run States for responses. + */ +export type DAGRunStates = { + queued: number; + running: number; + success: number; + failed: number; +}; + +/** + * DAG Run Types for responses. + */ +export type DAGRunTypes = { + backfill: number; + scheduled: number; + manual: number; + dataset_triggered: number; +}; + /** * All possible states that a DagRun can be in. * @@ -152,6 +172,34 @@ export type HTTPValidationError = { detail?: Array; }; +/** + * Historical Metric Data serializer for responses. + */ +export type HistoricalMetricDataResponse = { + dag_run_types: DAGRunTypes; + dag_run_states: DAGRunStates; + task_instance_states: TaskInstantState; +}; + +/** + * TaskInstance serializer for responses. + */ +export type TaskInstantState = { + no_status: number; + removed: number; + scheduled: number; + queued: number; + running: number; + success: number; + restarting: number; + failed: number; + up_for_retry: number; + up_for_reschedule: number; + upstream_failed: number; + skipped: number; + deferred: number; +}; + export type ValidationError = { loc: Array; msg: string; @@ -166,6 +214,13 @@ export type NextRunAssetsResponse = { [key: string]: unknown; }; +export type HistoricalMetricsData = { + endDate: string; + startDate: string; +}; + +export type HistoricalMetricsResponse = HistoricalMetricDataResponse; + export type GetDagsData = { dagDisplayNamePattern?: string | null; dagIdPattern?: string | null; @@ -246,6 +301,25 @@ export type $OpenApiTs = { }; }; }; + "/ui/dashboard/historical_metrics_data": { + get: { + req: HistoricalMetricsData; + res: { + /** + * Successful Response + */ + 200: HistoricalMetricDataResponse; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; "/public/dags/": { get: { req: GetDagsData; diff --git a/airflow/www/views.py b/airflow/www/views.py index 7782da955c92e..8dba4c4fcc0d8 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -3348,6 +3348,7 @@ def grid_data(self): @expose("/object/historical_metrics_data") @auth.has_access_view(AccessView.CLUSTER_ACTIVITY) + @mark_fastapi_migration_done def historical_metrics_data(self): """Return cluster activity historical metrics.""" start_date = _safe_parse_datetime(request.args.get("start_date")) diff --git a/tests/api_fastapi/views/ui/test_dashboard.py b/tests/api_fastapi/views/ui/test_dashboard.py new file mode 100644 index 0000000000000..970b79ad3568c --- /dev/null +++ b/tests/api_fastapi/views/ui/test_dashboard.py @@ -0,0 +1,151 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import timedelta + +import pendulum +import pytest + +from airflow.models import DagBag +from airflow.operators.empty import EmptyOperator +from airflow.utils.state import DagRunState, TaskInstanceState +from airflow.utils.types import DagRunType +from tests.test_utils.db import clear_db_runs + +pytestmark = pytest.mark.db_test + + +@pytest.fixture(autouse=True, scope="module") +def examples_dag_bag(): + # Speed up: We don't want example dags for this module + + return DagBag(include_examples=False, read_dags_from_db=True) + + +@pytest.fixture(autouse=True) +def clean(): + clear_db_runs() + yield + clear_db_runs() + + +# freeze time fixture so that it is applied before `make_dag_runs` is! +@pytest.fixture +def freeze_time_for_dagruns(time_machine): + time_machine.move_to("2023-05-02T00:00:00+00:00", tick=False) + + +@pytest.fixture +def make_dag_runs(dag_maker, session, time_machine): + with dag_maker( + dag_id="test_dag_id", + serialized=True, + session=session, + start_date=pendulum.DateTime(2023, 2, 1, 0, 0, 0, tzinfo=pendulum.UTC), + ): + EmptyOperator(task_id="task_1") >> EmptyOperator(task_id="task_2") + + date = dag_maker.dag.start_date + + run1 = dag_maker.create_dagrun( + run_id="run_1", + state=DagRunState.SUCCESS, + run_type=DagRunType.SCHEDULED, + execution_date=date, + start_date=date, + ) + + run2 = dag_maker.create_dagrun( + run_id="run_2", + state=DagRunState.FAILED, + run_type=DagRunType.DATASET_TRIGGERED, + execution_date=date + timedelta(days=1), + start_date=date + timedelta(days=1), + ) + + run3 = dag_maker.create_dagrun( + run_id="run_3", + state=DagRunState.RUNNING, + run_type=DagRunType.SCHEDULED, + execution_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), + start_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), + ) + run3.end_date = None + + for ti in run1.task_instances: + ti.state = TaskInstanceState.SUCCESS + + for ti in run2.task_instances: + ti.state = TaskInstanceState.FAILED + + dag_maker.dagbag.sync_to_db() + time_machine.move_to("2023-07-02T00:00:00+00:00", tick=False) + + +class TestHistoricalMetricsDataEndpoint: + @pytest.mark.usefixtures("freeze_time_for_dagruns", "make_dag_runs") + def test_historical_metrics_data(self, test_client, time_machine): + params = {"start_date": "2023-01-01T00:00", "end_date": "2023-08-02T00:00"} + response = test_client.get("/ui/dashboard/historical_metrics_data", params=params) + + assert response.status_code == 200 + assert response.json() == { + "dag_run_states": {"failed": 1, "queued": 0, "running": 1, "success": 1}, + "dag_run_types": {"backfill": 0, "dataset_triggered": 1, "manual": 0, "scheduled": 2}, + "task_instance_states": { + "deferred": 0, + "failed": 2, + "no_status": 2, + "queued": 0, + "removed": 0, + "restarting": 0, + "running": 0, + "scheduled": 0, + "skipped": 0, + "success": 2, + "up_for_reschedule": 0, + "up_for_retry": 0, + "upstream_failed": 0, + }, + } + + @pytest.mark.usefixtures("freeze_time_for_dagruns", "make_dag_runs") + def test_historical_metrics_data_date_filters(self, test_client): + params = {"start_date": "2023-02-02T00:00", "end_date": "2023-06-02T00:00"} + response = test_client.get("/ui/dashboard/historical_metrics_data", params=params) + assert response.status_code == 200 + assert response.json() == { + "dag_run_states": {"failed": 1, "queued": 0, "running": 0, "success": 0}, + "dag_run_types": {"backfill": 0, "dataset_triggered": 1, "manual": 0, "scheduled": 0}, + "task_instance_states": { + "deferred": 0, + "failed": 2, + "no_status": 0, + "queued": 0, + "removed": 0, + "restarting": 0, + "running": 0, + "scheduled": 0, + "skipped": 0, + "success": 0, + "up_for_reschedule": 0, + "up_for_retry": 0, + "upstream_failed": 0, + }, + } From 55afad6533e1ac8a0b43289e8022171260414cab Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Wed, 9 Oct 2024 13:45:14 +0530 Subject: [PATCH 027/199] Add support to filter by last dagrun state in UI. (#42779) * Add support to filter by last dagrun state in UI. * Fix lint errors. * Fix lint errors. * Fix PR comments over null checks and query parameter to be last_dag_run_state. --- airflow/ui/src/pages/DagsList/DagsFilters.tsx | 55 +++++++++++++++++-- airflow/ui/src/pages/DagsList/DagsList.tsx | 5 +- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/airflow/ui/src/pages/DagsList/DagsFilters.tsx b/airflow/ui/src/pages/DagsList/DagsFilters.tsx index cb2be8322e500..6316816f48f95 100644 --- a/airflow/ui/src/pages/DagsList/DagsFilters.tsx +++ b/airflow/ui/src/pages/DagsList/DagsFilters.tsx @@ -25,11 +25,17 @@ import { useTableURLState } from "src/components/DataTable/useTableUrlState"; import { QuickFilterButton } from "src/components/QuickFilterButton"; const PAUSED_PARAM = "paused"; +const STATE_PARAM = "last_dag_run_state"; export const DagsFilters = () => { const [searchParams, setSearchParams] = useSearchParams(); const showPaused = searchParams.get(PAUSED_PARAM); + const state = searchParams.get(STATE_PARAM); + const isAll = state === null; + const isRunning = state === "running"; + const isFailed = state === "failed"; + const isSuccess = state === "success"; const { setTableURLState, tableURLState } = useTableURLState(); const { pagination, sorting } = tableURLState; @@ -51,6 +57,23 @@ export const DagsFilters = () => { [pagination, searchParams, setSearchParams, setTableURLState, sorting], ); + const handleStateChange: React.MouseEventHandler = + useCallback( + ({ currentTarget: { value } }) => { + if (value === "all") { + searchParams.delete(STATE_PARAM); + } else { + searchParams.set(STATE_PARAM, value); + } + setSearchParams(searchParams); + setTableURLState({ + pagination: { ...pagination, pageIndex: 0 }, + sorting, + }); + }, + [pagination, searchParams, setSearchParams, setTableURLState, sorting], + ); + return ( @@ -59,10 +82,34 @@ export const DagsFilters = () => { State: - All - Failed - Running - Successful + + All + + + Failed + + + Running + + + Successful + diff --git a/airflow/ui/src/pages/DagsList/DagsList.tsx b/airflow/ui/src/pages/DagsList/DagsList.tsx index 7b87e8d253fb2..178663baf7478 100644 --- a/airflow/ui/src/pages/DagsList/DagsList.tsx +++ b/airflow/ui/src/pages/DagsList/DagsList.tsx @@ -29,7 +29,7 @@ import { type ChangeEventHandler, useCallback, useState } from "react"; import { useSearchParams } from "react-router-dom"; import { useDagServiceGetDags } from "openapi/queries"; -import type { DAGResponse } from "openapi/requests/types.gen"; +import type { DAGResponse, DagRunState } from "openapi/requests/types.gen"; import { DataTable } from "src/components/DataTable"; import { ToggleTableDisplay } from "src/components/DataTable/ToggleTableDisplay"; import type { CardDef } from "src/components/DataTable/types"; @@ -97,12 +97,14 @@ const cardDef: CardDef = { }; const PAUSED_PARAM = "paused"; +const STATE_PARAM = "last_dag_run_state"; export const DagsList = () => { const [searchParams] = useSearchParams(); const [display, setDisplay] = useState<"card" | "table">("card"); const showPaused = searchParams.get(PAUSED_PARAM); + const lastDagRunState = searchParams.get(STATE_PARAM) as DagRunState; const { setTableURLState, tableURLState } = useTableURLState(); const { pagination, sorting } = tableURLState; @@ -112,6 +114,7 @@ export const DagsList = () => { const orderBy = sort ? `${sort.desc ? "-" : ""}${sort.id}` : undefined; const { data, isFetching, isLoading } = useDagServiceGetDags({ + lastDagRunState, limit: pagination.pageSize, offset: pagination.pageIndex * pagination.pageSize, onlyActive: true, From f2b4a443794d5c245fba9f6e4834899ddac8547d Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 9 Oct 2024 18:52:10 +0800 Subject: [PATCH 028/199] feat(datasets): make strict_dataset_uri_validation default to True (#41814) * feat(datasets): make strict_dataset_uri_validation default to True * docs(assets): add uri scheme restrict instruction for AIP-60 * feat(assets): add an error logging to URI that is not AIP-60 compliant * Better test string --------- Co-authored-by: Tzu-ping Chung --- airflow/assets/__init__.py | 20 ++-- airflow/config_templates/config.yml | 4 +- .../authoring-and-scheduling/assets.rst | 109 ++++++++++-------- newsfragments/41814.significant.rst | 1 + tests/assets/tests_asset.py | 11 -- tests/io/test_path.py | 2 +- 6 files changed, 77 insertions(+), 70 deletions(-) create mode 100644 newsfragments/41814.significant.rst diff --git a/airflow/assets/__init__.py b/airflow/assets/__init__.py index deb9aa593ded5..e11b9c49df36d 100644 --- a/airflow/assets/__init__.py +++ b/airflow/assets/__init__.py @@ -17,6 +17,7 @@ from __future__ import annotations +import logging import os import urllib.parse import warnings @@ -41,6 +42,9 @@ __all__ = ["Asset", "AssetAll", "AssetAny"] +log = logging.getLogger(__name__) + + def normalize_noop(parts: SplitResult) -> SplitResult: """ Place-hold a :class:`~urllib.parse.SplitResult`` normalizer. @@ -109,14 +113,16 @@ def _sanitize_uri(uri: str) -> str: try: parsed = normalizer(parsed) except ValueError as exception: - if conf.getboolean("core", "strict_asset_uri_validation", fallback=False): + if conf.getboolean("core", "strict_asset_uri_validation", fallback=True): + log.error( + ( + "The Asset URI %s is not AIP-60 compliant: %s. " + "Please check https://airflow.apache.org/docs/apache-airflow/stable/authoring-and-scheduling/assets.html" + ), + uri, + exception, + ) raise - warnings.warn( - f"The Asset URI {uri} is not AIP-60 compliant: {exception}. " - f"In Airflow 3, this will raise an exception.", - UserWarning, - stacklevel=3, - ) return urllib.parse.urlunsplit(parsed) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index b96c07f237376..0be77a3b6829a 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -486,9 +486,7 @@ core: strict_asset_uri_validation: description: | Asset URI validation should raise an exception if it is not compliant with AIP-60. - By default this configuration is false, meaning that Airflow 2.x only warns the user. - In Airflow 3, this configuration will be enabled by default. - default: "False" + default: "True" example: ~ version_added: 2.9.2 type: boolean diff --git a/docs/apache-airflow/authoring-and-scheduling/assets.rst b/docs/apache-airflow/authoring-and-scheduling/assets.rst index d37143367fabe..7940a9051679c 100644 --- a/docs/apache-airflow/authoring-and-scheduling/assets.rst +++ b/docs/apache-airflow/authoring-and-scheduling/assets.rst @@ -23,23 +23,23 @@ Data-aware scheduling Quickstart ---------- -In addition to scheduling DAGs based on time, you can also schedule DAGs to run based on when a task updates a asset. +In addition to scheduling DAGs based on time, you can also schedule DAGs to run based on when a task updates an asset. .. code-block:: python - from airflow.assets import asset + from airflow.assets import Asset with DAG(...): MyOperator( # this task updates example.csv - outlets=[asset("s3://asset-bucket/example.csv")], + outlets=[Asset("s3://asset-bucket/example.csv")], ..., ) with DAG( # this DAG should be run when example.csv is updated (by dag1) - schedule=[asset("s3://asset-bucket/example.csv")], + schedule=[Asset("s3://asset-bucket/example.csv")], ..., ): ... @@ -48,7 +48,7 @@ In addition to scheduling DAGs based on time, you can also schedule DAGs to run .. image:: /img/asset-scheduled-dags.png -What is a "asset"? +What is an "Asset"? -------------------- An Airflow asset is a logical grouping of data. Upstream producer tasks can update assets, and asset updates contribute to scheduling downstream consumer DAGs. @@ -57,13 +57,19 @@ An Airflow asset is a logical grouping of data. Upstream producer tasks can upda .. code-block:: python - from airflow.assets import asset + from airflow.assets import Asset - example_asset = asset("s3://asset-bucket/example.csv") + example_asset = Asset("s3://asset-bucket/example.csv") Airflow makes no assumptions about the content or location of the data represented by the URI, and treats the URI like a string. This means that Airflow treats any regular expressions, like ``input_\d+.csv``, or file glob patterns, such as ``input_2022*.csv``, as an attempt to create multiple assets from one declaration, and they will not work. -You must create assets with a valid URI. Airflow core and providers define various URI schemes that you can use, such as ``file`` (core), ``postgres`` (by the Postgres provider), and ``s3`` (by the Amazon provider). Third-party providers and plugins might also provide their own schemes. These pre-defined schemes have individual semantics that are expected to be followed. +You must create assets with a valid URI. Airflow core and providers define various URI schemes that you can use, such as ``file`` (core), ``postgres`` (by the Postgres provider), and ``s3`` (by the Amazon provider). Third-party providers and plugins might also provide their own schemes. These pre-defined schemes have individual semantics that are expected to be followed. You can use the optional name argument to provide a more human-readable identifier to the asset. + +.. code-block:: python + + from airflow.assets import Asset + + example_asset = Asset(uri="s3://asset-bucket/example.csv", name="bucket-1") What is valid URI? ------------------ @@ -72,6 +78,13 @@ Technically, the URI must conform to the valid character set in RFC 3986, which The URI is also case sensitive, so ``s3://example/asset`` and ``s3://Example/asset`` are considered different. Note that the *host* part of the URI is also case sensitive, which differs from RFC 3986. +For pre-defined schemes (e.g., ``file``, ``postgres``, and ``s3``), you must provide a meaning URI. If you can't provide one, use another scheme altogether that don't have the semantic restrictions. Airflow will never require a semantic for user-defined URI schemes (with a prefix x-), so that can be a good alternative. If you have a URI that can only be obtained later (e.g., during task execution), consider using ``AssetAlias`` instead and update the URI later. + +.. code-block:: python + + # invalid asset: + must_contain_bucket_name = Asset("s3://") + Do not use the ``airflow`` scheme, which is is reserved for Airflow's internals. Airflow always prefers using lower cases in schemes, and case sensitivity is needed in the host part of the URI to correctly distinguish between resources. @@ -79,65 +92,65 @@ Airflow always prefers using lower cases in schemes, and case sensitivity is nee .. code-block:: python # invalid assets: - reserved = asset("airflow://example_asset") - not_ascii = asset("èxample_datašet") + reserved = Asset("airflow://example_asset") + not_ascii = Asset("èxample_datašet") If you want to define assets with a scheme that doesn't include additional semantic constraints, use a scheme with the prefix ``x-``. Airflow skips any semantic validation on URIs with these schemes. .. code-block:: python # valid asset, treated as a plain string - my_ds = asset("x-my-thing://foobarbaz") + my_ds = Asset("x-my-thing://foobarbaz") The identifier does not have to be absolute; it can be a scheme-less, relative URI, or even just a simple path or string: .. code-block:: python # valid assets: - schemeless = asset("//example/asset") - csv_file = asset("example_asset") + schemeless = Asset("//example/asset") + csv_file = Asset("example_asset") Non-absolute identifiers are considered plain strings that do not carry any semantic meanings to Airflow. Extra information on asset ---------------------------- -If needed, you can include an extra dictionary in a asset: +If needed, you can include an extra dictionary in an asset: .. code-block:: python - example_asset = asset( + example_asset = Asset( "s3://asset/example.csv", extra={"team": "trainees"}, ) -This can be used to supply custom description to the asset, such as who has ownership to the target file, or what the file is for. The extra information does not affect a asset's identity. This means a DAG will be triggered by a asset with an identical URI, even if the extra dict is different: +This can be used to supply custom description to the asset, such as who has ownership to the target file, or what the file is for. The extra information does not affect an asset's identity. This means a DAG will be triggered by an asset with an identical URI, even if the extra dict is different: .. code-block:: python with DAG( dag_id="consumer", - schedule=[asset("s3://asset/example.csv", extra={"different": "extras"})], + schedule=[Asset("s3://asset/example.csv", extra={"different": "extras"})], ): ... with DAG(dag_id="producer", ...): MyOperator( # triggers "consumer" with the given extra! - outlets=[asset("s3://asset/example.csv", extra={"team": "trainees"})], + outlets=[Asset("s3://asset/example.csv", extra={"team": "trainees"})], ..., ) -.. note:: **Security Note:** asset URI and extra fields are not encrypted, they are stored in cleartext in Airflow's metadata database. Do NOT store any sensitive values, especially credentials, in either asset URIs or extra key values! +.. note:: **Security Note:** Asset URI and extra fields are not encrypted, they are stored in cleartext in Airflow's metadata database. Do NOT store any sensitive values, especially credentials, in either asset URIs or extra key values! How to use assets in your DAGs -------------------------------- -You can use assets to specify data dependencies in your DAGs. The following example shows how after the ``producer`` task in the ``producer`` DAG successfully completes, Airflow schedules the ``consumer`` DAG. Airflow marks a asset as ``updated`` only if the task completes successfully. If the task fails or if it is skipped, no update occurs, and Airflow doesn't schedule the ``consumer`` DAG. +You can use assets to specify data dependencies in your DAGs. The following example shows how after the ``producer`` task in the ``producer`` DAG successfully completes, Airflow schedules the ``consumer`` DAG. Airflow marks an asset as ``updated`` only if the task completes successfully. If the task fails or if it is skipped, no update occurs, and Airflow doesn't schedule the ``consumer`` DAG. .. code-block:: python - example_asset = asset("s3://asset/example.csv") + example_asset = Asset("s3://asset/example.csv") with DAG(dag_id="producer", ...): BashOperator(task_id="producer", outlets=[example_asset], ...) @@ -147,7 +160,7 @@ You can use assets to specify data dependencies in your DAGs. The following exam You can find a listing of the relationships between assets and DAGs in the -:ref:`assets View` +:ref:`Assets View` Multiple assets ----------------- @@ -228,17 +241,17 @@ Attaching extra information to an emitting asset event .. versionadded:: 2.10.0 -A task with a asset outlet can optionally attach extra information before it emits a asset event. This is different -from `Extra information on asset`_. Extra information on a asset statically describes the entity pointed to by the asset URI; extra information on the *asset event* instead should be used to annotate the triggering data change, such as how many rows in the database are changed by the update, or the date range covered by it. +A task with an asset outlet can optionally attach extra information before it emits an asset event. This is different +from `Extra information on asset`_. Extra information on an asset statically describes the entity pointed to by the asset URI; extra information on the *asset event* instead should be used to annotate the triggering data change, such as how many rows in the database are changed by the update, or the date range covered by it. The easiest way to attach extra information to the asset event is by ``yield``-ing a ``Metadata`` object from a task: .. code-block:: python - from airflow.assets import asset + from airflow.assets import Asset from airflow.assets.metadata import Metadata - example_s3_asset = asset("s3://asset/example.csv") + example_s3_asset = Asset("s3://asset/example.csv") @task(outlets=[example_s3_asset]) @@ -268,7 +281,7 @@ Fetching information from previously emitted asset events .. versionadded:: 2.10.0 -Events of a asset defined in a task's ``outlets``, as described in the previous section, can be read by a task that declares the same asset in its ``inlets``. A asset event entry contains ``extra`` (see previous section for details), ``timestamp`` indicating when the event was emitted from a task, and ``source_task_instance`` linking the event back to its source. +Events of an asset defined in a task's ``outlets``, as described in the previous section, can be read by a task that declares the same asset in its ``inlets``. A asset event entry contains ``extra`` (see previous section for details), ``timestamp`` indicating when the event was emitted from a task, and ``source_task_instance`` linking the event back to its source. Inlet asset events can be read with the ``inlet_events`` accessor in the execution context. Continuing from the ``write_to_s3`` task in the previous section: @@ -291,7 +304,7 @@ Example: .. code-block:: python - example_snowflake_asset = asset("snowflake://my_db/my_schema/my_table") + example_snowflake_asset = Asset("snowflake://my_db/my_schema/my_table") with DAG(dag_id="load_snowflake_data", schedule="@hourly", ...): SQLExecuteQueryOperator( @@ -332,7 +345,7 @@ In this example, the DAG ``waiting_for_asset_1_and_2`` will be triggered when ta with DAG( dag_id="waiting_for_asset_1_and_2", - schedule=[asset("asset-1"), asset("asset-2")], + schedule=[Asset("asset-1"), Asset("asset-2")], ..., ): ... @@ -344,8 +357,8 @@ In this example, the DAG ``waiting_for_asset_1_and_2`` will be triggered when ta * Get queued asset events for a DAG: ``/dags/{dag_id}/assets/queuedEvent`` * Delete a queued asset event for a DAG: ``/assets/queuedEvent/{uri}`` * Delete queued asset events for a DAG: ``/dags/{dag_id}/assets/queuedEvent`` -* Get queued asset events for a asset: ``/dags/{dag_id}/assets/queuedEvent/{uri}`` -* Delete queued asset events for a asset: ``DELETE /dags/{dag_id}/assets/queuedEvent/{uri}`` +* Get queued asset events for an asset: ``/dags/{dag_id}/assets/queuedEvent/{uri}`` +* Delete queued asset events for an asset: ``DELETE /dags/{dag_id}/assets/queuedEvent/{uri}`` For how to use REST API and the parameters needed for these endpoints, please refer to :doc:`Airflow API `. @@ -373,8 +386,8 @@ To schedule a DAG to run only when two specific assets have both been updated, u .. code-block:: python - dag1_asset = asset("s3://dag1/output_1.txt") - dag2_asset = asset("s3://dag2/output_1.txt") + dag1_asset = Asset("s3://dag1/output_1.txt") + dag2_asset = Asset("s3://dag2/output_1.txt") with DAG( # Consume asset 1 and 2 with asset expressions @@ -402,7 +415,7 @@ For scenarios requiring more intricate conditions, such as triggering a DAG when .. code-block:: python - dag3_asset = asset("s3://dag3/output_3.txt") + dag3_asset = Asset("s3://dag3/output_3.txt") with DAG( # Consume asset 1 or both 2 and 3 with asset expressions @@ -421,9 +434,9 @@ How to use AssetAlias ``AssetAlias`` has one single argument ``name`` that uniquely identifies the asset. The task must first declare the alias as an outlet, and use ``outlet_events`` or yield ``Metadata`` to add events to it. -The following example creates a asset event against the S3 URI ``f"s3://bucket/my-task"`` with optional extra information ``extra``. If the asset does not exist, Airflow will dynamically create it and log a warning message. +The following example creates an asset event against the S3 URI ``f"s3://bucket/my-task"`` with optional extra information ``extra``. If the asset does not exist, Airflow will dynamically create it and log a warning message. -**Emit a asset event during task execution through outlet_events** +**Emit an asset event during task execution through outlet_events** .. code-block:: python @@ -432,10 +445,10 @@ The following example creates a asset event against the S3 URI ``f"s3://bucket/m @task(outlets=[AssetAlias("my-task-outputs")]) def my_task_with_outlet_events(*, outlet_events): - outlet_events["my-task-outputs"].add(asset("s3://bucket/my-task"), extra={"k": "v"}) + outlet_events["my-task-outputs"].add(Asset("s3://bucket/my-task"), extra={"k": "v"}) -**Emit a asset event during task execution through yielding Metadata** +**Emit an asset event during task execution through yielding Metadata** .. code-block:: python @@ -444,7 +457,7 @@ The following example creates a asset event against the S3 URI ``f"s3://bucket/m @task(outlets=[AssetAlias("my-task-outputs")]) def my_task_with_metadata(): - s3_asset = asset("s3://bucket/my-task") + s3_asset = Asset("s3://bucket/my-task") yield Metadata(s3_asset, extra={"k": "v"}, alias="my-task-outputs") Only one asset event is emitted for an added asset, even if it is added to the alias multiple times, or added to multiple aliases. However, if different ``extra`` values are passed, it can emit multiple asset events. In the following example, two asset events will be emitted. @@ -462,15 +475,15 @@ Only one asset event is emitted for an added asset, even if it is added to the a ] ) def my_task_with_outlet_events(*, outlet_events): - outlet_events["my-task-outputs-1"].add(asset("s3://bucket/my-task"), extra={"k": "v"}) + outlet_events["my-task-outputs-1"].add(Asset("s3://bucket/my-task"), extra={"k": "v"}) # This line won't emit an additional asset event as the asset and extra are the same as the previous line. - outlet_events["my-task-outputs-2"].add(asset("s3://bucket/my-task"), extra={"k": "v"}) + outlet_events["my-task-outputs-2"].add(Asset("s3://bucket/my-task"), extra={"k": "v"}) # This line will emit an additional asset event as the extra is different. - outlet_events["my-task-outputs-3"].add(asset("s3://bucket/my-task"), extra={"k2": "v2"}) + outlet_events["my-task-outputs-3"].add(Asset("s3://bucket/my-task"), extra={"k2": "v2"}) Scheduling based on asset aliases ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Since asset events added to an alias are just simple asset events, a downstream DAG depending on the actual asset can read asset events of it normally, without considering the associated aliases. A downstream DAG can also depend on an asset alias. The authoring syntax is referencing the ``AssetAlias`` by name, and the associated asset events are picked up for scheduling. Note that a DAG can be triggered by a task with ``outlets=AssetAlias("xxx")`` if and only if the alias is resolved into ``asset("s3://bucket/my-task")``. The DAG runs whenever a task with outlet ``AssetAlias("out")`` gets associated with at least one asset at runtime, regardless of the asset's identity. The downstream DAG is not triggered if no assets are associated to the alias for a particular given task run. This also means we can do conditional asset-triggering. +Since asset events added to an alias are just simple asset events, a downstream DAG depending on the actual asset can read asset events of it normally, without considering the associated aliases. A downstream DAG can also depend on an asset alias. The authoring syntax is referencing the ``AssetAlias`` by name, and the associated asset events are picked up for scheduling. Note that a DAG can be triggered by a task with ``outlets=AssetAlias("xxx")`` if and only if the alias is resolved into ``Asset("s3://bucket/my-task")``. The DAG runs whenever a task with outlet ``AssetAlias("out")`` gets associated with at least one asset at runtime, regardless of the asset's identity. The downstream DAG is not triggered if no assets are associated to the alias for a particular given task run. This also means we can do conditional asset-triggering. The asset alias is resolved to the assets during DAG parsing. Thus, if the "min_file_process_interval" configuration is set to a high value, there is a possibility that the asset alias may not be resolved. To resolve this issue, you can trigger DAG parsing. @@ -478,7 +491,7 @@ The asset alias is resolved to the assets during DAG parsing. Thus, if the "min_ with DAG(dag_id="asset-producer"): - @task(outlets=[asset("example-alias")]) + @task(outlets=[Asset("example-alias")]) def produce_asset_events(): pass @@ -487,17 +500,17 @@ The asset alias is resolved to the assets during DAG parsing. Thus, if the "min_ @task(outlets=[AssetAlias("example-alias")]) def produce_asset_events(*, outlet_events): - outlet_events["example-alias"].add(asset("s3://bucket/my-task")) + outlet_events["example-alias"].add(Asset("s3://bucket/my-task")) - with DAG(dag_id="asset-consumer", schedule=asset("s3://bucket/my-task")): + with DAG(dag_id="asset-consumer", schedule=Asset("s3://bucket/my-task")): ... with DAG(dag_id="asset-alias-consumer", schedule=AssetAlias("example-alias")): ... -In the example provided, once the DAG ``asset-alias-producer`` is executed, the asset alias ``AssetAlias("example-alias")`` will be resolved to ``asset("s3://bucket/my-task")``. However, the DAG ``asset-alias-consumer`` will have to wait for the next DAG re-parsing to update its schedule. To address this, Airflow will re-parse the DAGs relying on the asset alias ``AssetAlias("example-alias")`` when it's resolved into assets that these DAGs did not previously depend on. As a result, both the "asset-consumer" and "asset-alias-consumer" DAGs will be triggered after the execution of DAG ``asset-alias-producer``. +In the example provided, once the DAG ``asset-alias-producer`` is executed, the asset alias ``AssetAlias("example-alias")`` will be resolved to ``Asset("s3://bucket/my-task")``. However, the DAG ``asset-alias-consumer`` will have to wait for the next DAG re-parsing to update its schedule. To address this, Airflow will re-parse the DAGs relying on the asset alias ``AssetAlias("example-alias")`` when it's resolved into assets that these DAGs did not previously depend on. As a result, both the "asset-consumer" and "asset-alias-consumer" DAGs will be triggered after the execution of DAG ``asset-alias-producer``. Fetching information from previously emitted asset events through resolved asset aliases @@ -511,7 +524,7 @@ As mentioned in :ref:`Fetching information from previously emitted asset events< @task(outlets=[AssetAlias("example-alias")]) def produce_asset_events(*, outlet_events): - outlet_events["example-alias"].add(asset("s3://bucket/my-task"), extra={"row_count": 1}) + outlet_events["example-alias"].add(Asset("s3://bucket/my-task"), extra={"row_count": 1}) with DAG(dag_id="asset-alias-consumer", schedule=None): diff --git a/newsfragments/41814.significant.rst b/newsfragments/41814.significant.rst new file mode 100644 index 0000000000000..f3b6003a5c39f --- /dev/null +++ b/newsfragments/41814.significant.rst @@ -0,0 +1 @@ +Change the default value of ``strict_dataset_uri_validation`` to True. diff --git a/tests/assets/tests_asset.py b/tests/assets/tests_asset.py index da6ef8ee79e39..afbb46827f3a9 100644 --- a/tests/assets/tests_asset.py +++ b/tests/assets/tests_asset.py @@ -39,7 +39,6 @@ from airflow.models.serialized_dag import SerializedDagModel from airflow.operators.empty import EmptyOperator from airflow.serialization.serialized_objects import BaseSerialization, SerializedDAG -from tests.test_utils.config import conf_vars @pytest.fixture @@ -492,16 +491,6 @@ def normalizer(uri): @patch("airflow.assets._get_uri_normalizer", _mock_get_uri_normalizer_raising_error) -@patch("airflow.assets.warnings.warn") -def test_sanitize_uri_raises_warning(mock_warn): - _sanitize_uri("postgres://localhost:5432/database.schema.table") - msg = mock_warn.call_args.args[0] - assert "The Asset URI postgres://localhost:5432/database.schema.table is not AIP-60 compliant" in msg - assert "In Airflow 3, this will raise an exception." in msg - - -@patch("airflow.assets._get_uri_normalizer", _mock_get_uri_normalizer_raising_error) -@conf_vars({("core", "strict_asset_uri_validation"): "True"}) def test_sanitize_uri_raises_exception(): with pytest.raises(ValueError) as e_info: _sanitize_uri("postgres://localhost:5432/database.schema.table") diff --git a/tests/io/test_path.py b/tests/io/test_path.py index 0e504b586b3d2..9d944c5f51c3c 100644 --- a/tests/io/test_path.py +++ b/tests/io/test_path.py @@ -406,7 +406,7 @@ def test_asset(self): attach("s3", fs=FakeRemoteFileSystem()) p = "s3" - f = "/tmp/foo" + f = "bucket/object" i = Asset(uri=f"{p}://{f}", extra={"foo": "bar"}) o = ObjectStoragePath(i) assert o.protocol == p From c1ef8c59a089ea9a465619ceb04ede777f599c1a Mon Sep 17 00:00:00 2001 From: GPK Date: Wed, 9 Oct 2024 12:40:39 +0100 Subject: [PATCH 029/199] Standard provider bash operator (#42252) --- .github/workflows/basic-tests.yml | 5 +++++ .pre-commit-config.yaml | 14 +++----------- airflow/decorators/bash.py | 2 +- airflow/example_dags/example_assets.py | 2 +- airflow/example_dags/example_bash_operator.py | 2 +- airflow/example_dags/example_complex.py | 2 +- .../example_dags/example_inlet_event_extra.py | 2 +- .../example_dags/example_outlet_event_extra.py | 2 +- .../example_passing_params_via_test_command.py | 2 +- airflow/example_dags/example_sensors.py | 4 ++-- airflow/example_dags/example_setup_teardown.py | 2 +- airflow/example_dags/example_task_group.py | 2 +- .../example_dags/example_trigger_target_dag.py | 2 +- airflow/example_dags/example_xcom.py | 2 +- airflow/example_dags/example_xcomargs.py | 2 +- airflow/example_dags/tutorial.py | 2 +- .../celery/executors/celery_executor_utils.py | 8 ++++++-- .../edge/example_dags/integration_test.py | 6 +++++- airflow/providers/openlineage/provider.yaml | 3 ++- .../{ => providers/standard}/operators/bash.py | 0 airflow/providers/standard/provider.yaml | 2 ++ .../{ => providers/standard}/sensors/bash.py | 0 dev/breeze/tests/test_selective_checks.py | 14 +++++++------- dev/perf/dags/elastic_dag.py | 2 +- dev/perf/dags/perf_dag_2.py | 2 +- .../chime_notifier_howto_guide.rst | 2 +- .../notifications/sns.rst | 2 +- .../notifications/sqs.rst | 2 +- .../apprise_notifier_howto_guide.rst | 2 +- .../jira-notifier-howto-guide.rst | 2 +- .../operators/cloud/mlengine.rst | 2 +- .../operators/cloud/pubsub.rst | 2 +- .../guides/developer.rst | 2 +- .../guides/user.rst | 4 ++-- .../pagerduty_notifier_howto_guide.rst | 2 +- .../slack_notifier_howto_guide.rst | 2 +- .../slackwebhook_notifier_howto_guide.rst | 2 +- .../smtp_notifier_howto_guide.rst | 2 +- .../administration-and-deployment/lineage.rst | 2 +- docs/apache-airflow/best-practices.rst | 2 +- docs/apache-airflow/core-concepts/dag-run.rst | 4 ++-- docs/apache-airflow/core-concepts/dags.rst | 2 +- .../apache-airflow/core-concepts/operators.rst | 2 +- docs/apache-airflow/core-concepts/tasks.rst | 2 +- docs/apache-airflow/howto/notifications.rst | 2 +- docs/apache-airflow/howto/operator/bash.rst | 4 ++-- docs/apache-airflow/index.rst | 2 +- .../apache-airflow/operators-and-hooks-ref.rst | 4 ++-- docs/apache-airflow/tutorial/taskflow.rst | 2 +- docs/exts/templates/openlineage.rst.jinja2 | 2 +- generated/provider_dependencies.json | 7 +++++-- kubernetes_tests/test_kubernetes_executor.py | 5 ++++- kubernetes_tests/test_other_executors.py | 5 ++++- tests/callbacks/test_callback_requests.py | 6 +++--- tests/cli/commands/test_task_command.py | 2 +- tests/core/test_core.py | 2 +- tests/dags/subdir2/test_dont_ignore_this.py | 2 +- tests/dags/test_assets.py | 2 +- .../test_backfill_with_upstream_failed_task.py | 2 +- tests/dags/test_default_impersonation.py | 2 +- tests/dags/test_example_bash_operator.py | 2 +- tests/dags/test_failing.py | 2 +- tests/dags/test_heartbeat_failed_fast.py | 2 +- tests/dags/test_impersonation.py | 2 +- tests/dags/test_miscellaneous.py | 2 +- tests/dags/test_multiple_dags.py | 2 +- tests/dags/test_no_impersonation.py | 2 +- tests/dags/test_on_failure_callback.py | 2 +- tests/dags/test_retry_handling_job.py | 2 +- tests/dags/test_sensor.py | 2 +- tests/decorators/test_setup_teardown.py | 2 +- .../executors/test_celery_executor.py | 2 +- tests/jobs/test_scheduler_job.py | 2 +- tests/listeners/test_listeners.py | 2 +- tests/models/test_dag.py | 2 +- tests/models/test_dagrun.py | 2 +- tests/models/test_renderedtifields.py | 2 +- tests/models/test_serialized_dag.py | 2 +- tests/models/test_taskinstance.py | 2 +- tests/models/test_xcom_arg.py | 2 +- .../executors/test_kubernetes_executor.py | 2 +- .../cncf/kubernetes/test_template_rendering.py | 2 +- .../openlineage/extractors/test_bash.py | 2 +- .../openlineage/extractors/test_python.py | 2 +- .../openlineage/plugins/test_adapter.py | 2 +- .../openlineage/plugins/test_facets.py | 2 +- .../openlineage/plugins/test_utils.py | 13 +++++++------ .../providers/openlineage/utils/test_utils.py | 18 +++++++++++------- .../standard}/operators/test_bash.py | 2 +- .../standard}/sensors/test_bash.py | 2 +- tests/sensors/test_external_task_sensor.py | 2 +- tests/serialization/test_dag_serialization.py | 8 ++++---- .../example_external_task_child_deferrable.py | 2 +- .../providers/amazon/aws/example_appflow.py | 2 +- .../providers/amazon/aws/example_http_to_s3.py | 2 +- tests/system/providers/amazon/aws/utils/k8s.py | 2 +- .../apache/hive/example_twitter_dag.py | 2 +- .../apache/iceberg/example_iceberg.py | 2 +- .../cncf/kubernetes/example_kubernetes.py | 2 +- .../kubernetes/example_kubernetes_async.py | 2 +- .../system/providers/docker/example_docker.py | 2 +- .../docker/example_docker_copy_data.py | 2 +- .../cloud/bigquery/example_bigquery_dataset.py | 2 +- .../cloud/bigquery/example_bigquery_queries.py | 2 +- .../bigquery/example_bigquery_queries_async.py | 2 +- .../bigquery/example_bigquery_to_mssql.py | 2 +- .../bigquery/example_bigquery_to_postgres.py | 2 +- .../cloud/cloud_build/example_cloud_build.py | 2 +- .../example_cloud_memorystore_memcached.py | 2 +- .../example_cloud_memorystore_redis.py | 2 +- .../cloud/gcs/example_gcs_copy_delete.py | 2 +- .../google/cloud/gcs/example_gcs_to_gcs.py | 2 +- .../google/cloud/gcs/example_mysql_to_gcs.py | 2 +- .../google/cloud/gcs/example_sftp_to_gcs.py | 2 +- .../google/cloud/gcs/example_sheets.py | 2 +- .../example_kubernetes_engine.py | 2 +- .../example_kubernetes_engine_async.py | 2 +- .../google/cloud/ml_engine/example_mlengine.py | 2 +- .../example_natural_language.py | 2 +- .../google/cloud/pubsub/example_pubsub.py | 2 +- .../sql_to_sheets/example_sql_to_sheets.py | 2 +- .../google/cloud/tasks/example_queue.py | 2 +- .../cloud/transfers/example_postgres_to_gcs.py | 2 +- .../cloud/translate/example_translate.py | 2 +- .../example_video_intelligence.py | 2 +- .../vision/example_vision_annotate_image.py | 2 +- .../datacatalog/example_datacatalog_entries.py | 2 +- .../example_datacatalog_search_catalog.py | 2 +- .../example_datacatalog_tag_templates.py | 2 +- .../datacatalog/example_datacatalog_tags.py | 2 +- .../opsgenie/example_opsgenie_notifier.py | 2 +- .../singularity/example_singularity.py | 2 +- tests/test_utils/compat.py | 13 ++++++++++--- tests/utils/test_dot_renderer.py | 2 +- tests/utils/test_task_group.py | 2 +- tests/www/views/test_views_rendered.py | 3 +-- tests/www/views/test_views_tasks.py | 3 +-- 137 files changed, 203 insertions(+), 176 deletions(-) rename airflow/{ => providers/standard}/operators/bash.py (100%) rename airflow/{ => providers/standard}/sensors/bash.py (100%) rename tests/{ => providers/standard}/operators/test_bash.py (99%) rename tests/{ => providers/standard}/sensors/test_bash.py (97%) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 2ccb239487679..49d6a7245bc88 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -200,6 +200,11 @@ jobs: breeze release-management prepare-provider-packages fab --package-format wheel --skip-tag-check - name: "Install Airflow with fab for webserver tests" run: pip install . dist/apache_airflow_providers_fab-*.whl + - name: "Prepare Standard provider packages: wheel" + run: > + breeze release-management prepare-provider-packages standard --package-format wheel --skip-tag-check + - name: "Install Airflow with standard provider for webserver tests" + run: pip install . dist/apache_airflow_providers_standard-*.whl - name: "Install Python client" run: pip install ./dist/apache_airflow_client-*.whl - name: "Initialize Airflow DB and start webserver" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a568c79c265aa..ce557dba431b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -550,17 +550,9 @@ repos: - id: check-no-providers-in-core-examples language: pygrep name: No providers imports in core example DAGs - description: The core example DAGs have no dependencies other than core Airflow - entry: "^\\s*from airflow\\.providers.*" + description: The core example DAGs have no dependencies other than standard provider or core Airflow + entry: "^\\s*from airflow\\.providers.(?!standard.)" pass_filenames: true - exclude: > - (?x) - ^airflow/example_dags/example_branch_datetime_operator.py| - ^airflow/example_dags/example_branch_day_of_week_operator.py| - ^airflow/example_dags/example_sensors.py| - ^airflow/example_dags/example_sensors.py| - ^airflow/example_dags/example_sensors.py| - ^airflow/example_dags/example_time_delta_sensor_async.py files: ^airflow/example_dags/.*\.py$ - id: check-no-airflow-deprecation-in-providers language: pygrep @@ -717,7 +709,7 @@ repos: files: > (?x) ^airflow/providers/.*\.py$ - exclude: ^.*/.*_vendor/ + exclude: ^.*/.*_vendor/|airflow/providers/standard/operators/bash.py - id: check-get-lineage-collector-providers language: python name: Check providers import hook lineage code from compat diff --git a/airflow/decorators/bash.py b/airflow/decorators/bash.py index 39d3131d28c7b..44738492da098 100644 --- a/airflow/decorators/bash.py +++ b/airflow/decorators/bash.py @@ -21,7 +21,7 @@ from typing import Any, Callable, Collection, Mapping, Sequence from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.context import Context, context_merge from airflow.utils.operator_helpers import determine_kwargs from airflow.utils.types import NOTSET diff --git a/airflow/example_dags/example_assets.py b/airflow/example_dags/example_assets.py index 66369794ed999..451f17a3a3abd 100644 --- a/airflow/example_dags/example_assets.py +++ b/airflow/example_dags/example_assets.py @@ -56,7 +56,7 @@ from airflow.assets import Asset from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.timetables.assets import AssetOrTimeSchedule from airflow.timetables.trigger import CronTriggerTimetable diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py index b08d31c9930c7..27702d4cb5f10 100644 --- a/airflow/example_dags/example_bash_operator.py +++ b/airflow/example_dags/example_bash_operator.py @@ -24,8 +24,8 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.operators.bash import BashOperator with DAG( dag_id="example_bash_operator", diff --git a/airflow/example_dags/example_complex.py b/airflow/example_dags/example_complex.py index e7eba78eae815..6d7d504f13d51 100644 --- a/airflow/example_dags/example_complex.py +++ b/airflow/example_dags/example_complex.py @@ -25,7 +25,7 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator with DAG( dag_id="example_complex", diff --git a/airflow/example_dags/example_inlet_event_extra.py b/airflow/example_dags/example_inlet_event_extra.py index 974534c295b79..9773df7a3f913 100644 --- a/airflow/example_dags/example_inlet_event_extra.py +++ b/airflow/example_dags/example_inlet_event_extra.py @@ -28,7 +28,7 @@ from airflow.assets import Asset from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator asset = Asset("s3://output/1.txt") diff --git a/airflow/example_dags/example_outlet_event_extra.py b/airflow/example_dags/example_outlet_event_extra.py index 893090460b538..0d097eab0ac27 100644 --- a/airflow/example_dags/example_outlet_event_extra.py +++ b/airflow/example_dags/example_outlet_event_extra.py @@ -29,7 +29,7 @@ from airflow.assets.metadata import Metadata from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator ds = Asset("s3://output/1.txt") diff --git a/airflow/example_dags/example_passing_params_via_test_command.py b/airflow/example_dags/example_passing_params_via_test_command.py index 2fcb8e4edab7b..7dcd963c09681 100644 --- a/airflow/example_dags/example_passing_params_via_test_command.py +++ b/airflow/example_dags/example_passing_params_via_test_command.py @@ -27,7 +27,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator @task(task_id="run_this") diff --git a/airflow/example_dags/example_sensors.py b/airflow/example_dags/example_sensors.py index 6fb564e63ae43..f639083858101 100644 --- a/airflow/example_dags/example_sensors.py +++ b/airflow/example_dags/example_sensors.py @@ -22,11 +22,11 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator +from airflow.providers.standard.sensors.bash import BashSensor from airflow.providers.standard.sensors.time import TimeSensor, TimeSensorAsync from airflow.providers.standard.sensors.time_delta import TimeDeltaSensor, TimeDeltaSensorAsync from airflow.providers.standard.sensors.weekday import DayOfWeekSensor -from airflow.sensors.bash import BashSensor from airflow.sensors.filesystem import FileSensor from airflow.sensors.python import PythonSensor from airflow.utils.trigger_rule import TriggerRule diff --git a/airflow/example_dags/example_setup_teardown.py b/airflow/example_dags/example_setup_teardown.py index 9fab87df7568b..81994fabc202d 100644 --- a/airflow/example_dags/example_setup_teardown.py +++ b/airflow/example_dags/example_setup_teardown.py @@ -22,7 +22,7 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup with DAG( diff --git a/airflow/example_dags/example_task_group.py b/airflow/example_dags/example_task_group.py index 6435a912cc419..5129ad3cc61e1 100644 --- a/airflow/example_dags/example_task_group.py +++ b/airflow/example_dags/example_task_group.py @@ -22,8 +22,8 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup # [START howto_task_group] diff --git a/airflow/example_dags/example_trigger_target_dag.py b/airflow/example_dags/example_trigger_target_dag.py index 7a009b8dcc6d1..3af68a25607a4 100644 --- a/airflow/example_dags/example_trigger_target_dag.py +++ b/airflow/example_dags/example_trigger_target_dag.py @@ -27,7 +27,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator @task(task_id="run_this") diff --git a/airflow/example_dags/example_xcom.py b/airflow/example_dags/example_xcom.py index fa99b91834658..2563eda77ee19 100644 --- a/airflow/example_dags/example_xcom.py +++ b/airflow/example_dags/example_xcom.py @@ -24,7 +24,7 @@ from airflow.decorators import task from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator value_1 = [1, 2, 3] value_2 = {"a": "b"} diff --git a/airflow/example_dags/example_xcomargs.py b/airflow/example_dags/example_xcomargs.py index d9d0c94f4ea01..a7103dc191135 100644 --- a/airflow/example_dags/example_xcomargs.py +++ b/airflow/example_dags/example_xcomargs.py @@ -25,7 +25,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator log = logging.getLogger(__name__) diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py index 0e31775c7a9a7..6e27bbcd2e5fe 100644 --- a/airflow/example_dags/tutorial.py +++ b/airflow/example_dags/tutorial.py @@ -32,7 +32,7 @@ from airflow.models.dag import DAG # Operators; we need this to operate! -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator # [END import_module] diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index 8f25f040c90ad..a7aa6a87ea2cf 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -111,8 +111,12 @@ def on_celery_import_modules(*args, **kwargs): import airflow.jobs.local_task_job_runner import airflow.macros - import airflow.operators.bash - import airflow.operators.python # noqa: F401 + import airflow.operators.python + + try: + import airflow.providers.standard.operators.bash + except ImportError: + import airflow.operators.bash # noqa: F401 with contextlib.suppress(ImportError): import numpy # noqa: F401 diff --git a/airflow/providers/edge/example_dags/integration_test.py b/airflow/providers/edge/example_dags/integration_test.py index d6074abd30ddc..0aad61d354c05 100644 --- a/airflow/providers/edge/example_dags/integration_test.py +++ b/airflow/providers/edge/example_dags/integration_test.py @@ -32,10 +32,14 @@ from airflow.models.dag import DAG from airflow.models.param import Param from airflow.models.variable import Variable -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +try: + from airflow.providers.standard.operators.bash import BashOperator +except ImportError: + from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] + with DAG( dag_id="integration_test", dag_display_name="Integration Test", diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index b249ff46c8591..5b08ac6a77ae7 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -84,7 +84,8 @@ config: Exclude some Operators from emitting OpenLineage events by passing a string of semicolon separated full import paths of Operators to disable. type: string - example: "airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator" + example: "airflow.providers.standard.operators.bash.BashOperator; + airflow.operators.python.PythonOperator" default: "" version_added: 1.1.0 selective_enable: diff --git a/airflow/operators/bash.py b/airflow/providers/standard/operators/bash.py similarity index 100% rename from airflow/operators/bash.py rename to airflow/providers/standard/operators/bash.py diff --git a/airflow/providers/standard/provider.yaml b/airflow/providers/standard/provider.yaml index 83d8acf0a68b3..2d4c4f29bef5c 100644 --- a/airflow/providers/standard/provider.yaml +++ b/airflow/providers/standard/provider.yaml @@ -42,6 +42,7 @@ operators: python-modules: - airflow.providers.standard.operators.datetime - airflow.providers.standard.operators.weekday + - airflow.providers.standard.operators.bash sensors: - integration-name: Standard @@ -50,3 +51,4 @@ sensors: - airflow.providers.standard.sensors.time_delta - airflow.providers.standard.sensors.time - airflow.providers.standard.sensors.weekday + - airflow.providers.standard.sensors.bash diff --git a/airflow/sensors/bash.py b/airflow/providers/standard/sensors/bash.py similarity index 100% rename from airflow/sensors/bash.py rename to airflow/providers/standard/sensors/bash.py diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4b28c7a0a63d8..3b58a45ae1b9a 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -703,9 +703,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Only Always and common providers tests should run when only common.io and tests/always changed", ), pytest.param( - ("airflow/operators/bash.py",), + ("airflow/providers/standard/operators/bash.py",), { - "affected-providers-list-as-string": None, + "affected-providers-list-as-string": "celery edge standard", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "python-versions": "['3.9']", @@ -717,14 +717,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "false", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always Core Operators Serialization", + "parallel-test-types-list-as-string": "Always Core Providers[celery,edge,standard] Serialization", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-folders": "['providers']", }, - id="Force Core and Serialization tests to run when airflow bash.py changed", + id="Providers standard tests and Serialization tests to run when airflow bash.py changed", ), pytest.param( ("tests/operators/bash.py",), @@ -1720,7 +1720,7 @@ def test_upgrade_to_newer_dependencies( ), pytest.param( ("airflow/providers/celery/file.py",), - {"docs-list-as-string": "apache-airflow celery cncf.kubernetes"}, + {"docs-list-as-string": "apache-airflow celery cncf.kubernetes standard"}, id="Celery python files changed", ), pytest.param( diff --git a/dev/perf/dags/elastic_dag.py b/dev/perf/dags/elastic_dag.py index e0adcdf5caf11..30bfc9acf992b 100644 --- a/dev/perf/dags/elastic_dag.py +++ b/dev/perf/dags/elastic_dag.py @@ -24,7 +24,7 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator # DAG File used in performance tests. Its shape can be configured by environment variables. RE_TIME_DELTA = re.compile( diff --git a/dev/perf/dags/perf_dag_2.py b/dev/perf/dags/perf_dag_2.py index 641bb7565c8f3..592bbe6087838 100644 --- a/dev/perf/dags/perf_dag_2.py +++ b/dev/perf/dags/perf_dag_2.py @@ -24,7 +24,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator args = { "owner": "airflow", diff --git a/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst b/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst index a52540fe78282..e15c3a8c0c8e4 100644 --- a/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.chime import send_chime_notification with DAG( diff --git a/docs/apache-airflow-providers-amazon/notifications/sns.rst b/docs/apache-airflow-providers-amazon/notifications/sns.rst index bbaad4f814712..262cd966ae418 100644 --- a/docs/apache-airflow-providers-amazon/notifications/sns.rst +++ b/docs/apache-airflow-providers-amazon/notifications/sns.rst @@ -33,7 +33,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.sns import send_sns_notification dag_failure_sns_notification = send_sns_notification( diff --git a/docs/apache-airflow-providers-amazon/notifications/sqs.rst b/docs/apache-airflow-providers-amazon/notifications/sqs.rst index 6951caa9fdd67..d74a2477d62ca 100644 --- a/docs/apache-airflow-providers-amazon/notifications/sqs.rst +++ b/docs/apache-airflow-providers-amazon/notifications/sqs.rst @@ -33,7 +33,7 @@ Example Code: from datetime import datetime, timezone from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.sqs import send_sqs_notification dag_failure_sqs_notification = send_sqs_notification( diff --git a/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst b/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst index 777a3d46a3b2c..2a0aeaaa10764 100644 --- a/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst @@ -30,7 +30,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.apprise.notifications.apprise import send_apprise_notification from apprise import NotifyType diff --git a/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst b/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst index e0ed12558145f..a5617b9035de0 100644 --- a/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst +++ b/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst @@ -31,7 +31,7 @@ Example Code from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.atlassian.jira.notifications.jira import send_jira_notification with DAG( diff --git a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst index 9c71e885d8d6a..f64705e1c267c 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst @@ -96,7 +96,7 @@ instead. You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` fields to dynamically determine their values. The result are saved to :ref:`XCom `, allowing them to be used by other operators. In this case, the -:class:`~airflow.operators.bash.BashOperator` is used to print the model information. +:class:`~airflow.providers.standard.operators.bash.BashOperator` is used to print the model information. .. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py :language: python diff --git a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst index 7d9a6dd5ff2ab..8fb497a14f01e 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst @@ -101,7 +101,7 @@ Also for this action you can use sensor in the deferrable mode: :start-after: [START howto_operator_gcp_pubsub_pull_message_with_operator] :end-before: [END howto_operator_gcp_pubsub_pull_message_with_operator] -To pull messages from XCom use the :class:`~airflow.operators.bash.BashOperator`. +To pull messages from XCom use the :class:`~airflow.providers.standard.operators.bash.BashOperator`. .. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py :language: python diff --git a/docs/apache-airflow-providers-openlineage/guides/developer.rst b/docs/apache-airflow-providers-openlineage/guides/developer.rst index c2a5ffdc8fda4..ccab215fc1846 100644 --- a/docs/apache-airflow-providers-openlineage/guides/developer.rst +++ b/docs/apache-airflow-providers-openlineage/guides/developer.rst @@ -390,7 +390,7 @@ An Operator inside the Airflow DAG can be annotated with inlets and outlets like import pendulum from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.lineage.entities import Table, File, Column, User diff --git a/docs/apache-airflow-providers-openlineage/guides/user.rst b/docs/apache-airflow-providers-openlineage/guides/user.rst index 2c299b8c6d39e..4f95253e1f242 100644 --- a/docs/apache-airflow-providers-openlineage/guides/user.rst +++ b/docs/apache-airflow-providers-openlineage/guides/user.rst @@ -257,13 +257,13 @@ full import paths of Airflow Operators to disable as ``disabled_for_operators`` [openlineage] transport = {"type": "http", "url": "http://example.com:5000", "endpoint": "api/v1/lineage"} - disabled_for_operators = 'airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator' + disabled_for_operators = 'airflow.providers.standard.operators.bash.BashOperator;airflow.operators.python.PythonOperator' ``AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS`` environment variable is an equivalent. .. code-block:: ini - AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator' + AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.providers.standard.operators.bash.BashOperator;airflow.operators.python.PythonOperator' Full Task Info ^^^^^^^^^^^^^^ diff --git a/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst b/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst index d16f9b2b9e48a..658054bd0a5ec 100644 --- a/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.pagerduty.notifications.pagerduty import send_pagerduty_notification with DAG( diff --git a/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst b/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst index a4f891f8a57bb..3b6a1e7879924 100644 --- a/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.slack.notifications.slack import send_slack_notification with DAG( diff --git a/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst b/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst index 66ced818a7d18..e6ef3ab41409c 100644 --- a/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst @@ -32,7 +32,7 @@ Example Code: from datetime import datetime, timezone from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.slack.notifications.slack_webhook import send_slack_webhook_notification dag_failure_slack_webhook_notification = send_slack_webhook_notification( diff --git a/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst b/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst index 4cb1bf310e03d..e47f9e340c93b 100644 --- a/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.smtp.notifications.smtp import send_smtp_notification with DAG( diff --git a/docs/apache-airflow/administration-and-deployment/lineage.rst b/docs/apache-airflow/administration-and-deployment/lineage.rst index b274809175c03..3740e8b56f937 100644 --- a/docs/apache-airflow/administration-and-deployment/lineage.rst +++ b/docs/apache-airflow/administration-and-deployment/lineage.rst @@ -36,7 +36,7 @@ works. from airflow.lineage import AUTO from airflow.lineage.entities import File from airflow.models import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator FILE_CATEGORIES = ["CAT1", "CAT2", "CAT3"] diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst index 80a5996f36768..466f546ff7147 100644 --- a/docs/apache-airflow/best-practices.rst +++ b/docs/apache-airflow/best-practices.rst @@ -480,7 +480,7 @@ It's easier to grab the concept with an example. Let's say that we have the foll from airflow import DAG from airflow.decorators import task from airflow.exceptions import AirflowException - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule diff --git a/docs/apache-airflow/core-concepts/dag-run.rst b/docs/apache-airflow/core-concepts/dag-run.rst index 0621d3f771e2c..97fd4e28a7b58 100644 --- a/docs/apache-airflow/core-concepts/dag-run.rst +++ b/docs/apache-airflow/core-concepts/dag-run.rst @@ -101,7 +101,7 @@ in the configuration file. When turned off, the scheduler creates a DAG run only https://github.com/apache/airflow/blob/main/airflow/example_dags/tutorial.py """ from airflow.models.dag import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator import datetime import pendulum @@ -241,7 +241,7 @@ Example of a parameterized DAG: import pendulum from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator dag = DAG( "example_parameterized_dag", diff --git a/docs/apache-airflow/core-concepts/dags.rst b/docs/apache-airflow/core-concepts/dags.rst index f9dc7d64c72e0..64726e08010e1 100644 --- a/docs/apache-airflow/core-concepts/dags.rst +++ b/docs/apache-airflow/core-concepts/dags.rst @@ -574,7 +574,7 @@ TaskGroup also supports ``default_args`` like DAG, it will overwrite the ``defau from airflow import DAG from airflow.decorators import task_group - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator with DAG( diff --git a/docs/apache-airflow/core-concepts/operators.rst b/docs/apache-airflow/core-concepts/operators.rst index 354697c308537..6a0825df28773 100644 --- a/docs/apache-airflow/core-concepts/operators.rst +++ b/docs/apache-airflow/core-concepts/operators.rst @@ -28,7 +28,7 @@ An Operator is conceptually a template for a predefined :doc:`Task `, tha Airflow has a very extensive set of operators available, with some built-in to the core or pre-installed providers. Some popular operators from core include: -- :class:`~airflow.operators.bash.BashOperator` - executes a bash command +- :class:`~airflow.providers.standard.operators.bash.BashOperator` - executes a bash command - :class:`~airflow.operators.python.PythonOperator` - calls an arbitrary Python function - :class:`~airflow.operators.email.EmailOperator` - sends an email - Use the ``@task`` decorator to execute an arbitrary Python function. It doesn't support rendering jinja templates passed as arguments. diff --git a/docs/apache-airflow/core-concepts/tasks.rst b/docs/apache-airflow/core-concepts/tasks.rst index ad03283ef772d..5adfe8be46024 100644 --- a/docs/apache-airflow/core-concepts/tasks.rst +++ b/docs/apache-airflow/core-concepts/tasks.rst @@ -236,7 +236,7 @@ If you'd like to reproduce zombie tasks for development/testing processes, follo .. code-block:: python from airflow.decorators import dag - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from datetime import datetime diff --git a/docs/apache-airflow/howto/notifications.rst b/docs/apache-airflow/howto/notifications.rst index c477ec1d3c173..993a36b389423 100644 --- a/docs/apache-airflow/howto/notifications.rst +++ b/docs/apache-airflow/howto/notifications.rst @@ -59,7 +59,7 @@ Here's an example of using the above notifier: from datetime import datetime from airflow.models.dag import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from myprovider.notifier import MyNotifier diff --git a/docs/apache-airflow/howto/operator/bash.rst b/docs/apache-airflow/howto/operator/bash.rst index daf430fa14cde..e4af9bcad6b83 100644 --- a/docs/apache-airflow/howto/operator/bash.rst +++ b/docs/apache-airflow/howto/operator/bash.rst @@ -22,7 +22,7 @@ BashOperator ============ -Use the :class:`~airflow.operators.bash.BashOperator` to execute +Use the :class:`~airflow.providers.standard.operators.bash.BashOperator` to execute commands in a `Bash `__ shell. The Bash command or script to execute is determined by: @@ -390,7 +390,7 @@ There are numerous possibilities with this type of pre-execution enrichment. BashSensor ========== -Use the :class:`~airflow.sensors.bash.BashSensor` to use arbitrary command for sensing. The command +Use the :class:`~airflow.providers.standard.sensors.bash.BashSensor` to use arbitrary command for sensing. The command should return 0 when it succeeds, any other value otherwise. .. exampleinclude:: /../../airflow/example_dags/example_sensors.py diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst index 44dcd9a3bd36c..38d62ecd04a72 100644 --- a/docs/apache-airflow/index.rst +++ b/docs/apache-airflow/index.rst @@ -41,7 +41,7 @@ Take a look at the following snippet of code: from airflow import DAG from airflow.decorators import task - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator # A DAG represents a workflow, a collection of tasks with DAG(dag_id="demo", start_date=datetime(2022, 1, 1), schedule="0 0 * * *") as dag: diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst index 16b74305a958b..c82a4f3a66d73 100644 --- a/docs/apache-airflow/operators-and-hooks-ref.rst +++ b/docs/apache-airflow/operators-and-hooks-ref.rst @@ -50,7 +50,7 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Operators - Guides - * - :mod:`airflow.operators.bash` + * - :mod:`airflow.providers.standard.operators.bash` - :doc:`How to use ` * - :mod:`airflow.operators.branch` @@ -82,7 +82,7 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Sensors - Guides - * - :mod:`airflow.sensors.bash` + * - :mod:`airflow.providers.standard.sensors.bash` - :ref:`How to use ` * - :mod:`airflow.sensors.external_task` diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index c77debab8f328..aac04f9b53454 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -437,7 +437,7 @@ the parameter value is used. Adding dependencies between decorated and traditional tasks ----------------------------------------------------------- The above tutorial shows how to create dependencies between TaskFlow functions. However, dependencies can also -be set between traditional tasks (such as :class:`~airflow.operators.bash.BashOperator` +be set between traditional tasks (such as :class:`~airflow.providers.standard.operators.bash.BashOperator` or :class:`~airflow.sensors.filesystem.FileSensor`) and TaskFlow functions. Building this dependency is shown in the code below: diff --git a/docs/exts/templates/openlineage.rst.jinja2 b/docs/exts/templates/openlineage.rst.jinja2 index 7dffc175f84b2..dfac543cbe7f1 100644 --- a/docs/exts/templates/openlineage.rst.jinja2 +++ b/docs/exts/templates/openlineage.rst.jinja2 @@ -22,7 +22,7 @@ At the moment, two core operators supports OpenLineage. These operators function capable of running any code, which might limit the extent of lineage extraction. - :class:`~airflow.operators.python.PythonOperator` (via :class:`airflow.providers.openlineage.extractors.python.PythonExtractor`) -- :class:`~airflow.operators.bash.BashOperator` (via :class:`airflow.providers.openlineage.extractors.bash.BashExtractor`) +- :class:`~airflow.providers.standard.operators.bash.BashOperator` (via :class:`airflow.providers.openlineage.extractors.bash.BashExtractor`) :class:`~airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator` diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index b1ff0f7dcfab7..57bca636ed605 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -344,7 +344,8 @@ "devel-deps": [], "plugins": [], "cross-providers-deps": [ - "cncf.kubernetes" + "cncf.kubernetes", + "standard" ], "excluded-python-versions": [], "state": "ready" @@ -532,7 +533,9 @@ "plugin-class": "airflow.providers.edge.plugins.edge_executor_plugin.EdgeExecutorPlugin" } ], - "cross-providers-deps": [], + "cross-providers-deps": [ + "standard" + ], "excluded-python-versions": [], "state": "not-ready" }, diff --git a/kubernetes_tests/test_kubernetes_executor.py b/kubernetes_tests/test_kubernetes_executor.py index a270243bfac78..42b181b443041 100644 --- a/kubernetes_tests/test_kubernetes_executor.py +++ b/kubernetes_tests/test_kubernetes_executor.py @@ -20,7 +20,10 @@ import pytest -from kubernetes_tests.test_base import EXECUTOR, BaseK8STest # isort:skip (needed to workaround isort bug) +from kubernetes_tests.test_base import ( + EXECUTOR, + BaseK8STest, # isort:skip (needed to workaround isort bug) +) @pytest.mark.skipif(EXECUTOR != "KubernetesExecutor", reason="Only runs on KubernetesExecutor") diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py index 97b7e3df728ee..3a1aea16f1291 100644 --- a/kubernetes_tests/test_other_executors.py +++ b/kubernetes_tests/test_other_executors.py @@ -20,7 +20,10 @@ import pytest -from kubernetes_tests.test_base import EXECUTOR, BaseK8STest # isort:skip (needed to workaround isort bug) +from kubernetes_tests.test_base import ( + EXECUTOR, + BaseK8STest, # isort:skip (needed to workaround isort bug) +) # These tests are here because only KubernetesExecutor can run the tests in diff --git a/tests/callbacks/test_callback_requests.py b/tests/callbacks/test_callback_requests.py index 5992ee6fbbf70..7bbe41387750c 100644 --- a/tests/callbacks/test_callback_requests.py +++ b/tests/callbacks/test_callback_requests.py @@ -27,7 +27,7 @@ ) from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType @@ -100,7 +100,7 @@ def test_simple_ti_roundtrip_exec_config_pod(self): from airflow.callbacks.callback_requests import TaskCallbackRequest from airflow.models import TaskInstance from airflow.models.taskinstance import SimpleTaskInstance - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator test_pod = k8s.V1Pod(metadata=k8s.V1ObjectMeta(name="hello", namespace="ns")) op = BashOperator(task_id="hi", executor_config={"pod_override": test_pod}, bash_command="hi") @@ -115,7 +115,7 @@ def test_simple_ti_roundtrip_dates(self, dag_maker): from airflow.callbacks.callback_requests import TaskCallbackRequest from airflow.models import TaskInstance from airflow.models.taskinstance import SimpleTaskInstance - from airflow.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator with dag_maker(schedule=timedelta(weeks=1), serialized=True): op = BashOperator(task_id="hi", bash_command="hi") diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index 36cbcc85a7214..3397005b80eb0 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -45,8 +45,8 @@ from airflow.executors.local_executor import LocalExecutor from airflow.models import DagBag, DagRun, Pool, TaskInstance from airflow.models.serialized_dag import SerializedDagModel -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState diff --git a/tests/core/test_core.py b/tests/core/test_core.py index a75428b33a18b..d44235f955243 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -27,9 +27,9 @@ from airflow.exceptions import AirflowTaskTimeout from airflow.models import DagRun, TaskFail, TaskInstance from airflow.models.baseoperator import BaseOperator -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail diff --git a/tests/dags/subdir2/test_dont_ignore_this.py b/tests/dags/subdir2/test_dont_ignore_this.py index 72c1796a424b2..07f04293d7d54 100644 --- a/tests/dags/subdir2/test_dont_ignore_this.py +++ b/tests/dags/subdir2/test_dont_ignore_this.py @@ -20,7 +20,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2019, 12, 1) diff --git a/tests/dags/test_assets.py b/tests/dags/test_assets.py index a4ecd6aad4a6a..014ae6fd0ca9f 100644 --- a/tests/dags/test_assets.py +++ b/tests/dags/test_assets.py @@ -22,8 +22,8 @@ from airflow.assets import Asset from airflow.exceptions import AirflowFailException, AirflowSkipException from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator skip_task_dag_dataset = Asset("s3://dag_with_skip_task/output_1.txt", extra={"hi": "bye"}) fail_task_dag_dataset = Asset("s3://dag_with_fail_task/output_1.txt", extra={"hi": "bye"}) diff --git a/tests/dags/test_backfill_with_upstream_failed_task.py b/tests/dags/test_backfill_with_upstream_failed_task.py index d2cb6353bfaa3..865b0da4ff426 100644 --- a/tests/dags/test_backfill_with_upstream_failed_task.py +++ b/tests/dags/test_backfill_with_upstream_failed_task.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator dag = DAG( dag_id="test_backfill_with_upstream_failed_task", diff --git a/tests/dags/test_default_impersonation.py b/tests/dags/test_default_impersonation.py index 468b7dce072dd..4bee30457b7bd 100644 --- a/tests/dags/test_default_impersonation.py +++ b/tests/dags/test_default_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_example_bash_operator.py b/tests/dags/test_example_bash_operator.py index eb472b8011ebd..52126f0e10206 100644 --- a/tests/dags/test_example_bash_operator.py +++ b/tests/dags/test_example_bash_operator.py @@ -20,8 +20,8 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.operators.bash import BashOperator dag = DAG( dag_id="test_example_bash_operator", diff --git a/tests/dags/test_failing.py b/tests/dags/test_failing.py index 28e2fb5881987..646665a8802ef 100644 --- a/tests/dags/test_failing.py +++ b/tests/dags/test_failing.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator dag = DAG( dag_id="test_failing_bash_operator", diff --git a/tests/dags/test_heartbeat_failed_fast.py b/tests/dags/test_heartbeat_failed_fast.py index aee7a67030585..890756ef2017d 100644 --- a/tests/dags/test_heartbeat_failed_fast.py +++ b/tests/dags/test_heartbeat_failed_fast.py @@ -20,7 +20,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_impersonation.py b/tests/dags/test_impersonation.py index 33a3c89d328d9..6c2ca2d810026 100644 --- a/tests/dags/test_impersonation.py +++ b/tests/dags/test_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_miscellaneous.py b/tests/dags/test_miscellaneous.py index c19277a617571..4a2c6b56a365d 100644 --- a/tests/dags/test_miscellaneous.py +++ b/tests/dags/test_miscellaneous.py @@ -22,8 +22,8 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from tests.test_utils.compat import BashOperator args = { "owner": "airflow", diff --git a/tests/dags/test_multiple_dags.py b/tests/dags/test_multiple_dags.py index 5801084fab712..27f159bfb1272 100644 --- a/tests/dags/test_multiple_dags.py +++ b/tests/dags/test_multiple_dags.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator args = {"owner": "airflow", "retries": 3, "start_date": datetime.datetime(2022, 1, 1)} diff --git a/tests/dags/test_no_impersonation.py b/tests/dags/test_no_impersonation.py index 2a75d5321473c..22b47fcc878c8 100644 --- a/tests/dags/test_no_impersonation.py +++ b/tests/dags/test_no_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_on_failure_callback.py b/tests/dags/test_on_failure_callback.py index e2f4ab9027a8c..f6765a3698097 100644 --- a/tests/dags/test_on_failure_callback.py +++ b/tests/dags/test_on_failure_callback.py @@ -21,8 +21,8 @@ from airflow.exceptions import AirflowFailException from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_retry_handling_job.py b/tests/dags/test_retry_handling_job.py index 7040e8c8756f8..ede9c4c6ace50 100644 --- a/tests/dags/test_retry_handling_job.py +++ b/tests/dags/test_retry_handling_job.py @@ -20,7 +20,7 @@ from datetime import datetime, timedelta from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator default_args = { "owner": "airflow", diff --git a/tests/dags/test_sensor.py b/tests/dags/test_sensor.py index d023949e31a98..07c9cc7efdffb 100644 --- a/tests/dags/test_sensor.py +++ b/tests/dags/test_sensor.py @@ -20,8 +20,8 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.standard.sensors.date_time import DateTimeSensor from airflow.utils import timezone +from tests.test_utils.compat import DateTimeSensor with DAG( dag_id="test_sensor", start_date=datetime.datetime(2022, 1, 1), catchup=False, schedule="@once" diff --git a/tests/decorators/test_setup_teardown.py b/tests/decorators/test_setup_teardown.py index 13451ba379eca..1f2a3dcdbc630 100644 --- a/tests/decorators/test_setup_teardown.py +++ b/tests/decorators/test_setup_teardown.py @@ -22,7 +22,7 @@ from airflow.decorators import setup, task, task_group, teardown from airflow.decorators.setup_teardown import context_wrapper from airflow.exceptions import AirflowException -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 9c7fe96ff186d..4ec1cc458c3a5 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -43,7 +43,7 @@ from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance from airflow.models.taskinstancekey import TaskInstanceKey -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.state import State, TaskInstanceState from tests.test_utils import db diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index b4e4c10cff456..d26369f5d728e 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -60,8 +60,8 @@ from airflow.models.pool import Pool from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance, TaskInstanceKey -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone from airflow.utils.file import list_py_file_paths diff --git a/tests/listeners/test_listeners.py b/tests/listeners/test_listeners.py index 3c34ab0ff8ab2..29ec25a9a8d2a 100644 --- a/tests/listeners/test_listeners.py +++ b/tests/listeners/test_listeners.py @@ -25,7 +25,7 @@ from airflow.exceptions import AirflowException from airflow.jobs.job import Job, run_job from airflow.listeners.listener import get_listener_manager -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import DagRunState, TaskInstanceState diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index ab67c3778c262..c79ca24e03a25 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -74,9 +74,9 @@ from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskfail import TaskFail from airflow.models.taskinstance import TaskInstance as TI -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.security import permissions from airflow.templates import NativeEnvironment, SandboxedEnvironment from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index c7dacaeb291e4..9184f561b3df9 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -36,9 +36,9 @@ from airflow.models.taskinstance import TaskInstance, TaskInstanceNote, clear_task_instances from airflow.models.taskmap import TaskMap from airflow.models.taskreschedule import TaskReschedule -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import ShortCircuitOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.stats import Stats from airflow.triggers.base import StartTriggerArgs diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index b8c45193814aa..1de83954ee4bd 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -31,7 +31,7 @@ from airflow.decorators import task as task_decorator from airflow.models import Variable from airflow.models.renderedtifields import RenderedTaskInstanceFields as RTIF -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.timezone import datetime from tests.test_utils.asserts import assert_queries_count diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py index d9a77e55edaf5..93845e95832cf 100644 --- a/tests/models/test_serialized_dag.py +++ b/tests/models/test_serialized_dag.py @@ -31,7 +31,7 @@ from airflow.models.dagbag import DagBag from airflow.models.dagcode import DagCode from airflow.models.serialized_dag import SerializedDagModel as SDM -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 8c334366f0488..c09d3575d1eca 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -76,9 +76,9 @@ from airflow.models.variable import Variable from airflow.models.xcom import LazyXComSelectSequence, XCom from airflow.notifications.basenotifier import BaseNotifier -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.sensors.base import BaseSensorOperator from airflow.sensors.python import PythonSensor from airflow.serialization.serialized_objects import SerializedBaseOperator, SerializedDAG diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index 6108c5e81930f..fcc2e546009c9 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -19,8 +19,8 @@ import pytest from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.types import NOTSET from tests.test_utils.config import conf_vars from tests.test_utils.db import clear_db_dags, clear_db_runs diff --git a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py index 4622d31b575fc..12435426dd899 100644 --- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -30,7 +30,6 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models.taskinstancekey import TaskInstanceKey -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.cncf.kubernetes import pod_generator from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import ( @@ -55,6 +54,7 @@ from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator from airflow.utils import timezone from airflow.utils.state import State, TaskInstanceState +from tests.test_utils.compat import BashOperator from tests.test_utils.config import conf_vars pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/tests/providers/cncf/kubernetes/test_template_rendering.py b/tests/providers/cncf/kubernetes/test_template_rendering.py index ab2820284d553..4c087d6040e68 100644 --- a/tests/providers/cncf/kubernetes/test_template_rendering.py +++ b/tests/providers/cncf/kubernetes/test_template_rendering.py @@ -24,11 +24,11 @@ from airflow.configuration import TEST_DAGS_FOLDER from airflow.models.renderedtifields import RenderedTaskInstanceFields, RenderedTaskInstanceFields as RTIF -from airflow.operators.bash import BashOperator from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec, render_k8s_pod_yaml from airflow.utils.session import create_session from airflow.version import version from tests.models import DEFAULT_DATE +from tests.test_utils.compat import BashOperator pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/providers/openlineage/extractors/test_bash.py b/tests/providers/openlineage/extractors/test_bash.py index de65a1d176d8a..fc862e5ee30b9 100644 --- a/tests/providers/openlineage/extractors/test_bash.py +++ b/tests/providers/openlineage/extractors/test_bash.py @@ -26,8 +26,8 @@ from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.operators.bash import BashOperator from airflow.providers.openlineage.extractors.bash import BashExtractor +from tests.test_utils.compat import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/extractors/test_python.py b/tests/providers/openlineage/extractors/test_python.py index 81284383d8648..44c5503b712d8 100644 --- a/tests/providers/openlineage/extractors/test_python.py +++ b/tests/providers/openlineage/extractors/test_python.py @@ -28,9 +28,9 @@ from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from airflow.providers.openlineage.extractors.python import PythonExtractor +from tests.test_utils.compat import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/plugins/test_adapter.py b/tests/providers/openlineage/plugins/test_adapter.py index 260883470875f..b01fe46fdca13 100644 --- a/tests/providers/openlineage/plugins/test_adapter.py +++ b/tests/providers/openlineage/plugins/test_adapter.py @@ -40,7 +40,6 @@ from airflow import DAG from airflow.models.dagrun import DagRun, DagRunState from airflow.models.taskinstance import TaskInstance, TaskInstanceState -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.openlineage.conf import namespace from airflow.providers.openlineage.extractors import OperatorLineage @@ -52,6 +51,7 @@ ) from airflow.providers.openlineage.utils.utils import get_airflow_job_facet from airflow.utils.task_group import TaskGroup +from tests.test_utils.compat import BashOperator from tests.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/plugins/test_facets.py b/tests/providers/openlineage/plugins/test_facets.py index 0ed5b4bf7c440..d46cadc9d69c6 100644 --- a/tests/providers/openlineage/plugins/test_facets.py +++ b/tests/providers/openlineage/plugins/test_facets.py @@ -81,7 +81,7 @@ def test_airflow_dag_run_facet(): }, tasks={ "task_0": { - "operator": "airflow.operators.bash.BashOperator", + "operator": "airflow.providers.standard.operators.bash.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": "#f0ede4", diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py index 5335739a8ff92..65874a5ecebf3 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/tests/providers/openlineage/plugins/test_utils.py @@ -29,7 +29,6 @@ from pkg_resources import parse_version from airflow.models import DAG as AIRFLOW_DAG, DagModel -from airflow.operators.bash import BashOperator from airflow.providers.openlineage.plugins.facets import AirflowDebugRunFacet from airflow.providers.openlineage.utils.utils import ( InfoJsonEncodable, @@ -44,11 +43,15 @@ from airflow.utils import timezone from airflow.utils.log.secrets_masker import _secrets_masker from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType +BASH_OPERATOR_PATH = "airflow.providers.standard.operators.bash" +if not AIRFLOW_V_2_10_PLUS: + BASH_OPERATOR_PATH = "airflow.operators.bash" + class SafeStrDict(dict): def __str__(self): @@ -262,7 +265,7 @@ def test_get_fully_qualified_class_name(): from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter result = get_fully_qualified_class_name(BashOperator(task_id="test", bash_command="exit 0;")) - assert result == "airflow.operators.bash.BashOperator" + assert result == f"{BASH_OPERATOR_PATH}.BashOperator" result = get_fully_qualified_class_name(OpenLineageAdapter()) assert result == "airflow.providers.openlineage.plugins.adapter.OpenLineageAdapter" @@ -278,7 +281,7 @@ def test_is_operator_disabled(mock_disabled_operators): assert is_operator_disabled(op) is False mock_disabled_operators.return_value = { - "airflow.operators.bash.BashOperator", + f"{BASH_OPERATOR_PATH}.BashOperator", "airflow.operators.python.PythonOperator", } assert is_operator_disabled(op) is True @@ -303,8 +306,6 @@ def test_includes_full_task_info(mock_include_full_task_info): @patch("airflow.providers.openlineage.conf.include_full_task_info") def test_does_not_include_full_task_info(mock_include_full_task_info): - from airflow.operators.bash import BashOperator - mock_include_full_task_info.return_value = False # There should be no 'bash_command' in excludes and it's not in includes - so # it's a good choice for checking TaskInfo vs TaskInfoComplete diff --git a/tests/providers/openlineage/utils/test_utils.py b/tests/providers/openlineage/utils/test_utils.py index d97a447e99949..20eba76adeb18 100644 --- a/tests/providers/openlineage/utils/test_utils.py +++ b/tests/providers/openlineage/utils/test_utils.py @@ -27,7 +27,6 @@ from airflow.models.dagrun import DagRun from airflow.models.mappedoperator import MappedOperator from airflow.models.taskinstance import TaskInstance, TaskInstanceState -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator from airflow.providers.openlineage.plugins.facets import AirflowDagRunFacet, AirflowJobFacet @@ -44,8 +43,13 @@ from airflow.serialization.serialized_objects import SerializedBaseOperator from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType +from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator from tests.test_utils.mock_operators import MockOperator +BASH_OPERATOR_PATH = "airflow.providers.standard.operators.bash" +if not AIRFLOW_V_2_10_PLUS: + BASH_OPERATOR_PATH = "airflow.operators.bash" + class CustomOperatorForTest(BashOperator): pass @@ -82,7 +86,7 @@ def test_get_airflow_job_facet(): }, tasks={ "task_0": { - "operator": "airflow.operators.bash.BashOperator", + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": "#f0ede4", @@ -166,7 +170,7 @@ def test_get_airflow_dag_run_facet(): def test_get_fully_qualified_class_name_serialized_operator(): - op_module_path = "airflow.operators.bash" + op_module_path = BASH_OPERATOR_PATH op_name = "BashOperator" op = BashOperator(task_id="test", bash_command="echo 1") @@ -191,7 +195,7 @@ def test_get_fully_qualified_class_name_mapped_operator(): def test_get_fully_qualified_class_name_bash_operator(): result = get_fully_qualified_class_name(BashOperator(task_id="test", bash_command="echo 0;")) - expected_result = "airflow.operators.bash.BashOperator" + expected_result = f"{BASH_OPERATOR_PATH}.BashOperator" assert result == expected_result @@ -319,7 +323,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_0": { - "operator": "airflow.operators.bash.BashOperator", + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -360,7 +364,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_3": { - "operator": "airflow.operators.bash.BashOperator", + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -388,7 +392,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_5": { - "operator": "airflow.operators.bash.BashOperator", + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, diff --git a/tests/operators/test_bash.py b/tests/providers/standard/operators/test_bash.py similarity index 99% rename from tests/operators/test_bash.py rename to tests/providers/standard/operators/test_bash.py index 8aacb3b7c77ea..2c29a0b96dc93 100644 --- a/tests/operators/test_bash.py +++ b/tests/providers/standard/operators/test_bash.py @@ -28,7 +28,7 @@ import pytest from airflow.exceptions import AirflowException, AirflowSkipException, AirflowTaskTimeout -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType diff --git a/tests/sensors/test_bash.py b/tests/providers/standard/sensors/test_bash.py similarity index 97% rename from tests/sensors/test_bash.py rename to tests/providers/standard/sensors/test_bash.py index 3282f6b971221..d51db033be308 100644 --- a/tests/sensors/test_bash.py +++ b/tests/providers/standard/sensors/test_bash.py @@ -23,7 +23,7 @@ from airflow.exceptions import AirflowFailException, AirflowSensorTimeout from airflow.models.dag import DAG -from airflow.sensors.bash import BashSensor +from airflow.providers.standard.sensors.bash import BashSensor class TestBashSensor: diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index 3d6268834dce2..9947a197a0335 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -35,9 +35,9 @@ from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.standard.sensors.time import TimeSensor from airflow.sensors.external_task import ( ExternalTaskMarker, diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index d063b3e78035a..f0f517042314a 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -60,11 +60,11 @@ from airflow.models.mappedoperator import MappedOperator from airflow.models.param import Param, ParamsDict from airflow.models.xcom import XCom -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator +from airflow.providers.standard.operators.bash import BashOperator +from airflow.providers.standard.sensors.bash import BashSensor from airflow.security import permissions -from airflow.sensors.bash import BashSensor from airflow.serialization.enums import Encoding from airflow.serialization.json_schema import load_dag_schema_dict from airflow.serialization.serialized_objects import ( @@ -154,7 +154,7 @@ "template_fields_renderers": {"bash_command": "bash", "env": "json"}, "bash_command": "echo {{ task.task_id }}", "_task_type": "BashOperator", - "_task_module": "airflow.operators.bash", + "_task_module": "airflow.providers.standard.operators.bash", "pool": "default_pool", "is_setup": False, "is_teardown": False, @@ -2284,7 +2284,7 @@ def test_operator_expand_serde(): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "airflow.operators.bash", + "_task_module": "airflow.providers.standard.operators.bash", "_task_type": "BashOperator", "start_trigger_args": None, "start_from_trigger": False, diff --git a/tests/system/core/example_external_task_child_deferrable.py b/tests/system/core/example_external_task_child_deferrable.py index 9af83b7699ab1..781ad4ea5ef1f 100644 --- a/tests/system/core/example_external_task_child_deferrable.py +++ b/tests/system/core/example_external_task_child_deferrable.py @@ -19,7 +19,7 @@ from datetime import datetime from airflow import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator with DAG( dag_id="child_dag", diff --git a/tests/system/providers/amazon/aws/example_appflow.py b/tests/system/providers/amazon/aws/example_appflow.py index 0fb2764c0b71c..5ba38533b0211 100644 --- a/tests/system/providers/amazon/aws/example_appflow.py +++ b/tests/system/providers/amazon/aws/example_appflow.py @@ -20,7 +20,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.amazon.aws.operators.appflow import ( AppflowRecordsShortCircuitOperator, AppflowRunAfterOperator, @@ -28,6 +27,7 @@ AppflowRunDailyOperator, AppflowRunFullOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() diff --git a/tests/system/providers/amazon/aws/example_http_to_s3.py b/tests/system/providers/amazon/aws/example_http_to_s3.py index 3654140b4a1ed..d6424f9802155 100644 --- a/tests/system/providers/amazon/aws/example_http_to_s3.py +++ b/tests/system/providers/amazon/aws/example_http_to_s3.py @@ -23,9 +23,9 @@ from airflow.models.baseoperator import chain from airflow.models.connection import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.http_to_s3 import HttpToS3Operator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder diff --git a/tests/system/providers/amazon/aws/utils/k8s.py b/tests/system/providers/amazon/aws/utils/k8s.py index 551d09629e9d3..a882d9e42842e 100644 --- a/tests/system/providers/amazon/aws/utils/k8s.py +++ b/tests/system/providers/amazon/aws/utils/k8s.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator if TYPE_CHECKING: from airflow.models.operator import Operator diff --git a/tests/system/providers/apache/hive/example_twitter_dag.py b/tests/system/providers/apache/hive/example_twitter_dag.py index 53b824f50923e..4ceb119ba551c 100644 --- a/tests/system/providers/apache/hive/example_twitter_dag.py +++ b/tests/system/providers/apache/hive/example_twitter_dag.py @@ -26,8 +26,8 @@ from airflow import DAG from airflow.decorators import task -from airflow.operators.bash import BashOperator from airflow.providers.apache.hive.operators.hive import HiveOperator +from airflow.providers.standard.operators.bash import BashOperator # -------------------------------------------------------------------------------- # Caveat: This Dag will not run because of missing scripts. diff --git a/tests/system/providers/apache/iceberg/example_iceberg.py b/tests/system/providers/apache/iceberg/example_iceberg.py index 0318a8e22b770..41e751624b5c0 100644 --- a/tests/system/providers/apache/iceberg/example_iceberg.py +++ b/tests/system/providers/apache/iceberg/example_iceberg.py @@ -19,8 +19,8 @@ from datetime import datetime, timedelta from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.apache.iceberg.hooks.iceberg import IcebergHook +from airflow.providers.standard.operators.bash import BashOperator bash_command = f""" echo "Our token: {IcebergHook().get_token_macro()}" diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes.py b/tests/system/providers/cncf/kubernetes/example_kubernetes.py index 57bab063a9e62..3756d0c4e21d8 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes.py @@ -27,9 +27,9 @@ from kubernetes.client import models as k8s from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret +from airflow.providers.standard.operators.bash import BashOperator # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py index 881bfd61f7c8d..cb3d25a33fcbc 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py @@ -27,9 +27,9 @@ from kubernetes.client import models as k8s from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret +from airflow.providers.standard.operators.bash import BashOperator # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/tests/system/providers/docker/example_docker.py b/tests/system/providers/docker/example_docker.py index 069f4794de632..18f7d2f0ea0c6 100644 --- a/tests/system/providers/docker/example_docker.py +++ b/tests/system/providers/docker/example_docker.py @@ -21,8 +21,8 @@ from datetime import datetime from airflow import models -from airflow.operators.bash import BashOperator from airflow.providers.docker.operators.docker import DockerOperator +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "docker_test" diff --git a/tests/system/providers/docker/example_docker_copy_data.py b/tests/system/providers/docker/example_docker_copy_data.py index 50373af792515..4e4e8466e501f 100644 --- a/tests/system/providers/docker/example_docker_copy_data.py +++ b/tests/system/providers/docker/example_docker_copy_data.py @@ -32,9 +32,9 @@ from docker.types import Mount from airflow import models -from airflow.operators.bash import BashOperator from airflow.operators.python import ShortCircuitOperator from airflow.providers.docker.operators.docker import DockerOperator +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "docker_sample_copy_data" diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py index bd74e49d40964..004f996975be1 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py @@ -25,13 +25,13 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.bigquery import ( BigQueryCreateEmptyDatasetOperator, BigQueryDeleteDatasetOperator, BigQueryGetDatasetOperator, BigQueryUpdateDatasetOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py index 6878a9822de1e..ab7a4b3757b9b 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py @@ -25,7 +25,6 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.bigquery import ( BigQueryCheckOperator, BigQueryColumnCheckOperator, @@ -38,6 +37,7 @@ BigQueryTableCheckOperator, BigQueryValueCheckOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py index 2f1ed573aa053..a007e1cd639c0 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py @@ -25,7 +25,6 @@ from datetime import datetime, timedelta from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.bigquery import ( BigQueryCheckOperator, BigQueryCreateEmptyDatasetOperator, @@ -36,6 +35,7 @@ BigQueryIntervalCheckOperator, BigQueryValueCheckOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py index 0f7acd8a14d36..e9b3269ecfb6c 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py @@ -36,7 +36,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -51,6 +50,7 @@ ComputeEngineInsertInstanceOperator, ) from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py index 990820bfe28c6..4a3b0386da0f4 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py @@ -35,7 +35,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -50,6 +49,7 @@ ) from airflow.providers.google.cloud.transfers.bigquery_to_postgres import BigQueryToPostgresOperator from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py b/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py index 42cfbc8808f81..cb31a3b4d091d 100644 --- a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +++ b/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py @@ -31,7 +31,6 @@ from airflow.decorators import task_group from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.cloud_build import ( CloudBuildCancelBuildOperator, CloudBuildCreateBuildOperator, @@ -39,6 +38,7 @@ CloudBuildListBuildsOperator, CloudBuildRetryBuildOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py index dce93182090d0..4884122751e09 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +++ b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py @@ -32,7 +32,6 @@ from google.protobuf.field_mask_pb2 import FieldMask from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.cloud_memorystore import ( CloudMemorystoreMemcachedApplyParametersOperator, CloudMemorystoreMemcachedCreateInstanceOperator, @@ -42,6 +41,7 @@ CloudMemorystoreMemcachedUpdateInstanceOperator, CloudMemorystoreMemcachedUpdateParametersOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py index aeee437ef9b57..c46d966371dac 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +++ b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py @@ -27,7 +27,6 @@ from google.cloud.redis_v1 import FailoverInstanceRequest, Instance from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.cloud_memorystore import ( CloudMemorystoreCreateInstanceAndImportOperator, CloudMemorystoreCreateInstanceOperator, @@ -46,6 +45,7 @@ GCSCreateBucketOperator, GCSDeleteBucketOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py b/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py index 7d9046088a740..aebb1e3e7ed85 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py +++ b/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py @@ -27,7 +27,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import ( GCSCreateBucketOperator, GCSDeleteBucketOperator, @@ -35,6 +34,7 @@ GCSListObjectsOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py index 9e92102fa924c..55bec85a50562 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py @@ -29,7 +29,6 @@ from airflow.decorators import task from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from airflow.providers.google.cloud.operators.gcs import ( GCSCreateBucketOperator, @@ -39,6 +38,7 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py index 56fb0811b0621..a673ab88f722e 100644 --- a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py @@ -34,7 +34,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -47,6 +46,7 @@ GCSDeleteBucketOperator, ) from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py index 31951cd023010..2860d8552e101 100644 --- a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py @@ -27,9 +27,9 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.sftp_to_gcs import SFTPToGCSOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_sheets.py b/tests/system/providers/google/cloud/gcs/example_sheets.py index 958bd90360285..2247819494f91 100644 --- a/tests/system/providers/google/cloud/gcs/example_sheets.py +++ b/tests/system/providers/google/cloud/gcs/example_sheets.py @@ -26,11 +26,11 @@ from airflow.models import Connection from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.sheets_to_gcs import GoogleSheetsToGCSOperator from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator from airflow.providers.google.suite.transfers.gcs_to_sheets import GCSToGoogleSheetsOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py index 031f8326ee99c..173fddad3a065 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py @@ -25,12 +25,12 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.kubernetes_engine import ( GKECreateClusterOperator, GKEDeleteClusterOperator, GKEStartPodOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py index 5e3f4ddbf7044..e974a628c7a52 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py @@ -25,12 +25,12 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.kubernetes_engine import ( GKECreateClusterOperator, GKEDeleteClusterOperator, GKEStartPodOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py index 87602da88c46c..bde2c0bbaf9ee 100644 --- a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py +++ b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py @@ -29,7 +29,6 @@ from google.protobuf.struct_pb2 import Value from airflow import models -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import ( GCSCreateBucketOperator, GCSDeleteBucketOperator, @@ -53,6 +52,7 @@ ListModelVersionsOperator, SetDefaultVersionOnModelOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") diff --git a/tests/system/providers/google/cloud/natural_language/example_natural_language.py b/tests/system/providers/google/cloud/natural_language/example_natural_language.py index 5bc38f9220874..e04fdf4fb601b 100644 --- a/tests/system/providers/google/cloud/natural_language/example_natural_language.py +++ b/tests/system/providers/google/cloud/natural_language/example_natural_language.py @@ -27,13 +27,13 @@ from google.cloud.language_v1 import Document from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.natural_language import ( CloudNaturalLanguageAnalyzeEntitiesOperator, CloudNaturalLanguageAnalyzeEntitySentimentOperator, CloudNaturalLanguageAnalyzeSentimentOperator, CloudNaturalLanguageClassifyTextOperator, ) +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "gcp_natural_language" diff --git a/tests/system/providers/google/cloud/pubsub/example_pubsub.py b/tests/system/providers/google/cloud/pubsub/example_pubsub.py index 29ba6469ea0ce..4ff3091e5fa53 100644 --- a/tests/system/providers/google/cloud/pubsub/example_pubsub.py +++ b/tests/system/providers/google/cloud/pubsub/example_pubsub.py @@ -25,7 +25,6 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.pubsub import ( PubSubCreateSubscriptionOperator, PubSubCreateTopicOperator, @@ -35,6 +34,7 @@ PubSubPullOperator, ) from airflow.providers.google.cloud.sensors.pubsub import PubSubPullSensor +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py index cf256c8f9fa2f..11231c0dfd40c 100644 --- a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py +++ b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py @@ -34,7 +34,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -45,6 +44,7 @@ from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator from airflow.providers.google.suite.transfers.sql_to_sheets import SQLToGoogleSheetsOperator from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session, json from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/tasks/example_queue.py b/tests/system/providers/google/cloud/tasks/example_queue.py index 53919fb146da6..4c29b584f5bfc 100644 --- a/tests/system/providers/google/cloud/tasks/example_queue.py +++ b/tests/system/providers/google/cloud/tasks/example_queue.py @@ -35,7 +35,6 @@ from airflow.decorators import task from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.tasks import ( CloudTasksQueueCreateOperator, CloudTasksQueueDeleteOperator, @@ -46,6 +45,7 @@ CloudTasksQueuesListOperator, CloudTasksQueueUpdateOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py index 8394c99fcef51..33a289c1ffa1c 100644 --- a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py +++ b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py @@ -33,7 +33,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -47,6 +46,7 @@ ) from airflow.providers.google.cloud.transfers.postgres_to_gcs import PostgresToGCSOperator from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/translate/example_translate.py b/tests/system/providers/google/cloud/translate/example_translate.py index 87f424673ecff..b593060f6e5bb 100644 --- a/tests/system/providers/google/cloud/translate/example_translate.py +++ b/tests/system/providers/google/cloud/translate/example_translate.py @@ -25,8 +25,8 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.translate import CloudTranslateTextOperator +from airflow.providers.standard.operators.bash import BashOperator DAG_ID = "gcp_translate" diff --git a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py index eae6a54a89c3c..499db2d6427ba 100644 --- a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +++ b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py @@ -33,7 +33,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.operators.video_intelligence import ( CloudVideoIntelligenceDetectVideoExplicitContentOperator, @@ -41,6 +40,7 @@ CloudVideoIntelligenceDetectVideoShotsOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py b/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py index 1d6167c6866ee..2a4d7b75f1332 100644 --- a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +++ b/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py @@ -22,7 +22,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.operators.vision import ( CloudVisionDetectImageLabelsOperator, @@ -32,6 +31,7 @@ CloudVisionTextDetectOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule # [START howto_operator_vision_retry_import] diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py b/tests/system/providers/google/datacatalog/example_datacatalog_entries.py index db7d74b18d985..47edfb96368f1 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_entries.py @@ -24,7 +24,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateEntryGroupOperator, CloudDataCatalogCreateEntryOperator, @@ -36,6 +35,7 @@ CloudDataCatalogUpdateEntryOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py b/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py index 8061ecaf110ae..781d047c53469 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py @@ -25,7 +25,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateEntryGroupOperator, CloudDataCatalogCreateEntryOperator, @@ -38,6 +37,7 @@ CloudDataCatalogSearchCatalogOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py b/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py index 6c1fa6f0cab45..b8dd9170c3c02 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py @@ -24,7 +24,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateTagTemplateFieldOperator, CloudDataCatalogCreateTagTemplateOperator, @@ -35,6 +34,7 @@ CloudDataCatalogUpdateTagTemplateFieldOperator, CloudDataCatalogUpdateTagTemplateOperator, ) +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py b/tests/system/providers/google/datacatalog/example_datacatalog_tags.py index 28764c3e0336d..17397fcea2806 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_tags.py @@ -25,7 +25,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateEntryGroupOperator, CloudDataCatalogCreateEntryOperator, @@ -39,6 +38,7 @@ CloudDataCatalogUpdateTagOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/opsgenie/example_opsgenie_notifier.py b/tests/system/providers/opsgenie/example_opsgenie_notifier.py index a9cdd70de0125..10edf8debdaed 100644 --- a/tests/system/providers/opsgenie/example_opsgenie_notifier.py +++ b/tests/system/providers/opsgenie/example_opsgenie_notifier.py @@ -21,8 +21,8 @@ from datetime import datetime from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.opsgenie.notifications.opsgenie import send_opsgenie_notification +from airflow.providers.standard.operators.bash import BashOperator with DAG( "opsgenie_notifier", diff --git a/tests/system/providers/singularity/example_singularity.py b/tests/system/providers/singularity/example_singularity.py index d802fbb31e820..4b60c080dcd2c 100644 --- a/tests/system/providers/singularity/example_singularity.py +++ b/tests/system/providers/singularity/example_singularity.py @@ -21,8 +21,8 @@ from datetime import datetime, timedelta from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.singularity.operators.singularity import SingularityOperator +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "singularity_sample" diff --git a/tests/test_utils/compat.py b/tests/test_utils/compat.py index 09f3653db82d8..67a111350daff 100644 --- a/tests/test_utils/compat.py +++ b/tests/test_utils/compat.py @@ -18,8 +18,6 @@ import contextlib import json -import os -from importlib.metadata import version from typing import TYPE_CHECKING, Any, cast from packaging.version import Version @@ -38,7 +36,6 @@ except ImportError: from airflow.models.errors import ImportError as ParseImportError # type: ignore[no-redef,attr-defined] - from airflow import __version__ as airflow_version AIRFLOW_VERSION = Version(airflow_version) @@ -53,6 +50,16 @@ # Compatibility for Airflow 2.7.* from airflow.models.baseoperator import BaseOperatorLink +try: + from airflow.providers.standard.operators.bash import BashOperator + from airflow.providers.standard.sensors.bash import BashSensor + from airflow.providers.standard.sensors.date_time import DateTimeSensor +except ImportError: + # Compatibility for Airflow < 2.10.* + from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] + from airflow.sensors.bash import BashSensor # type: ignore[no-redef,attr-defined] + from airflow.sensors.date_time import DateTimeSensor # type: ignore[no-redef,attr-defined] + if TYPE_CHECKING: from airflow.models.asset import ( diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index 5cb52696f19ce..0376848fce829 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -23,13 +23,13 @@ import pytest from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator from airflow.serialization.dag_dependency import DagDependency from airflow.utils import dot_renderer, timezone from airflow.utils.state import State from airflow.utils.task_group import TaskGroup +from tests.test_utils.compat import BashOperator from tests.test_utils.db import clear_db_dags START_DATE = timezone.utcnow() diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index 084d8c35ac03e..a6008dc58c03f 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -34,12 +34,12 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator from airflow.utils.dag_edges import dag_edges from airflow.utils.task_group import TASKGROUP_ARGS_EXPECTED_TYPES, TaskGroup, task_group_to_dict from tests.models import DEFAULT_DATE +from tests.test_utils.compat import BashOperator def make_task(name, type_="classic"): diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index f3947b141a347..2d1754af29f65 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -28,7 +28,6 @@ from airflow.models.dag import DAG from airflow.models.renderedtifields import RenderedTaskInstanceFields from airflow.models.variable import Variable -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone @@ -36,7 +35,7 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType from tests.conftest import initial_db_init -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields from tests.test_utils.www import check_content_in_response, check_content_not_in_response diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 7b65051724c27..4dcb7252a3658 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -34,7 +34,6 @@ from airflow.models.taskinstance import TaskInstance from airflow.models.taskreschedule import TaskReschedule from airflow.models.xcom import XCom -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.security import permissions @@ -49,7 +48,7 @@ delete_roles, delete_user, ) -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator from tests.test_utils.config import conf_vars from tests.test_utils.db import clear_db_runs, clear_db_xcom from tests.test_utils.www import check_content_in_response, check_content_not_in_response, client_with_login From bf09c39d634ea7e8f30e2bf8da8af7789cb4fdf2 Mon Sep 17 00:00:00 2001 From: rom sharon <33751805+romsharon98@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:31:17 +0300 Subject: [PATCH 030/199] Fix mark as success when pod fails while fetching log (#42815) * fix: always defer once more after log fetching to ensure pod completion is handled * add tests * removing print --------- Co-authored-by: Jean-Eudes Peloye --- airflow/providers/cncf/kubernetes/operators/pod.py | 4 +--- .../providers/cncf/kubernetes/operators/test_pod.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 5b9e57ec01743..68081b5a67075 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -790,9 +790,7 @@ def trigger_reentry(self, context: Context, event: dict[str, Any]) -> Any: since_time=last_log_time, ) - if pod_log_status.running: - self.log.info("Container still running; deferring again.") - self.invoke_defer_method(pod_log_status.last_log_time) + self.invoke_defer_method(pod_log_status.last_log_time) else: self.invoke_defer_method() diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index be8279bcabd9a..a4ccb4b44b4bf 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -1780,6 +1780,18 @@ def test_process_duplicate_label_pods__pod_removed_if_delete_pod( process_pod_deletion_mock.assert_called_once_with(pod_1) assert result.metadata.name == pod_2.metadata.name + @patch(POD_MANAGER_CLASS.format("fetch_container_logs")) + @patch(KUB_OP_PATH.format("invoke_defer_method")) + def test_defere_call_one_more_time_after_error(self, invoke_defer_method, fetch_container_logs): + fetch_container_logs.return_value = PodLoggingStatus(False, None) + op = KubernetesPodOperator(task_id="test_task", name="test-pod", get_logs=True) + + op.trigger_reentry( + create_context(op), event={"name": TEST_NAME, "namespace": TEST_NAMESPACE, "status": "running"} + ) + + invoke_defer_method.assert_called_with(None) + class TestSuppress: def test__suppress(self, caplog): From 44f0faf5889750885e2f805ef9efd3728da0c97d Mon Sep 17 00:00:00 2001 From: John Bampton Date: Thu, 10 Oct 2024 00:00:11 +1000 Subject: [PATCH 031/199] Fix spelling; `Airlfow` -> `Airflow` (#42855) --- airflow/providers/elasticsearch/CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/elasticsearch/CHANGELOG.rst b/airflow/providers/elasticsearch/CHANGELOG.rst index d54ea95a1185b..2e51c8572b227 100644 --- a/airflow/providers/elasticsearch/CHANGELOG.rst +++ b/airflow/providers/elasticsearch/CHANGELOG.rst @@ -502,7 +502,7 @@ Bug Fixes 3.0.1 (YANKED) .............. -.. warning:: This release has been **yanked** with a reason: ``Elasticsearch provider is incompatible with Airlfow <2.3`` +.. warning:: This release has been **yanked** with a reason: ``Elasticsearch provider is incompatible with Airflow <2.3`` Misc ~~~~~ @@ -512,7 +512,7 @@ Misc 3.0.0 (YANKED) .............. -.. warning:: This release has been **yanked** with a reason: ``Elasticsearch provider is incompatible with Airlfow <2.3`` +.. warning:: This release has been **yanked** with a reason: ``Elasticsearch provider is incompatible with Airflow <2.3`` Breaking changes ~~~~~~~~~~~~~~~~ From f3162735e06aefa56e4bfa217bf07d9f1be1f315 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Wed, 9 Oct 2024 20:02:39 +0530 Subject: [PATCH 032/199] Docs: Add Template field related info for python operators (#42847) closes #39584 This PR adds templating information related to Python operators in How-to Guide --- docs/apache-airflow/howto/operator/python.rst | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/apache-airflow/howto/operator/python.rst b/docs/apache-airflow/howto/operator/python.rst index 2f0defddd886c..e68a257bcd67e 100644 --- a/docs/apache-airflow/howto/operator/python.rst +++ b/docs/apache-airflow/howto/operator/python.rst @@ -79,7 +79,7 @@ Airflow passes in an additional set of keyword arguments: one for each of the :ref:`Jinja template variables ` and a ``templates_dict`` argument. -The ``templates_dict`` argument is templated, so each value in the dictionary +``templates_dict``, ``op_args``, ``op_kwargs`` arguments are templated, so each value in the dictionary is evaluated as a :ref:`Jinja template `. .. tab-set:: @@ -182,6 +182,7 @@ Otherwise you won't have access to the most context variables of Airflow in ``op If you want the context related to datetime objects like ``data_interval_start`` you can add ``pendulum`` and ``lazy_object_proxy``. + .. important:: The Python function body defined to be executed is cut out of the DAG into a temporary file w/o surrounding code. As in the examples you need to add all imports again and you can not rely on variables from the global Python context. @@ -199,6 +200,11 @@ If additional parameters for package installation are needed pass them in via th All supported options are listed in the `requirements file format `_. +Templating +^^^^^^^^^^ + +Jinja templating can be used in same way as described for the :ref:`howto/operator:PythonOperator`. + Virtual environment setup options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -336,6 +342,11 @@ If you want the context related to datetime objects like ``data_interval_start`` If you want to pass variables into the classic :class:`~airflow.operators.python.ExternalPythonOperator` use ``op_args`` and ``op_kwargs``. +Templating +^^^^^^^^^^ + +Jinja templating can be used in same way as described for the :ref:`howto/operator:PythonOperator`. + Context ^^^^^^^ @@ -393,7 +404,10 @@ tasks. :start-after: [START howto_operator_branch_python] :end-before: [END howto_operator_branch_python] -Argument passing and templating options are the same like with :ref:`howto/operator:PythonOperator`. +Passing in arguments and Templating +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Argument passing and templating options are the same as with :ref:`howto/operator:PythonOperator`. .. _howto/operator:BranchPythonVirtualenvOperator: @@ -427,7 +441,10 @@ tasks and is a hybrid of the :class:`~airflow.operators.python.PythonBranchOpera :start-after: [START howto_operator_branch_virtualenv] :end-before: [END howto_operator_branch_virtualenv] -Argument passing and templating options are the same like with :ref:`howto/operator:PythonVirtualenvOperator`. +Passing in arguments and Templating +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Argument passing and templating options are the same as with :ref:`howto/operator:PythonOperator`. .. _howto/operator:BranchExternalPythonOperator: @@ -462,7 +479,11 @@ external Python environment. :start-after: [START howto_operator_branch_ext_py] :end-before: [END howto_operator_branch_ext_py] -Argument passing and templating options are the same like with :ref:`howto/operator:ExternalPythonOperator`. + +Passing in arguments and Templating +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Argument passing and templating options are the same as with :ref:`howto/operator:PythonOperator`. .. _howto/operator:ShortCircuitOperator: @@ -538,16 +559,11 @@ tasks have completed running regardless of status (i.e. the ``TriggerRule.ALL_DO :start-after: [START howto_operator_short_circuit_trigger_rules] :end-before: [END howto_operator_short_circuit_trigger_rules] -Passing in arguments -^^^^^^^^^^^^^^^^^^^^ -Pass extra arguments to the ``@task.short_circuit``-decorated function as you would with a normal Python function. - - -Templating -^^^^^^^^^^ +Passing in arguments and Templating +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Jinja templating can be used in same way as described for the PythonOperator. +Argument passing and templating options are the same as with :ref:`howto/operator:PythonOperator`. .. _howto/operator:PythonSensor: From a343e7805357e79f8fdccbacfb04bfe2a3137105 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Wed, 9 Oct 2024 23:03:55 +0800 Subject: [PATCH 033/199] AIP-84 Get Variable (#42834) Handle redacted value for Variables --- .../endpoints/variable_endpoint.py | 1 + airflow/api_fastapi/openapi/v1-generated.yaml | 66 +++++++++++++++ airflow/api_fastapi/serializers/variables.py | 49 ++++++++++++ airflow/api_fastapi/views/public/variables.py | 16 ++++ airflow/ui/openapi-gen/queries/common.ts | 16 ++++ airflow/ui/openapi-gen/queries/prefetch.ts | 21 +++++ airflow/ui/openapi-gen/queries/queries.ts | 29 +++++++ airflow/ui/openapi-gen/queries/suspense.ts | 30 +++++++ .../ui/openapi-gen/requests/schemas.gen.ts | 35 ++++++++ .../ui/openapi-gen/requests/services.gen.ts | 28 +++++++ airflow/ui/openapi-gen/requests/types.gen.ts | 40 ++++++++++ .../views/public/test_variables.py | 80 +++++++++++++++++-- 12 files changed, 406 insertions(+), 5 deletions(-) create mode 100644 airflow/api_fastapi/serializers/variables.py diff --git a/airflow/api_connexion/endpoints/variable_endpoint.py b/airflow/api_connexion/endpoints/variable_endpoint.py index 9413f9158652d..8efddb58419c4 100644 --- a/airflow/api_connexion/endpoints/variable_endpoint.py +++ b/airflow/api_connexion/endpoints/variable_endpoint.py @@ -59,6 +59,7 @@ def delete_variable(*, variable_key: str) -> Response: return Response(status=HTTPStatus.NO_CONTENT) +@mark_fastapi_migration_done @security.requires_access_variable("GET") @provide_session def get_variable(*, variable_key: str, session: Session = NEW_SESSION) -> Response: diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index 28723b800879e..463cc1e92f4d6 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -535,6 +535,50 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Variable + summary: Get Variable + description: Get a variable entry. + operationId: get_variable + parameters: + - name: variable_key + in: path + required: true + schema: + type: string + title: Variable Key + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/VariableResponse' + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' components: schemas: ConnectionResponse: @@ -1193,3 +1237,25 @@ components: - msg - type title: ValidationError + VariableResponse: + properties: + key: + type: string + title: Key + value: + anyOf: + - type: string + - type: 'null' + title: Value + description: + anyOf: + - type: string + - type: 'null' + title: Description + type: object + required: + - key + - value + - description + title: VariableResponse + description: Variable serializer for responses. diff --git a/airflow/api_fastapi/serializers/variables.py b/airflow/api_fastapi/serializers/variables.py new file mode 100644 index 0000000000000..ded268432b89d --- /dev/null +++ b/airflow/api_fastapi/serializers/variables.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import json + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Self + +from airflow.utils.log.secrets_masker import redact + + +class VariableResponse(BaseModel): + """Variable serializer for responses.""" + + model_config = ConfigDict(populate_by_name=True) + + key: str + val: str | None = Field(alias="value") + description: str | None + + @model_validator(mode="after") + def redact_val(self) -> Self: + if self.val is None: + return self + try: + val_dict = json.loads(self.val) + redacted_dict = redact(val_dict, max_depth=1) + self.val = json.dumps(redacted_dict) + return self + except json.JSONDecodeError: + # value is not a serialized string representation of a dict. + self.val = redact(self.val, self.key) + return self diff --git a/airflow/api_fastapi/views/public/variables.py b/airflow/api_fastapi/views/public/variables.py index e4edb8601fd09..e6cbb136f1cac 100644 --- a/airflow/api_fastapi/views/public/variables.py +++ b/airflow/api_fastapi/views/public/variables.py @@ -17,11 +17,13 @@ from __future__ import annotations from fastapi import Depends, HTTPException +from sqlalchemy import select from sqlalchemy.orm import Session from typing_extensions import Annotated from airflow.api_fastapi.db.common import get_session from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.serializers.variables import VariableResponse from airflow.api_fastapi.views.router import AirflowRouter from airflow.models.variable import Variable @@ -40,3 +42,17 @@ async def delete_variable( """Delete a variable entry.""" if Variable.delete(variable_key, session) == 0: raise HTTPException(404, f"The Variable with key: `{variable_key}` was not found") + + +@variables_router.get("/{variable_key}", responses=create_openapi_http_exception_doc([401, 403, 404])) +async def get_variable( + variable_key: str, + session: Annotated[Session, Depends(get_session)], +) -> VariableResponse: + """Get a variable entry.""" + variable = session.scalar(select(Variable).where(Variable.key == variable_key).limit(1)) + + if variable is None: + raise HTTPException(404, f"The Variable with key: `{variable_key}` was not found") + + return VariableResponse.model_validate(variable, from_attributes=True) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index fbbbac5d60625..72fd0ef9ccde1 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -134,6 +134,22 @@ export const UseConnectionServiceGetConnectionKeyFn = ( useConnectionServiceGetConnectionKey, ...(queryKey ?? [{ connectionId }]), ]; +export type VariableServiceGetVariableDefaultResponse = Awaited< + ReturnType +>; +export type VariableServiceGetVariableQueryResult< + TData = VariableServiceGetVariableDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useVariableServiceGetVariableKey = "VariableServiceGetVariable"; +export const UseVariableServiceGetVariableKeyFn = ( + { + variableKey, + }: { + variableKey: string; + }, + queryKey?: Array, +) => [useVariableServiceGetVariableKey, ...(queryKey ?? [{ variableKey }])]; export type DagServicePatchDagsMutationResult = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index 7c8555b29d264..a114b9dc92c6c 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -6,6 +6,7 @@ import { ConnectionService, DagService, DashboardService, + VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; import * as Common from "./common"; @@ -165,3 +166,23 @@ export const prefetchUseConnectionServiceGetConnection = ( queryKey: Common.UseConnectionServiceGetConnectionKeyFn({ connectionId }), queryFn: () => ConnectionService.getConnection({ connectionId }), }); +/** + * Get Variable + * Get a variable entry. + * @param data The data for the request. + * @param data.variableKey + * @returns VariableResponse Successful Response + * @throws ApiError + */ +export const prefetchUseVariableServiceGetVariable = ( + queryClient: QueryClient, + { + variableKey, + }: { + variableKey: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseVariableServiceGetVariableKeyFn({ variableKey }), + queryFn: () => VariableService.getVariable({ variableKey }), + }); diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 9137ea4ed01bc..a3ce02257160a 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -208,6 +208,35 @@ export const useConnectionServiceGetConnection = < queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, ...options, }); +/** + * Get Variable + * Get a variable entry. + * @param data The data for the request. + * @param data.variableKey + * @returns VariableResponse Successful Response + * @throws ApiError + */ +export const useVariableServiceGetVariable = < + TData = Common.VariableServiceGetVariableDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + variableKey, + }: { + variableKey: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseVariableServiceGetVariableKeyFn( + { variableKey }, + queryKey, + ), + queryFn: () => VariableService.getVariable({ variableKey }) as TData, + ...options, + }); /** * Patch Dags * Patch multiple DAGs. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index 70be4beb0d479..fbef843c6e0ab 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -6,6 +6,7 @@ import { ConnectionService, DagService, DashboardService, + VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; import * as Common from "./common"; @@ -202,3 +203,32 @@ export const useConnectionServiceGetConnectionSuspense = < queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, ...options, }); +/** + * Get Variable + * Get a variable entry. + * @param data The data for the request. + * @param data.variableKey + * @returns VariableResponse Successful Response + * @throws ApiError + */ +export const useVariableServiceGetVariableSuspense = < + TData = Common.VariableServiceGetVariableDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + variableKey, + }: { + variableKey: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseVariableServiceGetVariableKeyFn( + { variableKey }, + queryKey, + ), + queryFn: () => VariableService.getVariable({ variableKey }) as TData, + ...options, + }); diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 26fe1180ae7b5..8f76ebd13c40f 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -1019,3 +1019,38 @@ export const $ValidationError = { required: ["loc", "msg", "type"], title: "ValidationError", } as const; + +export const $VariableResponse = { + properties: { + key: { + type: "string", + title: "Key", + }, + value: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Value", + }, + description: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Description", + }, + }, + type: "object", + required: ["key", "value", "description"], + title: "VariableResponse", + description: "Variable serializer for responses.", +} as const; diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 268f636404b16..7f61fd32f3493 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -21,6 +21,8 @@ import type { GetConnectionResponse, DeleteVariableData, DeleteVariableResponse, + GetVariableData, + GetVariableResponse, } from "./types.gen"; export class AssetService { @@ -305,4 +307,30 @@ export class VariableService { }, }); } + + /** + * Get Variable + * Get a variable entry. + * @param data The data for the request. + * @param data.variableKey + * @returns VariableResponse Successful Response + * @throws ApiError + */ + public static getVariable( + data: GetVariableData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/variables/{variable_key}", + path: { + variable_key: data.variableKey, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Validation Error", + }, + }); + } } diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 268960a596870..7b5fc54065a01 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -206,6 +206,15 @@ export type ValidationError = { type: string; }; +/** + * Variable serializer for responses. + */ +export type VariableResponse = { + key: string; + value: string | null; + description: string | null; +}; + export type NextRunAssetsData = { dagId: string; }; @@ -283,6 +292,12 @@ export type DeleteVariableData = { export type DeleteVariableResponse = void; +export type GetVariableData = { + variableKey: string; +}; + +export type GetVariableResponse = VariableResponse; + export type $OpenApiTs = { "/ui/next_run_assets/{dag_id}": { get: { @@ -504,5 +519,30 @@ export type $OpenApiTs = { 422: HTTPValidationError; }; }; + get: { + req: GetVariableData; + res: { + /** + * Successful Response + */ + 200: VariableResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; }; }; diff --git a/tests/api_fastapi/views/public/test_variables.py b/tests/api_fastapi/views/public/test_variables.py index 56d65b98b6589..0ee2cebd77340 100644 --- a/tests/api_fastapi/views/public/test_variables.py +++ b/tests/api_fastapi/views/public/test_variables.py @@ -25,15 +25,41 @@ pytestmark = pytest.mark.db_test TEST_VARIABLE_KEY = "test_variable_key" -TEST_VARIABLE_VAL = 3 +TEST_VARIABLE_VALUE = "test_variable_value" TEST_VARIABLE_DESCRIPTION = "Some description for the variable" -TEST_CONN_TYPE = "test_type" + + +TEST_VARIABLE_KEY2 = "password" +TEST_VARIABLE_VALUE2 = "some_password" +TEST_VARIABLE_DESCRIPTION2 = "Some description for the password" + + +TEST_VARIABLE_KEY3 = "dictionary_password" +TEST_VARIABLE_VALUE3 = '{"password": "some_password"}' +TEST_VARIABLE_DESCRIPTION3 = "Some description for the variable" @provide_session def _create_variable(session) -> None: Variable.set( - key=TEST_VARIABLE_KEY, value=TEST_VARIABLE_VAL, description=TEST_VARIABLE_DESCRIPTION, session=session + key=TEST_VARIABLE_KEY, + value=TEST_VARIABLE_VALUE, + description=TEST_VARIABLE_DESCRIPTION, + session=session, + ) + + Variable.set( + key=TEST_VARIABLE_KEY2, + value=TEST_VARIABLE_VALUE2, + description=TEST_VARIABLE_DESCRIPTION2, + session=session, + ) + + Variable.set( + key=TEST_VARIABLE_KEY3, + value=TEST_VARIABLE_VALUE3, + description=TEST_VARIABLE_DESCRIPTION3, + session=session, ) @@ -53,14 +79,58 @@ class TestDeleteVariable(TestVariableEndpoint): def test_delete_should_respond_204(self, test_client, session): self.create_variable() variables = session.query(Variable).all() - assert len(variables) == 1 + assert len(variables) == 3 response = test_client.delete(f"/public/variables/{TEST_VARIABLE_KEY}") assert response.status_code == 204 variables = session.query(Variable).all() - assert len(variables) == 0 + assert len(variables) == 2 def test_delete_should_respond_404(self, test_client): response = test_client.delete(f"/public/variables/{TEST_VARIABLE_KEY}") assert response.status_code == 404 body = response.json() assert f"The Variable with key: `{TEST_VARIABLE_KEY}` was not found" == body["detail"] + + +class TestGetVariable(TestVariableEndpoint): + @pytest.mark.enable_redact + @pytest.mark.parametrize( + "key, expected_response", + [ + ( + TEST_VARIABLE_KEY, + { + "key": TEST_VARIABLE_KEY, + "value": TEST_VARIABLE_VALUE, + "description": TEST_VARIABLE_DESCRIPTION, + }, + ), + ( + TEST_VARIABLE_KEY2, + { + "key": TEST_VARIABLE_KEY2, + "value": "***", + "description": TEST_VARIABLE_DESCRIPTION2, + }, + ), + ( + TEST_VARIABLE_KEY3, + { + "key": TEST_VARIABLE_KEY3, + "value": '{"password": "***"}', + "description": TEST_VARIABLE_DESCRIPTION3, + }, + ), + ], + ) + def test_get_should_respond_200(self, test_client, session, key, expected_response): + self.create_variable() + response = test_client.get(f"/public/variables/{key}") + assert response.status_code == 200 + assert response.json() == expected_response + + def test_get_should_respond_404(self, test_client): + response = test_client.get(f"/public/variables/{TEST_VARIABLE_KEY}") + assert response.status_code == 404 + body = response.json() + assert f"The Variable with key: `{TEST_VARIABLE_KEY}` was not found" == body["detail"] From ddc9d80b6cec0a88fe2be93b2e4d0c13aa9051f5 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Wed, 9 Oct 2024 08:15:37 -0700 Subject: [PATCH 034/199] Move test of DagRun.update_state to better place (#42845) Previously this lived in test_scheduler_job.py It only really tested the behavior of DagRun.update_state. As far as I can tell, it checks that if you null out the state on a TI of a finished dag, and then you call ``update_state``, then the DR will be set to running. --- tests/jobs/test_scheduler_job.py | 35 ------------------------------ tests/models/test_dagrun.py | 37 +++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index d26369f5d728e..bc2be0a12c743 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -2950,41 +2950,6 @@ def test_dagrun_root_fail(self): dagrun_state=State.FAILED, ) - def test_dagrun_root_fail_unfinished(self): - """ - DagRuns with one unfinished and one failed root task -> RUNNING - """ - # TODO: this should live in test_dagrun.py - # Run both the failed and successful tasks - dag_id = "test_dagrun_states_root_fail_unfinished" - dag = self.dagbag.get_dag(dag_id) - data_interval = dag.infer_automated_data_interval(DEFAULT_LOGICAL_DATE) - triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = dag.create_dagrun( - run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, - state=None, - data_interval=data_interval, - **triggered_by_kwargs, - ) - self.null_exec.mock_task_fail(dag_id, "test_dagrun_fail", dr.run_id) - - # todo: AIP-78 remove this test along with DAG.run() - # this only tests the backfill job runner, not the scheduler - with pytest.warns(RemovedInAirflow3Warning): - for _ in _mock_executor(self.null_exec): - with pytest.raises(AirflowException): - dag.run(start_date=dr.execution_date, end_date=dr.execution_date) - - # Mark the successful task as never having run since we want to see if the - # dagrun will be in a running state despite having an unfinished task. - with create_session() as session: - ti = dr.get_task_instance("test_dagrun_unfinished", session=session) - ti.state = State.NONE - session.commit() - dr.update_state() - assert dr.state == State.RUNNING - def test_dagrun_root_after_dagrun_unfinished(self, mock_executor): """ DagRuns with one successful and one future root task -> SUCCESS diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index 9184f561b3df9..dac2982b0ba34 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -37,7 +37,7 @@ from airflow.models.taskmap import TaskMap from airflow.models.taskreschedule import TaskReschedule from airflow.operators.empty import EmptyOperator -from airflow.operators.python import ShortCircuitOperator +from airflow.operators.python import PythonOperator, ShortCircuitOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.stats import Stats @@ -1050,6 +1050,41 @@ def test_states_sets(self, session): assert ti_success.state in State.success_states assert ti_failed.state in State.failed_states + def test_update_state_one_unfinished(self, dag_maker, session): + """ + Previously this lived in test_scheduler_job.py + + It only really tested the behavior of DagRun.update_state. + + As far as I can tell, it checks that if you null out the state on a TI of a finished dag, + and then you call ``update_state``, then the DR will be set to running. + """ + with dag_maker(session=session) as dag: + PythonOperator(task_id="t1", python_callable=lambda: print) + PythonOperator(task_id="t2", python_callable=lambda: print) + dr = dag.create_dagrun( + state=DagRunState.FAILED, + triggered_by=DagRunTriggeredByType.TEST, + run_id="abc123", + session=session, + ) + for ti in dr.get_task_instances(session=session): + ti.state = TaskInstanceState.FAILED + session.commit() + session.expunge_all() + dr = session.get(DagRun, dr.id) + assert dr.state == DagRunState.FAILED + ti = dr.get_task_instance("t1", session=session) + ti.state = State.NONE + session.commit() + dr = session.get(DagRun, dr.id) + assert dr.state == DagRunState.FAILED + dr.dag = dag + dr.update_state(session=session) + session.commit() + dr = session.get(DagRun, dr.id) + assert dr.state == State.RUNNING + @pytest.mark.parametrize( ("run_type", "expected_tis"), From d047fa8c84514dc3cc394457fc08d89801b89622 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Wed, 9 Oct 2024 17:28:06 +0200 Subject: [PATCH 035/199] Revert "Remove `sqlalchemy-redshift` dependency from Amazon provider (#42830)" (#42864) This reverts commit 3b4c73a644f40013927fb72227dfe018dc47a97f. --- airflow/providers/amazon/provider.yaml | 1 + docs/apache-airflow-providers-amazon/index.rst | 1 + generated/provider_dependencies.json | 1 + 3 files changed, 3 insertions(+) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 741efb70e7e51..1316cd05231a0 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -105,6 +105,7 @@ dependencies: - watchtower>=3.0.0,!=3.3.0,<4 - jsonpath_ng>=1.5.3 - redshift_connector>=2.0.918 + - sqlalchemy_redshift>=0.8.6 - asgiref>=2.3.0 - PyAthena>=3.0.10 - jmespath>=0.7.0 diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index 88e869bba8016..c58847dcd8876 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -119,6 +119,7 @@ PIP package Version required ``watchtower`` ``>=3.0.0,!=3.3.0,<4`` ``jsonpath_ng`` ``>=1.5.3`` ``redshift_connector`` ``>=2.0.918`` +``sqlalchemy_redshift`` ``>=0.8.6`` ``asgiref`` ``>=2.3.0`` ``PyAthena`` ``>=3.0.10`` ``jmespath`` ``>=0.7.0`` diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 57bca636ed605..2d0ab90a3508b 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -38,6 +38,7 @@ "jsonpath_ng>=1.5.3", "python3-saml>=1.16.0", "redshift_connector>=2.0.918", + "sqlalchemy_redshift>=0.8.6", "watchtower>=3.0.0,!=3.3.0,<4" ], "devel-deps": [ From 814786aef3e1a651b70e9dc5f3af1f76ce9675ba Mon Sep 17 00:00:00 2001 From: Elad Kalif <45845474+eladkal@users.noreply.github.com> Date: Wed, 9 Oct 2024 23:10:46 +0700 Subject: [PATCH 036/199] Prepare docs for Oct 1st adhoc wave of providers (#42862) * Prepare docs for Oct 1st adhoc wave of providers * Update * update amazon --- airflow/providers/amazon/CHANGELOG.rst | 58 +++++++++++++------ airflow/providers/amazon/__init__.py | 2 +- airflow/providers/amazon/provider.yaml | 3 +- airflow/providers/apache/beam/CHANGELOG.rst | 12 ++++ airflow/providers/apache/beam/__init__.py | 2 +- airflow/providers/apache/beam/provider.yaml | 3 +- airflow/providers/apache/kafka/CHANGELOG.rst | 12 ++++ airflow/providers/apache/kafka/__init__.py | 2 +- airflow/providers/apache/kafka/provider.yaml | 3 +- airflow/providers/apache/spark/CHANGELOG.rst | 8 +++ airflow/providers/apache/spark/__init__.py | 2 +- airflow/providers/apache/spark/provider.yaml | 3 +- airflow/providers/celery/CHANGELOG.rst | 18 ++++++ airflow/providers/celery/__init__.py | 2 +- airflow/providers/celery/provider.yaml | 3 +- airflow/providers/cloudant/CHANGELOG.rst | 12 ++++ airflow/providers/cloudant/__init__.py | 2 +- airflow/providers/cloudant/provider.yaml | 3 +- .../providers/cncf/kubernetes/CHANGELOG.rst | 26 ++++++++- airflow/providers/cncf/kubernetes/__init__.py | 2 +- airflow/providers/common/compat/CHANGELOG.rst | 12 ++++ airflow/providers/common/compat/__init__.py | 2 +- airflow/providers/common/compat/provider.yaml | 3 +- airflow/providers/common/io/CHANGELOG.rst | 13 +++++ airflow/providers/common/io/__init__.py | 2 +- airflow/providers/common/io/provider.yaml | 3 +- airflow/providers/common/sql/CHANGELOG.rst | 17 ++++++ airflow/providers/common/sql/__init__.py | 2 +- airflow/providers/common/sql/provider.yaml | 3 +- airflow/providers/databricks/CHANGELOG.rst | 18 ++++++ airflow/providers/databricks/__init__.py | 2 +- airflow/providers/databricks/provider.yaml | 3 +- airflow/providers/dbt/cloud/CHANGELOG.rst | 17 ++++++ airflow/providers/dbt/cloud/__init__.py | 2 +- airflow/providers/dbt/cloud/provider.yaml | 3 +- airflow/providers/elasticsearch/CHANGELOG.rst | 13 +++++ airflow/providers/elasticsearch/__init__.py | 2 +- airflow/providers/elasticsearch/provider.yaml | 3 +- airflow/providers/fab/CHANGELOG.rst | 14 +++++ airflow/providers/fab/__init__.py | 2 +- airflow/providers/fab/provider.yaml | 3 +- airflow/providers/google/CHANGELOG.rst | 30 ++++++++++ airflow/providers/google/__init__.py | 2 +- airflow/providers/google/provider.yaml | 3 +- airflow/providers/jdbc/CHANGELOG.rst | 12 ++++ airflow/providers/jdbc/__init__.py | 2 +- airflow/providers/jdbc/provider.yaml | 3 +- .../providers/microsoft/azure/CHANGELOG.rst | 18 ++++++ airflow/providers/microsoft/azure/__init__.py | 2 +- .../providers/microsoft/azure/provider.yaml | 3 +- airflow/providers/mysql/CHANGELOG.rst | 12 ++++ airflow/providers/mysql/__init__.py | 2 +- airflow/providers/mysql/provider.yaml | 3 +- airflow/providers/openlineage/CHANGELOG.rst | 14 +++++ airflow/providers/openlineage/__init__.py | 2 +- airflow/providers/openlineage/provider.yaml | 3 +- airflow/providers/opensearch/CHANGELOG.rst | 22 +++++++ airflow/providers/opensearch/__init__.py | 2 +- airflow/providers/opensearch/provider.yaml | 3 +- airflow/providers/postgres/CHANGELOG.rst | 12 ++++ airflow/providers/postgres/__init__.py | 2 +- airflow/providers/postgres/provider.yaml | 3 +- airflow/providers/snowflake/CHANGELOG.rst | 18 ++++++ airflow/providers/snowflake/__init__.py | 2 +- airflow/providers/snowflake/provider.yaml | 3 +- airflow/providers/trino/CHANGELOG.rst | 12 ++++ airflow/providers/trino/__init__.py | 2 +- airflow/providers/trino/provider.yaml | 3 +- airflow/providers/ydb/CHANGELOG.rst | 12 ++++ airflow/providers/ydb/__init__.py | 2 +- airflow/providers/ydb/provider.yaml | 3 +- .../commits.rst | 28 ++++++++- .../apache-airflow-providers-amazon/index.rst | 6 +- .../commits.rst | 14 ++++- .../index.rst | 6 +- .../commits.rst | 14 ++++- .../index.rst | 2 +- .../commits.rst | 14 ++++- .../index.rst | 6 +- .../commits.rst | 22 +++++-- .../apache-airflow-providers-celery/index.rst | 7 ++- .../commits.rst | 14 ++++- .../index.rst | 2 +- .../commits.rst | 22 ++++++- .../index.rst | 2 +- .../commits.rst | 14 ++++- .../index.rst | 30 +++++++++- .../commits.rst | 15 ++++- .../index.rst | 6 +- .../commits.rst | 15 ++++- .../index.rst | 6 +- .../commits.rst | 16 ++++- .../index.rst | 6 +- .../commits.rst | 21 +++++-- .../index.rst | 6 +- .../commits.rst | 15 ++++- .../index.rst | 6 +- docs/apache-airflow-providers-fab/commits.rst | 14 +++++ docs/apache-airflow-providers-fab/index.rst | 51 ++++++++++++---- .../commits.rst | 24 +++++++- .../apache-airflow-providers-google/index.rst | 6 +- .../apache-airflow-providers-jdbc/commits.rst | 14 ++++- docs/apache-airflow-providers-jdbc/index.rst | 6 +- .../commits.rst | 16 ++++- .../index.rst | 14 ++--- .../commits.rst | 14 ++++- docs/apache-airflow-providers-mysql/index.rst | 6 +- .../commits.rst | 16 ++++- .../index.rst | 19 +++--- .../commits.rst | 16 ++++- .../index.rst | 2 +- .../commits.rst | 14 ++++- .../index.rst | 6 +- .../commits.rst | 16 ++++- .../index.rst | 7 ++- .../commits.rst | 14 ++++- docs/apache-airflow-providers-trino/index.rst | 6 +- docs/apache-airflow-providers-ydb/commits.rst | 14 ++++- docs/apache-airflow-providers-ydb/index.rst | 6 +- 119 files changed, 970 insertions(+), 175 deletions(-) diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index f4837c54fc366..8099f3943aac9 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -26,12 +26,30 @@ Changelog --------- -Main -...... +9.0.0 +..... Breaking changes ~~~~~~~~~~~~~~~~ +.. warning:: + In order to support session reuse in RedshiftData operators, the following breaking changes were introduced: + + The ``database`` argument is now optional and as a result was moved after the ``sql`` argument which is a positional + one. Update your DAGs accordingly if they rely on argument order. Applies to: + + * ``RedshiftDataHook``'s ``execute_query`` method + * ``RedshiftDataOperator`` + + ``RedshiftDataHook``'s ``execute_query`` method now returns a ``QueryExecutionOutput`` object instead of just the + statement ID as a string. + + ``RedshiftDataHook``'s ``parse_statement_resposne`` method was renamed to ``parse_statement_response``. + + ``S3ToRedshiftOperator``'s ``schema`` argument is now optional and was moved after the ``s3_key`` positional argument. + Update your DAGs accordingly if they rely on argument order. + + .. warning:: All deprecated classes, parameters and features have been removed from the Amazon provider package. The following breaking changes were introduced: @@ -123,27 +141,33 @@ Breaking changes * Removed ``host`` from AWS connection, please set it in ``extra['endpoint_url']`` instead * Removed ``region`` parameter from ``AwsHookParams``. Use ``region_name`` instead -.. warning:: - In order to support session reuse in RedshiftData operators, the following breaking changes were introduced: - - The ``database`` argument is now optional and as a result was moved after the ``sql`` argument which is a positional - one. Update your DAGs accordingly if they rely on argument order. Applies to: +* ``Remove deprecated stuff from Amazon provider package (#42450)`` +* ``Support session reuse in 'RedshiftDataOperator' (#42218)`` - * ``RedshiftDataHook``'s ``execute_query`` method - * ``RedshiftDataOperator`` +Features +~~~~~~~~ - ``RedshiftDataHook``'s ``execute_query`` method now returns a ``QueryExecutionOutput`` object instead of just the - statement ID as a string. +* ``Add STOPPED to the failure cases for Sagemaker Training Jobs (#42423)`` - ``RedshiftDataHook``'s ``parse_statement_resposne`` method was renamed to ``parse_statement_response``. +Bug Fixes +~~~~~~~~~ - ``S3ToRedshiftOperator``'s ``schema`` argument is now optional and was moved after the ``s3_key`` positional argument. - Update your DAGs accordingly if they rely on argument order. +* ``'S3DeleteObjects' Operator: Handle dates passed as strings (#42464)`` +* ``Small fix to AWS AVP cli init script (#42479)`` +* ``Make the AWS logging faster by reducing the amount of sleep (#42449)`` +* ``Fix logout in AWS auth manager (#42447)`` +* ``fix(providers/amazon): handle ClientError raised after key is missing during table.get_item (#42408)`` -Features -~~~~~~~~ +Misc +~~~~ -* ``Support session reuse in RedshiftDataOperator, RedshiftToS3Operator and S3ToRedshiftOperator (#42218)`` +* ``Drop python3.8 support core and providers (#42766)`` +* ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` +* ``Rename dataset related python variable names to asset (#41348)`` +* ``Remove identity center auth manager cli (#42481)`` +* ``Refactor AWS Auth manager user output (#42454)`` +* ``Remove 'sqlalchemy-redshift' dependency from Amazon provider (#42830)`` +* ``Revert "Remove 'sqlalchemy-redshift' dependency from Amazon provider" (#42864)`` 8.29.0 ...... diff --git a/airflow/providers/amazon/__init__.py b/airflow/providers/amazon/__init__.py index 0ec3cb606e99e..2a7f8d044e6c5 100644 --- a/airflow/providers/amazon/__init__.py +++ b/airflow/providers/amazon/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "8.29.0" +__version__ = "9.0.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 1316cd05231a0..71d097f298198 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -22,9 +22,10 @@ description: | Amazon integration (including `Amazon Web Services (AWS) `__). state: ready -source-date-epoch: 1726859877 +source-date-epoch: 1728484310 # note that those versions are maintained by release manager - do not update them manually versions: + - 9.0.0 - 8.29.0 - 8.28.0 - 8.27.0 diff --git a/airflow/providers/apache/beam/CHANGELOG.rst b/airflow/providers/apache/beam/CHANGELOG.rst index 85f0ac8f72684..019858bae3cf1 100644 --- a/airflow/providers/apache/beam/CHANGELOG.rst +++ b/airflow/providers/apache/beam/CHANGELOG.rst @@ -26,6 +26,18 @@ Changelog --------- +5.8.1 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Bugfix/dataflow job location passing (#41887)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 5.8.0 ..... diff --git a/airflow/providers/apache/beam/__init__.py b/airflow/providers/apache/beam/__init__.py index f88dd1a9909c9..5b844184bd12b 100644 --- a/airflow/providers/apache/beam/__init__.py +++ b/airflow/providers/apache/beam/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "5.8.0" +__version__ = "5.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/apache/beam/provider.yaml b/airflow/providers/apache/beam/provider.yaml index 8011f564d7194..ae0c4d37005b8 100644 --- a/airflow/providers/apache/beam/provider.yaml +++ b/airflow/providers/apache/beam/provider.yaml @@ -22,9 +22,10 @@ description: | `Apache Beam `__. state: ready -source-date-epoch: 1723969156 +source-date-epoch: 1728484489 # note that those versions are maintained by release manager - do not update them manually versions: + - 5.8.1 - 5.8.0 - 5.7.2 - 5.7.1 diff --git a/airflow/providers/apache/kafka/CHANGELOG.rst b/airflow/providers/apache/kafka/CHANGELOG.rst index 045475e755b52..38e11241ca3a8 100644 --- a/airflow/providers/apache/kafka/CHANGELOG.rst +++ b/airflow/providers/apache/kafka/CHANGELOG.rst @@ -27,6 +27,18 @@ Changelog --------- +1.6.1 +..... + +Bug Fixes +~~~~~~~~~ + +* ``remove callable functions parameter from kafka operator template_fields (#42555)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.6.0 ..... diff --git a/airflow/providers/apache/kafka/__init__.py b/airflow/providers/apache/kafka/__init__.py index 862c4c19b49a9..b7050a89dab8e 100644 --- a/airflow/providers/apache/kafka/__init__.py +++ b/airflow/providers/apache/kafka/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.6.0" +__version__ = "1.6.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/apache/kafka/provider.yaml b/airflow/providers/apache/kafka/provider.yaml index 8a577ba4cdad0..23961ec3c89f5 100644 --- a/airflow/providers/apache/kafka/provider.yaml +++ b/airflow/providers/apache/kafka/provider.yaml @@ -20,11 +20,12 @@ package-name: apache-airflow-providers-apache-kafka name: Apache Kafka state: ready -source-date-epoch: 1723969246 +source-date-epoch: 1728484555 description: | `Apache Kafka `__ # note that those versions are maintained by release manager - do not update them manually versions: + - 1.6.1 - 1.6.0 - 1.5.0 - 1.4.1 diff --git a/airflow/providers/apache/spark/CHANGELOG.rst b/airflow/providers/apache/spark/CHANGELOG.rst index b8773628e3634..df28a68ef7b5a 100644 --- a/airflow/providers/apache/spark/CHANGELOG.rst +++ b/airflow/providers/apache/spark/CHANGELOG.rst @@ -29,6 +29,14 @@ Changelog --------- +4.11.1 +...... + +Misc +~~~~ + +* ``Refactor function resolve_kerberos_principal (#42777)`` + 4.11.0 ...... diff --git a/airflow/providers/apache/spark/__init__.py b/airflow/providers/apache/spark/__init__.py index a652688b00546..a5873c7b144aa 100644 --- a/airflow/providers/apache/spark/__init__.py +++ b/airflow/providers/apache/spark/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.11.0" +__version__ = "4.11.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index 77ee6d109970c..8d9c2c704ae5d 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -22,9 +22,10 @@ description: | `Apache Spark `__ state: ready -source-date-epoch: 1726860120 +source-date-epoch: 1728484593 # note that those versions are maintained by release manager - do not update them manually versions: + - 4.11.1 - 4.11.0 - 4.10.0 - 4.9.0 diff --git a/airflow/providers/celery/CHANGELOG.rst b/airflow/providers/celery/CHANGELOG.rst index 435edfd6600b8..4043aefb1d13a 100644 --- a/airflow/providers/celery/CHANGELOG.rst +++ b/airflow/providers/celery/CHANGELOG.rst @@ -27,6 +27,24 @@ Changelog --------- +3.8.3 +..... + +Bug Fixes +~~~~~~~~~ + +* ``All executors should inherit from BaseExecutor (#41904)`` +* ``Remove state sync during celery task processing (#41870)`` + +Misc +~~~~ + +* ``Change imports to use Standard provider for BashOperator (#42252)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.8.2 ..... diff --git a/airflow/providers/celery/__init__.py b/airflow/providers/celery/__init__.py index 826a046783995..07ca45921e83c 100644 --- a/airflow/providers/celery/__init__.py +++ b/airflow/providers/celery/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.8.2" +__version__ = "3.8.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index e0cf09a515c6b..156a810c1fee4 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -22,9 +22,10 @@ description: | `Celery `__ state: ready -source-date-epoch: 1726860145 +source-date-epoch: 1728484718 # note that those versions are maintained by release manager - do not update them manually versions: + - 3.8.3 - 3.8.2 - 3.8.1 - 3.8.0 diff --git a/airflow/providers/cloudant/CHANGELOG.rst b/airflow/providers/cloudant/CHANGELOG.rst index a32cd2b42f2ec..e4446503a8cf6 100644 --- a/airflow/providers/cloudant/CHANGELOG.rst +++ b/airflow/providers/cloudant/CHANGELOG.rst @@ -27,6 +27,18 @@ Changelog --------- +4.0.1 +..... + +Misc +~~~~ + +* ``Drop python3.8 support core and providers (#42766)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.0.0 ..... diff --git a/airflow/providers/cloudant/__init__.py b/airflow/providers/cloudant/__init__.py index 05483dfd46169..db8335bb72c79 100644 --- a/airflow/providers/cloudant/__init__.py +++ b/airflow/providers/cloudant/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.0.0" +__version__ = "4.0.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml index a2de5b4335468..57afe105209c5 100644 --- a/airflow/providers/cloudant/provider.yaml +++ b/airflow/providers/cloudant/provider.yaml @@ -22,9 +22,10 @@ description: | `IBM Cloudant `__ state: ready -source-date-epoch: 1723969866 +source-date-epoch: 1728484755 # note that those versions are maintained by release manager - do not update them manually versions: + - 4.0.1 - 4.0.0 - 3.6.0 - 3.5.2 diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index 3a8101943eaf3..98c179995fb3f 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -27,11 +27,33 @@ Changelog --------- -main +9.0.0 ..... +Breaking changes +~~~~~~~~~~~~~~~~ + .. warning:: - Support for identifying pods by execution_date during the upgrade from Airflow 1 to 2 has been removed. This may result in duplicate pods being launched for tasks originally started by Airflow 1, but only one of the task pods will succeed. + Support for identifying pods by execution_date during the upgrade from Airflow 1 to 2 has been removed. + This may result in duplicate pods being launched for tasks originally started by Airflow 1, but only one of the task pods will succeed. + +* ``kubernetes executor cleanup_stuck_queued_tasks optimization (#41220)`` + +Bug Fixes +~~~~~~~~~ + +* ``All executors should inherit from BaseExecutor (#41904)`` +* ``Fix mark as success when pod fails while fetching log (#42815)`` +* ``Fix SparkKubernetesOperator spark name. (#42427)`` +* ``KubernetesPodOperator never stops if credentials are refreshed (#42361)`` +* ``Restructured 'await_xcom_sidecar_container_start' method. (#42504)`` +* ``KubernetesHook kube_config extra can take dict (#41413)`` + +Misc +~~~~ + +* ``Drop python3.8 support core and providers (#42766)`` +* ``Remove airflow_version from k8s executor pod selector (#42751)`` 8.4.2 ..... diff --git a/airflow/providers/cncf/kubernetes/__init__.py b/airflow/providers/cncf/kubernetes/__init__.py index 5f4798d1d98dc..34c63b7fcf87f 100644 --- a/airflow/providers/cncf/kubernetes/__init__.py +++ b/airflow/providers/cncf/kubernetes/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "8.4.2" +__version__ = "9.0.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/common/compat/CHANGELOG.rst b/airflow/providers/common/compat/CHANGELOG.rst index 27770f785cc7b..49dfa7b41149b 100644 --- a/airflow/providers/common/compat/CHANGELOG.rst +++ b/airflow/providers/common/compat/CHANGELOG.rst @@ -25,6 +25,18 @@ Changelog --------- +1.2.1 +..... + +Misc +~~~~ + +* ``Rename dataset related python variable names to asset (#41348)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.2.0 ..... diff --git a/airflow/providers/common/compat/__init__.py b/airflow/providers/common/compat/__init__.py index ffe1acbcce74f..ef51cb422e513 100644 --- a/airflow/providers/common/compat/__init__.py +++ b/airflow/providers/common/compat/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.2.0" +__version__ = "1.2.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/common/compat/provider.yaml b/airflow/providers/common/compat/provider.yaml index 7ac3a02900924..3618ecfe6435d 100644 --- a/airflow/providers/common/compat/provider.yaml +++ b/airflow/providers/common/compat/provider.yaml @@ -22,9 +22,10 @@ description: | ``Common Compatibility Provider - providing compatibility code for previous Airflow versions.`` state: ready -source-date-epoch: 1723970013 +source-date-epoch: 1728484960 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.2.1 - 1.2.0 - 1.1.0 - 1.0.0 diff --git a/airflow/providers/common/io/CHANGELOG.rst b/airflow/providers/common/io/CHANGELOG.rst index 6b787fe9e4b30..a80b855efd268 100644 --- a/airflow/providers/common/io/CHANGELOG.rst +++ b/airflow/providers/common/io/CHANGELOG.rst @@ -25,6 +25,19 @@ Changelog --------- +1.4.2 +..... + +Misc +~~~~ + +* ``Drop python3.8 support core and providers (#42766)`` +* ``Rename dataset related python variable names to asset (#41348)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.4.1 ..... diff --git a/airflow/providers/common/io/__init__.py b/airflow/providers/common/io/__init__.py index ac5d2626cfc49..ac05e1f448dc7 100644 --- a/airflow/providers/common/io/__init__.py +++ b/airflow/providers/common/io/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.4.1" +__version__ = "1.4.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/common/io/provider.yaml b/airflow/providers/common/io/provider.yaml index 6743cfff86c40..c32f046eb929a 100644 --- a/airflow/providers/common/io/provider.yaml +++ b/airflow/providers/common/io/provider.yaml @@ -22,9 +22,10 @@ description: | ``Common IO Provider`` state: ready -source-date-epoch: 1726860435 +source-date-epoch: 1728484966 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.4.2 - 1.4.1 - 1.4.0 - 1.3.2 diff --git a/airflow/providers/common/sql/CHANGELOG.rst b/airflow/providers/common/sql/CHANGELOG.rst index 531353f8c800f..50338885101b4 100644 --- a/airflow/providers/common/sql/CHANGELOG.rst +++ b/airflow/providers/common/sql/CHANGELOG.rst @@ -25,6 +25,23 @@ Changelog --------- +1.18.0 +...... + +Features +~~~~~~~~ + +* ``feat(providers/common/sql): add warning to connection setter (#42736)`` + +Bug Fixes +~~~~~~~~~ + +* ``FIX: Only pass connection to sqlalchemy engine in JdbcHook (#42705)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.17.1 ...... diff --git a/airflow/providers/common/sql/__init__.py b/airflow/providers/common/sql/__init__.py index 6ef37aa0ed669..53b2468c53268 100644 --- a/airflow/providers/common/sql/__init__.py +++ b/airflow/providers/common/sql/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.17.1" +__version__ = "1.18.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index ec487aca3f001..3b520e9454a4f 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -22,9 +22,10 @@ description: | `Common SQL Provider `__ state: ready -source-date-epoch: 1727372263 +source-date-epoch: 1728485007 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.18.0 - 1.17.1 - 1.17.0 - 1.16.0 diff --git a/airflow/providers/databricks/CHANGELOG.rst b/airflow/providers/databricks/CHANGELOG.rst index 487d7dc50f4ff..28fec73032341 100644 --- a/airflow/providers/databricks/CHANGELOG.rst +++ b/airflow/providers/databricks/CHANGELOG.rst @@ -26,6 +26,24 @@ Changelog --------- +6.11.0 +...... + +Features +~~~~~~~~ + +* ``Add 'on_kill' to Databricks Workflow Operator (#42115)`` + +Misc +~~~~ + +* ``add warning log when task_key>100 (#42813)`` +* ``Add debug logs to print Request/Response data in Databricks provider (#42662)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 6.10.0 ...... diff --git a/airflow/providers/databricks/__init__.py b/airflow/providers/databricks/__init__.py index 7124228f796a1..d81dd8d4c5a22 100644 --- a/airflow/providers/databricks/__init__.py +++ b/airflow/providers/databricks/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "6.10.0" +__version__ = "6.11.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index db60aa187ea00..35d9a6aabcead 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -22,9 +22,10 @@ description: | `Databricks `__ state: ready -source-date-epoch: 1726860611 +source-date-epoch: 1728485027 # note that those versions are maintained by release manager - do not update them manually versions: + - 6.11.0 - 6.10.0 - 6.9.0 - 6.8.0 diff --git a/airflow/providers/dbt/cloud/CHANGELOG.rst b/airflow/providers/dbt/cloud/CHANGELOG.rst index db0e93fb8fd40..7eca133d4957d 100644 --- a/airflow/providers/dbt/cloud/CHANGELOG.rst +++ b/airflow/providers/dbt/cloud/CHANGELOG.rst @@ -28,6 +28,23 @@ Changelog --------- +3.11.0 +...... + +Features +~~~~~~~~ + +* ``Add ability to provide proxy for dbt Cloud connection (#42737)`` + +Misc +~~~~ + +* ``Simplify code for recent dbt provider change (#42840)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.10.1 ...... diff --git a/airflow/providers/dbt/cloud/__init__.py b/airflow/providers/dbt/cloud/__init__.py index f532436b649ce..a72beeb7aedc6 100644 --- a/airflow/providers/dbt/cloud/__init__.py +++ b/airflow/providers/dbt/cloud/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.10.1" +__version__ = "3.11.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/dbt/cloud/provider.yaml b/airflow/providers/dbt/cloud/provider.yaml index 2c80397a1506d..828aecf335435 100644 --- a/airflow/providers/dbt/cloud/provider.yaml +++ b/airflow/providers/dbt/cloud/provider.yaml @@ -22,9 +22,10 @@ description: | `dbt Cloud `__ state: ready -source-date-epoch: 1726860654 +source-date-epoch: 1728485039 # note that those versions are maintained by release manager - do not update them manually versions: + - 3.11.0 - 3.10.1 - 3.10.0 - 3.9.0 diff --git a/airflow/providers/elasticsearch/CHANGELOG.rst b/airflow/providers/elasticsearch/CHANGELOG.rst index 2e51c8572b227..46dafdd4cd0bb 100644 --- a/airflow/providers/elasticsearch/CHANGELOG.rst +++ b/airflow/providers/elasticsearch/CHANGELOG.rst @@ -27,6 +27,19 @@ Changelog --------- +5.5.2 +..... + +Misc +~~~~ + +* ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Fix spelling; 'Airlfow' -> 'Airflow' (#42855)`` + 5.5.1 ..... diff --git a/airflow/providers/elasticsearch/__init__.py b/airflow/providers/elasticsearch/__init__.py index 96eab6484f84a..3d480fc501db7 100644 --- a/airflow/providers/elasticsearch/__init__.py +++ b/airflow/providers/elasticsearch/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "5.5.1" +__version__ = "5.5.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index 3c90c6763d154..595a3d45be5cd 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -22,9 +22,10 @@ description: | `Elasticsearch `__ state: ready -source-date-epoch: 1726860725 +source-date-epoch: 1728485068 # note that those versions are maintained by release manager - do not update them manually versions: + - 5.5.2 - 5.5.1 - 5.5.0 - 5.4.2 diff --git a/airflow/providers/fab/CHANGELOG.rst b/airflow/providers/fab/CHANGELOG.rst index 1f1d09fea932d..4419184cc3c8b 100644 --- a/airflow/providers/fab/CHANGELOG.rst +++ b/airflow/providers/fab/CHANGELOG.rst @@ -20,6 +20,20 @@ Changelog --------- +1.4.1 +..... + +Misc +~~~~ + +* ``Update Rest API tests to no longer rely on FAB auth manager. Move tests specific to FAB permissions to FAB provider (#42523)`` +* ``Rename dataset related python variable names to asset (#41348)`` +* ``Simplify expression for get_permitted_dag_ids query (#42484)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.4.0 ..... diff --git a/airflow/providers/fab/__init__.py b/airflow/providers/fab/__init__.py index 263ca95e58a5c..7403b3712d2ff 100644 --- a/airflow/providers/fab/__init__.py +++ b/airflow/providers/fab/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.4.0" +__version__ = "1.4.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.9.0" diff --git a/airflow/providers/fab/provider.yaml b/airflow/providers/fab/provider.yaml index 1d5cc820f1f24..c5e6f76cccf26 100644 --- a/airflow/providers/fab/provider.yaml +++ b/airflow/providers/fab/provider.yaml @@ -28,10 +28,11 @@ description: | # For providers until we think it should be released. state: ready -source-date-epoch: 1726860772 +source-date-epoch: 1728485088 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.4.1 - 1.4.0 - 1.3.0 - 1.2.2 diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 7d0ae7e7985ca..e4d9a15099341 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -27,6 +27,36 @@ Changelog --------- +10.24.0 +....... + +Features +~~~~~~~~ + +* ``Add 'retry_if_resource_not_ready' logic for DataprocCreateClusterOperator and DataprocCreateBatchOperator (#42703)`` + +Bug Fixes +~~~~~~~~~ + +* ``Publish Dataproc Serverless Batch link after it starts if batch_id was provided (#41153)`` +* ``fix PubSubAsyncHook in PubsubPullTrigger to use gcp_conn_id (#42671)`` +* ``Fix consistent return response from PubSubPullSensor (#42080)`` +* ``Undo partition exclusion from the table name when splitting a full BigQuery table name (#42541)`` +* ``Fix gcp text to speech uri fetch (#42309)`` +* ``Refactor ''bucket.get_blob'' calls in ''GCSHook'' to handle validation for non-existent objects. (#42474)`` +* ``Bugfix/dataflow job location passing (#41887)`` + +Misc +~~~~ + +* ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` +* ``Rename dataset related python variable names to asset (#41348)`` +* ``Deprecate AutoMLBatchPredictOperator and refactor AutoMl system tests (#42260)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 10.23.0 ....... diff --git a/airflow/providers/google/__init__.py b/airflow/providers/google/__init__.py index ffbf74f2df63f..e5caa3ef42b10 100644 --- a/airflow/providers/google/__init__.py +++ b/airflow/providers/google/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "10.23.0" +__version__ = "10.24.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index a64b2ce17a76e..9e469d278775b 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -29,9 +29,10 @@ description: | - `Google Workspace `__ (formerly Google Suite) state: ready -source-date-epoch: 1726860901 +source-date-epoch: 1728485162 # note that those versions are maintained by release manager - do not update them manually versions: + - 10.24.0 - 10.23.0 - 10.22.0 - 10.21.1 diff --git a/airflow/providers/jdbc/CHANGELOG.rst b/airflow/providers/jdbc/CHANGELOG.rst index 872ddeea5963f..22894e7b73fb3 100644 --- a/airflow/providers/jdbc/CHANGELOG.rst +++ b/airflow/providers/jdbc/CHANGELOG.rst @@ -26,6 +26,18 @@ Changelog --------- +4.5.2 +..... + +Bug Fixes +~~~~~~~~~ + +* ``FIX: Only pass connection to sqlalchemy engine in JdbcHook (#42705)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.5.1 ..... diff --git a/airflow/providers/jdbc/__init__.py b/airflow/providers/jdbc/__init__.py index 2bda2a955fea3..dde5dab0cc937 100644 --- a/airflow/providers/jdbc/__init__.py +++ b/airflow/providers/jdbc/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.5.1" +__version__ = "4.5.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index c247b079f7731..5165d33939617 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -22,9 +22,10 @@ description: | `Java Database Connectivity (JDBC) `__ state: ready -source-date-epoch: 1726860958 +source-date-epoch: 1728485176 # note that those versions are maintained by release manager - do not update them manually versions: + - 4.5.2 - 4.5.1 - 4.5.0 - 4.4.0 diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index be33a4e95d3fa..148fce2e1152f 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -27,6 +27,24 @@ Changelog --------- +10.5.1 +...... + +Bug Fixes +~~~~~~~~~ + +* ``(bugfix): Paginated results in MSGraphAsyncOperator (#42414)`` + +Misc +~~~~ + +* ``Workaround pin azure kusto data (#42576)`` +* ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 10.5.0 ...... diff --git a/airflow/providers/microsoft/azure/__init__.py b/airflow/providers/microsoft/azure/__init__.py index bd769cef5e33d..44aafcaa1730a 100644 --- a/airflow/providers/microsoft/azure/__init__.py +++ b/airflow/providers/microsoft/azure/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "10.5.0" +__version__ = "10.5.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 45fe28eecffc7..a47ec86de9f78 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -20,9 +20,10 @@ name: Microsoft Azure description: | `Microsoft Azure `__ state: ready -source-date-epoch: 1726861002 +source-date-epoch: 1728485226 # note that those versions are maintained by release manager - do not update them manually versions: + - 10.5.1 - 10.5.0 - 10.4.0 - 10.3.0 diff --git a/airflow/providers/mysql/CHANGELOG.rst b/airflow/providers/mysql/CHANGELOG.rst index 7acdd410135b2..3426e0cd10883 100644 --- a/airflow/providers/mysql/CHANGELOG.rst +++ b/airflow/providers/mysql/CHANGELOG.rst @@ -26,6 +26,18 @@ Changelog --------- +5.7.2 +..... + +Misc +~~~~ + +* ``Rename dataset related python variable names to asset (#41348)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 5.7.1 ..... diff --git a/airflow/providers/mysql/__init__.py b/airflow/providers/mysql/__init__.py index 586de40e0c0df..4c897ee113178 100644 --- a/airflow/providers/mysql/__init__.py +++ b/airflow/providers/mysql/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "5.7.1" +__version__ = "5.7.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index f0f77f28d0e94..82e5b8cda2c3b 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -22,9 +22,10 @@ description: | `MySQL `__ state: ready -source-date-epoch: 1726861041 +source-date-epoch: 1728485234 # note that those versions are maintained by release manager - do not update them manually versions: + - 5.7.2 - 5.7.1 - 5.7.0 - 5.6.3 diff --git a/airflow/providers/openlineage/CHANGELOG.rst b/airflow/providers/openlineage/CHANGELOG.rst index 0e35dab6deaa2..05c20a241e624 100644 --- a/airflow/providers/openlineage/CHANGELOG.rst +++ b/airflow/providers/openlineage/CHANGELOG.rst @@ -26,6 +26,20 @@ Changelog --------- +1.12.2 +...... + +Misc +~~~~ + +* ``Change imports to use Standard provider for BashOperator (#42252)`` +* ``Drop python3.8 support core and providers (#42766)`` +* ``Rename dataset related python variable names to asset (#41348)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.12.1 ...... diff --git a/airflow/providers/openlineage/__init__.py b/airflow/providers/openlineage/__init__.py index 664e5530ebf97..676fae32d2594 100644 --- a/airflow/providers/openlineage/__init__.py +++ b/airflow/providers/openlineage/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.12.1" +__version__ = "1.12.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index 5b08ac6a77ae7..fadaf9c1e07a4 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -22,9 +22,10 @@ description: | `OpenLineage `__ state: ready -source-date-epoch: 1727372276 +source-date-epoch: 1728485291 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.12.2 - 1.12.1 - 1.12.0 - 1.11.0 diff --git a/airflow/providers/opensearch/CHANGELOG.rst b/airflow/providers/opensearch/CHANGELOG.rst index 4bc3235a79cf8..964a44f213d43 100644 --- a/airflow/providers/opensearch/CHANGELOG.rst +++ b/airflow/providers/opensearch/CHANGELOG.rst @@ -27,6 +27,28 @@ Changelog --------- +1.5.0 +..... + +Features +~~~~~~~~ + +* ``(feat): Add opensearch logging integration (#41799)`` + +Bug Fixes +~~~~~~~~~ + +* ``Handle empty login and password with opensearch client (#39982)`` + +Misc +~~~~ + +* ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.4.0 ..... diff --git a/airflow/providers/opensearch/__init__.py b/airflow/providers/opensearch/__init__.py index b98bfd8b5a51a..457d4fdeaaf70 100644 --- a/airflow/providers/opensearch/__init__.py +++ b/airflow/providers/opensearch/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.4.0" +__version__ = "1.5.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/opensearch/provider.yaml b/airflow/providers/opensearch/provider.yaml index 9cfea50540911..e5b3027ebef67 100644 --- a/airflow/providers/opensearch/provider.yaml +++ b/airflow/providers/opensearch/provider.yaml @@ -22,9 +22,10 @@ description: | `OpenSearch `__ state: ready -source-date-epoch: 1723970482 +source-date-epoch: 1728485323 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.5.0 - 1.4.0 - 1.3.0 - 1.2.1 diff --git a/airflow/providers/postgres/CHANGELOG.rst b/airflow/providers/postgres/CHANGELOG.rst index 8c5b65d3e32ed..b1a42da50d395 100644 --- a/airflow/providers/postgres/CHANGELOG.rst +++ b/airflow/providers/postgres/CHANGELOG.rst @@ -27,6 +27,18 @@ Changelog --------- +5.13.1 +...... + +Misc +~~~~ + +* ``Rename dataset related python variable names to asset (#41348)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 5.13.0 ...... diff --git a/airflow/providers/postgres/__init__.py b/airflow/providers/postgres/__init__.py index 5d86bd461a93a..fe91d927f8fd7 100644 --- a/airflow/providers/postgres/__init__.py +++ b/airflow/providers/postgres/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "5.13.0" +__version__ = "5.13.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index edbbaeb1da2c1..51332b9b0feaa 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -22,9 +22,10 @@ description: | `PostgreSQL `__ state: ready -source-date-epoch: 1726861120 +source-date-epoch: 1728485329 # note that those versions are maintained by release manager - do not update them manually versions: + - 5.13.1 - 5.13.0 - 5.12.0 - 5.11.3 diff --git a/airflow/providers/snowflake/CHANGELOG.rst b/airflow/providers/snowflake/CHANGELOG.rst index 1d5712b5d5be2..06554edf213e7 100644 --- a/airflow/providers/snowflake/CHANGELOG.rst +++ b/airflow/providers/snowflake/CHANGELOG.rst @@ -27,6 +27,24 @@ Changelog --------- +5.8.0 +..... + +Features +~~~~~~~~ + +* ``Add Snowpark operator and decorator (#42457)`` + +Bug Fixes +~~~~~~~~~ + +* ``fix: SnowflakeSqlApiOperator not resolving parameters in SQL (#42719)`` +* ``Make 'private_key_content' a sensitive field in Snowflake connection (#42649)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 5.7.1 ..... diff --git a/airflow/providers/snowflake/__init__.py b/airflow/providers/snowflake/__init__.py index 607ebb12fbff5..9a4e264521c73 100644 --- a/airflow/providers/snowflake/__init__.py +++ b/airflow/providers/snowflake/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "5.7.1" +__version__ = "5.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index 47de902ff65da..08a2ce861d477 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -22,9 +22,10 @@ description: | `Snowflake `__ state: ready -source-date-epoch: 1726861185 +source-date-epoch: 1728485406 # note that those versions are maintained by release manager - do not update them manually versions: + - 5.8.0 - 5.7.1 - 5.7.0 - 5.6.1 diff --git a/airflow/providers/trino/CHANGELOG.rst b/airflow/providers/trino/CHANGELOG.rst index a0f32bc76a3c5..09c2984477541 100644 --- a/airflow/providers/trino/CHANGELOG.rst +++ b/airflow/providers/trino/CHANGELOG.rst @@ -27,6 +27,18 @@ Changelog --------- +5.8.1 +..... + +Misc +~~~~ + +* ``Rename dataset related python variable names to asset (#41348)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 5.8.0 ..... diff --git a/airflow/providers/trino/__init__.py b/airflow/providers/trino/__init__.py index e729510404c11..a675398f4f562 100644 --- a/airflow/providers/trino/__init__.py +++ b/airflow/providers/trino/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "5.8.0" +__version__ = "5.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index 424be2cca67d9..876244aa1c6a8 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -22,9 +22,10 @@ description: | `Trino `__ state: ready -source-date-epoch: 1723970622 +source-date-epoch: 1728485413 # note that those versions are maintained by release manager - do not update them manually versions: + - 5.8.1 - 5.8.0 - 5.7.2 - 5.7.1 diff --git a/airflow/providers/ydb/CHANGELOG.rst b/airflow/providers/ydb/CHANGELOG.rst index a6ee9aa6b104e..afb2845e15175 100644 --- a/airflow/providers/ydb/CHANGELOG.rst +++ b/airflow/providers/ydb/CHANGELOG.rst @@ -27,6 +27,18 @@ Changelog --------- +1.4.0 +..... + +Features +~~~~~~~~ + +* ``Add an ability to use scan queries via new YDB operator (#42311)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.3.0 ..... diff --git a/airflow/providers/ydb/__init__.py b/airflow/providers/ydb/__init__.py index b7b03b98626b7..193a62740ada4 100644 --- a/airflow/providers/ydb/__init__.py +++ b/airflow/providers/ydb/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.3.0" +__version__ = "1.4.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.8.0" diff --git a/airflow/providers/ydb/provider.yaml b/airflow/providers/ydb/provider.yaml index a956100dc3f67..18f7140e1aa1c 100644 --- a/airflow/providers/ydb/provider.yaml +++ b/airflow/providers/ydb/provider.yaml @@ -22,9 +22,10 @@ description: | `YDB `__ state: ready -source-date-epoch: 1723970719 +source-date-epoch: 1728485455 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.4.0 - 1.3.0 - 1.2.0 - 1.1.0 diff --git a/docs/apache-airflow-providers-amazon/commits.rst b/docs/apache-airflow-providers-amazon/commits.rst index 309c2b5989fcd..07aa8f84e0aee 100644 --- a/docs/apache-airflow-providers-amazon/commits.rst +++ b/docs/apache-airflow-providers-amazon/commits.rst @@ -35,14 +35,40 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-09 ``Revert "Remove 'sqlalchemy-redshift' dependency from Amazon provider (#42830)" (#42864)`` +`3b4c73a644 `_ 2024-10-08 ``Remove 'sqlalchemy-redshift' dependency from Amazon provider (#42830)`` +`63ff22f403 `_ 2024-10-08 ``Drop python3.8 support core and providers (#42766)`` +`1cb9294c64 `_ 2024-10-06 ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` +`ede7cb27fd `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +`568162263d `_ 2024-09-26 ``Remove deprecated stuff from Amazon provider package (#42450)`` +`ac0711f0f1 `_ 2024-09-26 ``'S3DeleteObjects' Operator: Handle dates passed as strings (#42464)`` +`f6852c2c55 `_ 2024-09-25 ``Remove identity center auth manager cli (#42481)`` +`663da777f9 `_ 2024-09-25 ``Small fix to AWS AVP cli init script (#42479)`` +`18c54bcb42 `_ 2024-09-25 ``#42442 Make the AWS logging faster by reducing the amount of sleep (#42449)`` +`d87f9b0505 `_ 2024-09-25 ``Refactor AWS Auth manager user output (#42454)`` +`ab3429c318 `_ 2024-09-24 ``Add STOPPED to the failure cases for Sagemaker Training Jobs (#42423)`` +`8580e6d046 `_ 2024-09-24 ``Support session reuse in 'RedshiftDataOperator' (#42218)`` +`4c8c72f4a4 `_ 2024-09-24 ``Fix logout in AWS auth manager (#42447)`` +`b9629d99b2 `_ 2024-09-24 ``fix(providers/amazon): handle ClientError raised after key is missing during table.get_item (#42408)`` +================================================================================================= =========== ======================================================================================================== + 8.29.0 ...... -Latest change: 2024-09-16 +Latest change: 2024-09-21 ================================================================================================= =========== ================================================================================================= Commit Committed Subject ================================================================================================= =========== ================================================================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `4afc2569b1 `_ 2024-09-16 ``ECSExecutor: Drop params that aren't compatible with EC2 (#42228)`` `5fb0d65aaf `_ 2024-09-11 ``Adding support for volume configurations in ECSRunTaskOperator (#42087)`` `1bde32ab7b `_ 2024-09-11 ``Actually move saml to amazon provider (mistakenly added in papermill) (#42148)`` diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index c58847dcd8876..6d354b1a5f894 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -86,7 +86,7 @@ apache-airflow-providers-amazon package Amazon integration (including `Amazon Web Services (AWS) `__). -Release: 8.29.0 +Release: 9.0.0 Provider package ---------------- @@ -164,5 +164,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-amazon 8.29.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-amazon 8.29.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 9.0.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 9.0.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-apache-beam/commits.rst b/docs/apache-airflow-providers-apache-beam/commits.rst index 0173d7bbf59fa..961ece4142294 100644 --- a/docs/apache-airflow-providers-apache-beam/commits.rst +++ b/docs/apache-airflow-providers-apache-beam/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-09-24 ``Bugfix/dataflow job location passing (#41887)`` +================================================================================================= =========== ================================================= + 5.8.0 ..... -Latest change: 2024-08-12 +Latest change: 2024-08-19 ================================================================================================= =========== ======================================================================= Commit Committed Subject ================================================================================================= =========== ======================================================================= +`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` `fcbff15bda `_ 2024-08-12 ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` ================================================================================================= =========== ======================================================================= diff --git a/docs/apache-airflow-providers-apache-beam/index.rst b/docs/apache-airflow-providers-apache-beam/index.rst index 0934dc3cae5d7..a8b7396b6706e 100644 --- a/docs/apache-airflow-providers-apache-beam/index.rst +++ b/docs/apache-airflow-providers-apache-beam/index.rst @@ -75,7 +75,7 @@ apache-airflow-providers-apache-beam package `Apache Beam `__. -Release: 5.8.0 +Release: 5.8.1 Provider package ---------------- @@ -128,5 +128,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-beam 5.8.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-beam 5.8.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-beam 5.8.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-beam 5.8.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-apache-kafka/commits.rst b/docs/apache-airflow-providers-apache-kafka/commits.rst index 8aee39b02018d..7fba0473facb4 100644 --- a/docs/apache-airflow-providers-apache-kafka/commits.rst +++ b/docs/apache-airflow-providers-apache-kafka/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-09-29 ``remove callable functions parameter from kafka operator template_fields (#42555)`` +================================================================================================= =========== ==================================================================================== + 1.6.0 ..... -Latest change: 2024-08-12 +Latest change: 2024-08-19 ================================================================================================= =========== ======================================================================= Commit Committed Subject ================================================================================================= =========== ======================================================================= +`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` `fcbff15bda `_ 2024-08-12 ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` `d23881c648 `_ 2024-08-03 ``Prepare docs for Aug 1st wave of providers (#41230)`` `09a7bd1d58 `_ 2024-07-09 ``Prepare docs 1st wave July 2024 (#40644)`` diff --git a/docs/apache-airflow-providers-apache-kafka/index.rst b/docs/apache-airflow-providers-apache-kafka/index.rst index f735b162f4939..0879b57fb7f1b 100644 --- a/docs/apache-airflow-providers-apache-kafka/index.rst +++ b/docs/apache-airflow-providers-apache-kafka/index.rst @@ -82,7 +82,7 @@ apache-airflow-providers-apache-kafka package `Apache Kafka `__ -Release: 1.6.0 +Release: 1.6.1 Provider package ---------------- diff --git a/docs/apache-airflow-providers-apache-spark/commits.rst b/docs/apache-airflow-providers-apache-spark/commits.rst index 674f50007190f..45a324a63262f 100644 --- a/docs/apache-airflow-providers-apache-spark/commits.rst +++ b/docs/apache-airflow-providers-apache-spark/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-07 ``The function resolve_kerberos_principal updated when airflow version 2.8 and above (#42777)`` +================================================================================================= =========== =============================================================================================== + 4.11.0 ...... -Latest change: 2024-09-17 +Latest change: 2024-09-21 ================================================================================================= =========== ========================================================================================= Commit Committed Subject ================================================================================================= =========== ========================================================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `1f10532163 `_ 2024-09-17 ``Add kerberos related connection fields(principal, keytab) on SparkSubmitHook (#40757)`` ================================================================================================= =========== ========================================================================================= diff --git a/docs/apache-airflow-providers-apache-spark/index.rst b/docs/apache-airflow-providers-apache-spark/index.rst index db6da3392787e..6e36428a46291 100644 --- a/docs/apache-airflow-providers-apache-spark/index.rst +++ b/docs/apache-airflow-providers-apache-spark/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-apache-spark package `Apache Spark `__ -Release: 4.11.0 +Release: 4.11.1 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-spark 4.11.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-spark 4.11.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-spark 4.11.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-spark 4.11.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-celery/commits.rst b/docs/apache-airflow-providers-celery/commits.rst index 115438b7e7e45..302e845f4d138 100644 --- a/docs/apache-airflow-providers-celery/commits.rst +++ b/docs/apache-airflow-providers-celery/commits.rst @@ -35,16 +35,30 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-09 ``Standard provider bash operator (#42252)`` +`c72dad7eaf `_ 2024-10-01 ``All executors should inherit from BaseExecutor (#41904)`` +`f463e2a0dd `_ 2024-10-01 ``Remove state sync during celery task processing (#41870)`` +================================================================================================= =========== ============================================================ + 3.8.2 ..... -Latest change: 2024-08-25 +Latest change: 2024-09-21 -================================================================================================= =========== ============================= +================================================================================================= =========== ======================================================= Commit Committed Subject -================================================================================================= =========== ============================= +================================================================================================= =========== ======================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `1613e9ec1c `_ 2024-08-25 ``remove soft_fail (#41710)`` -================================================================================================= =========== ============================= +================================================================================================= =========== ======================================================= 3.8.1 ..... diff --git a/docs/apache-airflow-providers-celery/index.rst b/docs/apache-airflow-providers-celery/index.rst index 85117b0d1d656..151a35d71531a 100644 --- a/docs/apache-airflow-providers-celery/index.rst +++ b/docs/apache-airflow-providers-celery/index.rst @@ -67,7 +67,7 @@ apache-airflow-providers-celery package `Celery `__ -Release: 3.8.2 +Release: 3.8.3 Provider package ---------------- @@ -113,6 +113,7 @@ You can install such cross-provider dependencies when installing from PyPI. For Dependent package Extra ====================================================================================================================== =================== `apache-airflow-providers-cncf-kubernetes `_ ``cncf.kubernetes`` +`apache-airflow-providers-standard `_ ``standard`` ====================================================================================================================== =================== Downloading official packages @@ -121,5 +122,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-celery 3.8.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-celery 3.8.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-celery 3.8.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-celery 3.8.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-cloudant/commits.rst b/docs/apache-airflow-providers-cloudant/commits.rst index 5e36a53ba7cc8..bc4a00bc4e880 100644 --- a/docs/apache-airflow-providers-cloudant/commits.rst +++ b/docs/apache-airflow-providers-cloudant/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-08 ``Drop python3.8 support core and providers (#42766)`` +================================================================================================= =========== ====================================================== + 4.0.0 ..... -Latest change: 2024-09-06 +Latest change: 2024-09-21 ================================================================================================= =========== ================================================================================== Commit Committed Subject ================================================================================================= =========== ================================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `caa94fa466 `_ 2024-09-06 ``Switch cloudant provider from cloudant library to ibmcloudant library (#41555)`` ================================================================================================= =========== ================================================================================== diff --git a/docs/apache-airflow-providers-cloudant/index.rst b/docs/apache-airflow-providers-cloudant/index.rst index 28d76f291d61d..2de0afb0a7f70 100644 --- a/docs/apache-airflow-providers-cloudant/index.rst +++ b/docs/apache-airflow-providers-cloudant/index.rst @@ -55,7 +55,7 @@ apache-airflow-providers-cloudant package `IBM Cloudant `__ -Release: 4.0.0 +Release: 4.0.1 Provider package ---------------- diff --git a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst index 56bd24e26a786..99dcb3a74a032 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst @@ -35,14 +35,34 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-09 ``Fix mark as success when pod fails while fetching log (#42815)`` +`63ff22f403 `_ 2024-10-08 ``Drop python3.8 support core and providers (#42766)`` +`e5a474bdd1 `_ 2024-10-07 ``kubernetes executor cleanup_stuck_queued_tasks optimization (#41220)`` +`24b7c5b151 `_ 2024-10-04 ``Remove airflow_version from k8s executor pod selector (#42751)`` +`0120515f6c `_ 2024-10-03 ``Added unit tests and restructred 'await_xcom_sidecar_container_start' method. (#42504)`` +`c72dad7eaf `_ 2024-10-01 ``All executors should inherit from BaseExecutor (#41904)`` +`6808619ec5 `_ 2024-09-30 ``KubernetesHook kube_config extra can take dict (#41413)`` +`223acdb13f `_ 2024-09-27 ``Fix SparkKubernetesOperator spark name. (#42427)`` +`7782050201 `_ 2024-09-27 ``KubernetesPodOperator never stops if credentials are refreshed (#42361)`` +================================================================================================= =========== ========================================================================================== + 8.4.2 ..... -Latest change: 2024-09-16 +Latest change: 2024-09-21 ================================================================================================= =========== ================================================================ Commit Committed Subject ================================================================================================= =========== ================================================================ +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `d1e500c450 `_ 2024-09-16 ``Deprecated configuration removed (#42129)`` `1379376b66 `_ 2024-09-02 ``Add TODOs in providers code for Subdag code removal (#41963)`` `86af316c87 `_ 2024-09-02 ``docstring update (#41929)`` diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/docs/apache-airflow-providers-cncf-kubernetes/index.rst index fbf78d5bd3538..1db81c09a3f6d 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst @@ -87,7 +87,7 @@ apache-airflow-providers-cncf-kubernetes package `Kubernetes `__ -Release: 8.4.2 +Release: 9.0.0 Provider package ---------------- diff --git a/docs/apache-airflow-providers-common-compat/commits.rst b/docs/apache-airflow-providers-common-compat/commits.rst index 5349314f57e64..d56bddfc128c3 100644 --- a/docs/apache-airflow-providers-common-compat/commits.rst +++ b/docs/apache-airflow-providers-common-compat/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +================================================================================================= =========== ================================================================== + 1.2.0 ..... -Latest change: 2024-08-12 +Latest change: 2024-08-19 ================================================================================================= =========== ======================================================================= Commit Committed Subject ================================================================================================= =========== ======================================================================= +`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` `fcbff15bda `_ 2024-08-12 ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` ================================================================================================= =========== ======================================================================= diff --git a/docs/apache-airflow-providers-common-compat/index.rst b/docs/apache-airflow-providers-common-compat/index.rst index c03a05947bbc0..b5eadbddb0905 100644 --- a/docs/apache-airflow-providers-common-compat/index.rst +++ b/docs/apache-airflow-providers-common-compat/index.rst @@ -62,7 +62,7 @@ apache-airflow-providers-common-compat package ``Common Compatibility Provider - providing compatibility code for previous Airflow versions.`` -Release: 1.2.0 +Release: 1.2.1 Provider package ---------------- @@ -87,3 +87,31 @@ PIP package Version required ================== ================== ``apache-airflow`` ``>=2.8.0`` ================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider packages in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-common-compat[openlineage] + + +============================================================================================================== =============== +Dependent package Extra +============================================================================================================== =============== +`apache-airflow-providers-openlineage `_ ``openlineage`` +============================================================================================================== =============== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-common-compat 1.2.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-compat 1.2.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-common-io/commits.rst b/docs/apache-airflow-providers-common-io/commits.rst index e2d5fef0daeaa..29a3814ae384b 100644 --- a/docs/apache-airflow-providers-common-io/commits.rst +++ b/docs/apache-airflow-providers-common-io/commits.rst @@ -35,14 +35,27 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-08 ``Drop python3.8 support core and providers (#42766)`` +`ede7cb27fd `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +================================================================================================= =========== ================================================================== + 1.4.1 ..... -Latest change: 2024-09-01 +Latest change: 2024-09-21 ================================================================================================= =========== ============================================================================== Commit Committed Subject ================================================================================================= =========== ============================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `7a75f0a2bc `_ 2024-09-01 ``Protect against None components of universal pathlib xcom backend (#41921)`` ================================================================================================= =========== ============================================================================== diff --git a/docs/apache-airflow-providers-common-io/index.rst b/docs/apache-airflow-providers-common-io/index.rst index 9a3468d397d8d..606b6c5161bcd 100644 --- a/docs/apache-airflow-providers-common-io/index.rst +++ b/docs/apache-airflow-providers-common-io/index.rst @@ -79,7 +79,7 @@ apache-airflow-providers-common-io package ``Common IO Provider`` -Release: 1.4.1 +Release: 1.4.2 Provider package ---------------- @@ -131,5 +131,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-common-io 1.4.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-common-io 1.4.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-io 1.4.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-io 1.4.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-common-sql/commits.rst b/docs/apache-airflow-providers-common-sql/commits.rst index f719dd7b39811..a84e298074482 100644 --- a/docs/apache-airflow-providers-common-sql/commits.rst +++ b/docs/apache-airflow-providers-common-sql/commits.rst @@ -35,14 +35,27 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-07 ``FIX: Only pass connection to sqlalchemy engine in JdbcHook (#42705)`` +`5973c9cce1 `_ 2024-10-04 ``feat(providers/common/sql): add warning to connection setter (#42736)`` +================================================================================================= =========== ========================================================================= + 1.17.1 ...... -Latest change: 2024-09-26 +Latest change: 2024-09-27 ================================================================================================= =========== ============================================================================================== Commit Committed Subject ================================================================================================= =========== ============================================================================================== +`bcee926d49 `_ 2024-09-27 ``Prepare docs for Sep 2nd adhoc wave of providers (#42519)`` `7ad586ed6a `_ 2024-09-26 ``fix(providers/common/sql): add dummy connection setter for backward compatibility (#42490)`` `47c71108a8 `_ 2024-09-22 ``Changed type hinting for handler function (#42275)`` ================================================================================================= =========== ============================================================================================== diff --git a/docs/apache-airflow-providers-common-sql/index.rst b/docs/apache-airflow-providers-common-sql/index.rst index 573603b2d7d04..f764b57a8b680 100644 --- a/docs/apache-airflow-providers-common-sql/index.rst +++ b/docs/apache-airflow-providers-common-sql/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-common-sql package `Common SQL Provider `__ -Release: 1.17.1 +Release: 1.18.0 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-common-sql 1.17.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-common-sql 1.17.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-sql 1.18.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-sql 1.18.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-databricks/commits.rst b/docs/apache-airflow-providers-databricks/commits.rst index c053936f0ba31..3ead2b2ce6cca 100644 --- a/docs/apache-airflow-providers-databricks/commits.rst +++ b/docs/apache-airflow-providers-databricks/commits.rst @@ -35,14 +35,28 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-08 ``add warning log when task_key>100 (#42813)`` +`5d51beee35 `_ 2024-10-03 ``Add 'on_kill' to Databricks Workflow Operator (#42115)`` +`9b90d2f216 `_ 2024-10-03 ``Add debug logs to print Request/Response data in Databricks provider (#42662)`` +================================================================================================= =========== ================================================================================== + 6.10.0 ...... -Latest change: 2024-08-30 +Latest change: 2024-09-21 ================================================================================================= =========== ============================================================================================================== Commit Committed Subject ================================================================================================= =========== ============================================================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `365b42f5a1 `_ 2024-08-30 ``[FEAT] databricks repair run with reason match and appropriate new settings (#41412)`` `8765039214 `_ 2024-08-27 ``Removed deprecated method referance airflow.www.auth.has_access when min airflow version >= 2.8.0 (#41747)`` `1613e9ec1c `_ 2024-08-25 ``remove soft_fail (#41710)`` diff --git a/docs/apache-airflow-providers-databricks/index.rst b/docs/apache-airflow-providers-databricks/index.rst index d7406fdaacd42..5a124818ea4dc 100644 --- a/docs/apache-airflow-providers-databricks/index.rst +++ b/docs/apache-airflow-providers-databricks/index.rst @@ -78,7 +78,7 @@ apache-airflow-providers-databricks package `Databricks `__ -Release: 6.10.0 +Release: 6.11.0 Provider package ---------------- @@ -137,5 +137,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-databricks 6.10.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-databricks 6.10.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-databricks 6.11.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-databricks 6.11.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-dbt-cloud/commits.rst b/docs/apache-airflow-providers-dbt-cloud/commits.rst index 6effec6f41cf9..276f3092e534d 100644 --- a/docs/apache-airflow-providers-dbt-cloud/commits.rst +++ b/docs/apache-airflow-providers-dbt-cloud/commits.rst @@ -35,16 +35,29 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-08 ``Simplify code for recent dbt provider change (#42840)`` +`c57be2ffc5 `_ 2024-10-08 ``Add ability to provide proxy for dbt Cloud connection (#42737)`` +================================================================================================= =========== ================================================================== + 3.10.1 ...... -Latest change: 2024-08-25 +Latest change: 2024-09-21 -================================================================================================= =========== ============================= +================================================================================================= =========== ======================================================= Commit Committed Subject -================================================================================================= =========== ============================= +================================================================================================= =========== ======================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `1613e9ec1c `_ 2024-08-25 ``remove soft_fail (#41710)`` -================================================================================================= =========== ============================= +================================================================================================= =========== ======================================================= 3.10.0 ...... diff --git a/docs/apache-airflow-providers-dbt-cloud/index.rst b/docs/apache-airflow-providers-dbt-cloud/index.rst index 47e64ff933661..f9f214af8323f 100644 --- a/docs/apache-airflow-providers-dbt-cloud/index.rst +++ b/docs/apache-airflow-providers-dbt-cloud/index.rst @@ -81,7 +81,7 @@ apache-airflow-providers-dbt-cloud package `dbt Cloud `__ -Release: 3.10.1 +Release: 3.11.0 Provider package ---------------- @@ -136,5 +136,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-dbt-cloud 3.10.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-dbt-cloud 3.10.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dbt-cloud 3.11.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dbt-cloud 3.11.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-elasticsearch/commits.rst b/docs/apache-airflow-providers-elasticsearch/commits.rst index 03b11128209c3..0cbc310101489 100644 --- a/docs/apache-airflow-providers-elasticsearch/commits.rst +++ b/docs/apache-airflow-providers-elasticsearch/commits.rst @@ -35,14 +35,27 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-09 ``Fix spelling; 'Airlfow' -> 'Airflow' (#42855)`` +`1cb9294c64 `_ 2024-10-06 ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` +================================================================================================= =========== ================================================================================================== + 5.5.1 ..... -Latest change: 2024-09-04 +Latest change: 2024-09-21 ================================================================================================= =========== ================================================================================================= Commit Committed Subject ================================================================================================= =========== ================================================================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `2e813eb87d `_ 2024-09-04 ``Generalize caching of connection in DbApiHook to improve performance (#40751)`` `b1e4f37505 `_ 2024-09-01 ``Fix ElasticSearch SQLClient deprecation warning (#41871)`` `79f6383c38 `_ 2024-08-21 ``filename template arg in providers file task handlers backward compitability support (#41633)`` diff --git a/docs/apache-airflow-providers-elasticsearch/index.rst b/docs/apache-airflow-providers-elasticsearch/index.rst index 91c65403a88e9..727d852d7cf26 100644 --- a/docs/apache-airflow-providers-elasticsearch/index.rst +++ b/docs/apache-airflow-providers-elasticsearch/index.rst @@ -79,7 +79,7 @@ apache-airflow-providers-elasticsearch package `Elasticsearch `__ -Release: 5.5.1 +Release: 5.5.2 Provider package ---------------- @@ -132,5 +132,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-elasticsearch 5.5.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-elasticsearch 5.5.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-elasticsearch 5.5.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-elasticsearch 5.5.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-fab/commits.rst b/docs/apache-airflow-providers-fab/commits.rst index 10c116ccf268e..3401b626d8f8c 100644 --- a/docs/apache-airflow-providers-fab/commits.rst +++ b/docs/apache-airflow-providers-fab/commits.rst @@ -35,6 +35,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-01 ``Update Rest API tests to no longer rely on FAB auth manager. Move tests specific to FAB permissions to FAB provider (#42523)`` +`ede7cb27fd `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +`2beb6a765d `_ 2024-09-25 ``Simplify expression for get_permitted_dag_ids query (#42484)`` +================================================================================================= =========== ================================================================================================================================ + 1.4.0 ..... @@ -43,6 +56,7 @@ Latest change: 2024-09-21 ================================================================================================= =========== =================================================================================== Commit Committed Subject ================================================================================================= =========== =================================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `6a527c9fac `_ 2024-09-21 ``Fix pre-commit for auto update of fab migration versions (#42382)`` `8741e9c176 `_ 2024-09-20 ``Handle 'AUTH_ROLE_PUBLIC' in FAB auth manager (#42280)`` `9f167bbc34 `_ 2024-09-19 ``Add FAB migration commands (#41804)`` diff --git a/docs/apache-airflow-providers-fab/index.rst b/docs/apache-airflow-providers-fab/index.rst index 30fd4b5aa224f..14135d0d15eae 100644 --- a/docs/apache-airflow-providers-fab/index.rst +++ b/docs/apache-airflow-providers-fab/index.rst @@ -83,7 +83,7 @@ apache-airflow-providers-fab package `Flask App Builder `__ -Release: 1.4.0 +Release: 1.4.1 Provider package ---------------- @@ -103,13 +103,42 @@ Requirements The minimum Apache Airflow version supported by this provider package is ``2.9.0``. -==================== ================== -PIP package Version required -==================== ================== -``apache-airflow`` ``>=2.9.0`` -``flask`` ``>=2.2,<2.3`` -``flask-appbuilder`` ``==4.5.0`` -``flask-login`` ``>=0.6.2`` -``google-re2`` ``>=1.0`` -``jmespath`` ``>=0.7.0`` -==================== ================== +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=2.9.0`` +``apache-airflow-providers-common-compat`` ``>=1.2.0`` +``flask`` ``>=2.2,<2.3`` +``flask-appbuilder`` ``==4.5.0`` +``flask-login`` ``>=0.6.2`` +``google-re2`` ``>=1.0`` +``jmespath`` ``>=0.7.0`` +========================================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider packages in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-fab[common.compat] + + +================================================================================================================== ================= +Dependent package Extra +================================================================================================================== ================= +`apache-airflow-providers-common-compat `_ ``common.compat`` +================================================================================================================== ================= + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-fab 1.4.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-fab 1.4.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-google/commits.rst b/docs/apache-airflow-providers-google/commits.rst index fd42bac7809a6..6148a1cf45cf4 100644 --- a/docs/apache-airflow-providers-google/commits.rst +++ b/docs/apache-airflow-providers-google/commits.rst @@ -42,14 +42,36 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-06 ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` +`2ffc389574 `_ 2024-10-04 ``Add 'retry_if_resource_not_ready' logic for DataprocCreateClusterOperator and DataprocCreateBatchOperator (#42703)`` +`8a255f3e1e `_ 2024-10-04 ``Publish Dataproc Serverless Batch link after it starts if batch_id was provided (#41153)`` +`9ec21405c0 `_ 2024-10-03 ``fix PubSubAsyncHook in PubsubPullTrigger to use gcp_conn_id (#42671)`` +`64e972c977 `_ 2024-10-02 ``Fix consistent return response from PubSubPullSensor (#42080)`` +`ede7cb27fd `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +`5689d64911 `_ 2024-09-27 ``Undo partition exclusion from the table name when splitting a full BigQuery table name (#42541)`` +`dc43d31c5a `_ 2024-09-27 ``Fix gcp text to speech uri fetch (#42309)`` +`56ab422a97 `_ 2024-09-27 ``Deprecate AutoMLBatchPredictOperator and refactor AutoMl system tests (#42260)`` +`b0234cbaf7 `_ 2024-09-27 ``Refactor ''bucket.get_blob'' calls in ''GCSHook'' to handle validation for non-existent objects. (#42474)`` +`eed1d0d254 `_ 2024-09-24 ``Bugfix/dataflow job location passing (#41887)`` +================================================================================================= =========== ====================================================================================================================== + 10.23.0 ....... -Latest change: 2024-09-20 +Latest change: 2024-09-21 ================================================================================================= =========== ==================================================================================================================================== Commit Committed Subject ================================================================================================= =========== ==================================================================================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `c051d0aa4c `_ 2024-09-20 ``Add ability to create Flink Jobs in dataproc cluster (#42342)`` `f278e62255 `_ 2024-09-19 ``Minor fixes to ensure successful Vertex AI LLMops pipeline (#41997)`` `d7343816b3 `_ 2024-09-19 ``Mark VertexAI AutoMLText deprecation (#42251)`` diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst index 1a81bb433e04d..05e05725aa17e 100644 --- a/docs/apache-airflow-providers-google/index.rst +++ b/docs/apache-airflow-providers-google/index.rst @@ -87,7 +87,7 @@ Google services including: - `Google Workspace `__ (formerly Google Suite) -Release: 10.23.0 +Release: 10.24.0 Provider package ---------------- @@ -220,5 +220,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-google 10.23.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-google 10.23.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 10.24.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 10.24.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-jdbc/commits.rst b/docs/apache-airflow-providers-jdbc/commits.rst index c218e7ccf01cf..21ce7e0081e8a 100644 --- a/docs/apache-airflow-providers-jdbc/commits.rst +++ b/docs/apache-airflow-providers-jdbc/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-07 ``FIX: Only pass connection to sqlalchemy engine in JdbcHook (#42705)`` +================================================================================================= =========== ======================================================================= + 4.5.1 ..... -Latest change: 2024-09-04 +Latest change: 2024-09-21 ================================================================================================= =========== ================================================================================= Commit Committed Subject ================================================================================================= =========== ================================================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `2e813eb87d `_ 2024-09-04 ``Generalize caching of connection in DbApiHook to improve performance (#40751)`` ================================================================================================= =========== ================================================================================= diff --git a/docs/apache-airflow-providers-jdbc/index.rst b/docs/apache-airflow-providers-jdbc/index.rst index b5e48c0be3e4d..e31369841cd7c 100644 --- a/docs/apache-airflow-providers-jdbc/index.rst +++ b/docs/apache-airflow-providers-jdbc/index.rst @@ -78,7 +78,7 @@ apache-airflow-providers-jdbc package `Java Database Connectivity (JDBC) `__ -Release: 4.5.1 +Release: 4.5.2 Provider package ---------------- @@ -131,5 +131,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-jdbc 4.5.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-jdbc 4.5.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-jdbc 4.5.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-jdbc 4.5.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-microsoft-azure/commits.rst b/docs/apache-airflow-providers-microsoft-azure/commits.rst index 88454472e5b43..1f18be632c20b 100644 --- a/docs/apache-airflow-providers-microsoft-azure/commits.rst +++ b/docs/apache-airflow-providers-microsoft-azure/commits.rst @@ -35,14 +35,28 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-06 ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` +`d89f305a34 `_ 2024-09-29 ``Bugfix/42575 workaround pin azure kusto data (#42576)`` +`193defd289 `_ 2024-09-25 ``(bugfix): Paginated results in MSGraphAsyncOperator (#42414)`` +================================================================================================= =========== ================================================================================================== + 10.5.0 ...... -Latest change: 2024-09-05 +Latest change: 2024-09-21 ================================================================================================= =========== ======================================================================== Commit Committed Subject ================================================================================================= =========== ======================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `ede34ebee7 `_ 2024-09-05 ``Allow custom api versions in MSGraphAsyncOperator (#41331)`` `85ad5c374d `_ 2024-09-05 `` Add callback to process Azure Service Bus message contents (#41601)`` `1613e9ec1c `_ 2024-08-25 ``remove soft_fail (#41710)`` diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst index 879e896cd2371..7a073a3b72341 100644 --- a/docs/apache-airflow-providers-microsoft-azure/index.rst +++ b/docs/apache-airflow-providers-microsoft-azure/index.rst @@ -82,7 +82,7 @@ apache-airflow-providers-microsoft-azure package `Microsoft Azure `__ -Release: 10.5.0 +Release: 10.5.1 Provider package ---------------- @@ -102,9 +102,9 @@ Requirements The minimum Apache Airflow version supported by this provider package is ``2.8.0``. -================================ ================== +================================ =================== PIP package Version required -================================ ================== +================================ =================== ``apache-airflow`` ``>=2.8.0`` ``adlfs`` ``>=2023.10.0`` ``azure-batch`` ``>=8.0.0`` @@ -123,12 +123,12 @@ PIP package Version required ``azure-synapse-artifacts`` ``>=0.17.0`` ``adal`` ``>=1.2.7`` ``azure-storage-file-datalake`` ``>=12.9.1`` -``azure-kusto-data`` ``>=4.1.0`` +``azure-kusto-data`` ``>=4.1.0,!=4.6.0`` ``azure-mgmt-datafactory`` ``>=2.0.0`` ``azure-mgmt-containerregistry`` ``>=8.0.0`` ``azure-mgmt-containerinstance`` ``>=10.1.0`` ``msgraph-core`` ``>=1.0.0`` -================================ ================== +================================ =================== Cross provider package dependencies ----------------------------------- @@ -158,5 +158,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-microsoft-azure 10.5.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-microsoft-azure 10.5.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-microsoft-azure 10.5.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-microsoft-azure 10.5.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-mysql/commits.rst b/docs/apache-airflow-providers-mysql/commits.rst index 783a88e8784dd..d01a1f5b98ad8 100644 --- a/docs/apache-airflow-providers-mysql/commits.rst +++ b/docs/apache-airflow-providers-mysql/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +================================================================================================= =========== ================================================================== + 5.7.1 ..... -Latest change: 2024-09-04 +Latest change: 2024-09-21 ================================================================================================= =========== ================================================================================= Commit Committed Subject ================================================================================================= =========== ================================================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `2e813eb87d `_ 2024-09-04 ``Generalize caching of connection in DbApiHook to improve performance (#40751)`` ================================================================================================= =========== ================================================================================= diff --git a/docs/apache-airflow-providers-mysql/index.rst b/docs/apache-airflow-providers-mysql/index.rst index 63b68a2fcac2a..622084d88957c 100644 --- a/docs/apache-airflow-providers-mysql/index.rst +++ b/docs/apache-airflow-providers-mysql/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-mysql package `MySQL `__ -Release: 5.7.1 +Release: 5.7.2 Provider package ---------------- @@ -136,5 +136,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-mysql 5.7.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-mysql 5.7.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-mysql 5.7.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-mysql 5.7.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-openlineage/commits.rst b/docs/apache-airflow-providers-openlineage/commits.rst index d2e20868233c3..41025a8f7b445 100644 --- a/docs/apache-airflow-providers-openlineage/commits.rst +++ b/docs/apache-airflow-providers-openlineage/commits.rst @@ -35,14 +35,28 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-09 ``Standard provider bash operator (#42252)`` +`63ff22f403 `_ 2024-10-08 ``Drop python3.8 support core and providers (#42766)`` +`ede7cb27fd `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +================================================================================================= =========== ================================================================== + 1.12.1 ...... -Latest change: 2024-09-26 +Latest change: 2024-09-27 ================================================================================================= =========== ======================================================================== Commit Committed Subject ================================================================================================= =========== ======================================================================== +`bcee926d49 `_ 2024-09-27 ``Prepare docs for Sep 2nd adhoc wave of providers (#42519)`` `84e8cdf674 `_ 2024-09-26 ``fix: OL dag start event not being emitted (#42448)`` `ffff0e8b33 `_ 2024-09-23 ``Fix typo in error stack trace formatting for clearer output (#42017)`` ================================================================================================= =========== ======================================================================== diff --git a/docs/apache-airflow-providers-openlineage/index.rst b/docs/apache-airflow-providers-openlineage/index.rst index 817bddd6c0b74..f0173f4e2c6fe 100644 --- a/docs/apache-airflow-providers-openlineage/index.rst +++ b/docs/apache-airflow-providers-openlineage/index.rst @@ -73,7 +73,7 @@ apache-airflow-providers-openlineage package `OpenLineage `__ -Release: 1.12.1 +Release: 1.12.2 Provider package ---------------- @@ -114,14 +114,15 @@ You can install such cross-provider dependencies when installing from PyPI. For .. code-block:: bash - pip install apache-airflow-providers-openlineage[common.sql] + pip install apache-airflow-providers-openlineage[common.compat] -============================================================================================================ ============== -Dependent package Extra -============================================================================================================ ============== -`apache-airflow-providers-common-sql `_ ``common.sql`` -============================================================================================================ ============== +================================================================================================================== ================= +Dependent package Extra +================================================================================================================== ================= +`apache-airflow-providers-common-compat `_ ``common.compat`` +`apache-airflow-providers-common-sql `_ ``common.sql`` +================================================================================================================== ================= Downloading official packages ----------------------------- @@ -129,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-openlineage 1.12.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-openlineage 1.12.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-openlineage 1.12.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-openlineage 1.12.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-opensearch/commits.rst b/docs/apache-airflow-providers-opensearch/commits.rst index 7d797af110858..852704be06bee 100644 --- a/docs/apache-airflow-providers-opensearch/commits.rst +++ b/docs/apache-airflow-providers-opensearch/commits.rst @@ -35,14 +35,28 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-06 ``Removed conditional check for task context logging in airflow version 2.8.0 and above (#42764)`` +`b0a18d9019 `_ 2024-10-04 ``Handle empty login and password with opensearch client (#39982)`` +`ba1c602533 `_ 2024-09-21 ``(feat): Add opensearch logging integration (#41799)`` +================================================================================================= =========== ================================================================================================== + 1.4.0 ..... -Latest change: 2024-08-12 +Latest change: 2024-08-19 ================================================================================================= =========== ======================================================================= Commit Committed Subject ================================================================================================= =========== ======================================================================= +`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` `fcbff15bda `_ 2024-08-12 ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` ================================================================================================= =========== ======================================================================= diff --git a/docs/apache-airflow-providers-opensearch/index.rst b/docs/apache-airflow-providers-opensearch/index.rst index d15064228892b..de79657141fad 100644 --- a/docs/apache-airflow-providers-opensearch/index.rst +++ b/docs/apache-airflow-providers-opensearch/index.rst @@ -78,7 +78,7 @@ apache-airflow-providers-opensearch package `OpenSearch `__ -Release: 1.4.0 +Release: 1.5.0 Provider package ---------------- diff --git a/docs/apache-airflow-providers-postgres/commits.rst b/docs/apache-airflow-providers-postgres/commits.rst index 538927251f5f3..a1ec69eff688c 100644 --- a/docs/apache-airflow-providers-postgres/commits.rst +++ b/docs/apache-airflow-providers-postgres/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +================================================================================================= =========== ================================================================== + 5.13.0 ...... -Latest change: 2024-09-05 +Latest change: 2024-09-21 ================================================================================================= =========== ================================================================================= Commit Committed Subject ================================================================================================= =========== ================================================================================= +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `17c30b4f21 `_ 2024-09-05 ``feat: log client db messages for provider postgres (#40171)`` `2e813eb87d `_ 2024-09-04 ``Generalize caching of connection in DbApiHook to improve performance (#40751)`` ================================================================================================= =========== ================================================================================= diff --git a/docs/apache-airflow-providers-postgres/index.rst b/docs/apache-airflow-providers-postgres/index.rst index 9b6124999a972..6fbd69ba1c5aa 100644 --- a/docs/apache-airflow-providers-postgres/index.rst +++ b/docs/apache-airflow-providers-postgres/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-postgres package `PostgreSQL `__ -Release: 5.13.0 +Release: 5.13.1 Provider package ---------------- @@ -132,5 +132,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-postgres 5.13.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-postgres 5.13.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-postgres 5.13.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-postgres 5.13.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-snowflake/commits.rst b/docs/apache-airflow-providers-snowflake/commits.rst index 2b7c230e5e726..9d6c44634188b 100644 --- a/docs/apache-airflow-providers-snowflake/commits.rst +++ b/docs/apache-airflow-providers-snowflake/commits.rst @@ -35,14 +35,28 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-03 ``- Snowflake does not currently support variable binding in multi-statement SQL requests. (#42719)`` +`976064dc6c `_ 2024-10-02 ``Add Snowpark operator and decorator (#42457)`` +`8a5eb22df1 `_ 2024-10-02 ``Make 'private_key_content' a sensitive field in Snowflake connection (#42649)`` +================================================================================================= =========== ===================================================================================================== + 5.7.1 ..... -Latest change: 2024-08-27 +Latest change: 2024-09-21 ================================================================================================= =========== ==================================================================== Commit Committed Subject ================================================================================================= =========== ==================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` `33f81bfb93 `_ 2024-08-27 ``Update snowflake naming for account names and locators. (#41775)`` ================================================================================================= =========== ==================================================================== diff --git a/docs/apache-airflow-providers-snowflake/index.rst b/docs/apache-airflow-providers-snowflake/index.rst index b00ea39c5261e..c9746f7806943 100644 --- a/docs/apache-airflow-providers-snowflake/index.rst +++ b/docs/apache-airflow-providers-snowflake/index.rst @@ -78,7 +78,7 @@ apache-airflow-providers-snowflake package `Snowflake `__ -Release: 5.7.1 +Release: 5.8.0 Provider package ---------------- @@ -109,6 +109,7 @@ PIP package Version required ``pyarrow`` ``>=14.0.1`` ``snowflake-connector-python`` ``>=3.7.1`` ``snowflake-sqlalchemy`` ``>=1.4.0`` +``snowflake-snowpark-python`` ``>=1.17.0; python_version < "3.12"`` ========================================== ========================================= Cross provider package dependencies @@ -138,5 +139,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-snowflake 5.7.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-snowflake 5.7.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-snowflake 5.8.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-snowflake 5.8.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-trino/commits.rst b/docs/apache-airflow-providers-trino/commits.rst index f001e5d9dea15..46482cdfd610e 100644 --- a/docs/apache-airflow-providers-trino/commits.rst +++ b/docs/apache-airflow-providers-trino/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +================================================================================================= =========== ================================================================== + 5.8.0 ..... -Latest change: 2024-08-12 +Latest change: 2024-08-19 ================================================================================================= =========== ======================================================================= Commit Committed Subject ================================================================================================= =========== ======================================================================= +`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` `fcbff15bda `_ 2024-08-12 ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` ================================================================================================= =========== ======================================================================= diff --git a/docs/apache-airflow-providers-trino/index.rst b/docs/apache-airflow-providers-trino/index.rst index 07991e5da90b5..9a17b5907e5db 100644 --- a/docs/apache-airflow-providers-trino/index.rst +++ b/docs/apache-airflow-providers-trino/index.rst @@ -78,7 +78,7 @@ apache-airflow-providers-trino package `Trino `__ -Release: 5.8.0 +Release: 5.8.1 Provider package ---------------- @@ -135,5 +135,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-trino 5.8.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-trino 5.8.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-trino 5.8.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-trino 5.8.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-ydb/commits.rst b/docs/apache-airflow-providers-ydb/commits.rst index 01b90e938deb1..796104f3acde8 100644 --- a/docs/apache-airflow-providers-ydb/commits.rst +++ b/docs/apache-airflow-providers-ydb/commits.rst @@ -35,14 +35,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-03 ``Add an ability to use scan queries via new YDB operator (#42311)`` +================================================================================================= =========== ==================================================================== + 1.3.0 ..... -Latest change: 2024-08-14 +Latest change: 2024-08-19 ================================================================================================= =========== ==================================================================================================== Commit Committed Subject ================================================================================================= =========== ==================================================================================================== +`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` `6e22364278 `_ 2024-08-14 ``ydb provider: add database to table name in bulk upsert, use bulk upsert in system test (#41303)`` `fcbff15bda `_ 2024-08-12 ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` ================================================================================================= =========== ==================================================================================================== diff --git a/docs/apache-airflow-providers-ydb/index.rst b/docs/apache-airflow-providers-ydb/index.rst index 45cbd2b00b7a6..4dff0e421f9bd 100644 --- a/docs/apache-airflow-providers-ydb/index.rst +++ b/docs/apache-airflow-providers-ydb/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-ydb package `YDB `__ -Release: 1.3.0 +Release: 1.4.0 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-ydb 1.3.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-ydb 1.3.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-ydb 1.4.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-ydb 1.4.0 wheel package `_ (`asc `__, `sha512 `__) From f8a4fc6bcf7f1690b8434536f1faefd96d32061f Mon Sep 17 00:00:00 2001 From: Danny Liu Date: Wed, 9 Oct 2024 11:41:53 -0700 Subject: [PATCH 037/199] fix more PT004 PyDocStyle checks (#42841) --- tests/www/views/test_views_acl.py | 8 ++-- .../www/views/test_views_cluster_activity.py | 12 +++--- tests/www/views/test_views_connection.py | 2 +- tests/www/views/test_views_dagrun.py | 4 +- tests/www/views/test_views_dataset.py | 2 +- tests/www/views/test_views_decorators.py | 2 +- tests/www/views/test_views_extra_links.py | 6 +-- tests/www/views/test_views_grid.py | 6 +-- tests/www/views/test_views_home.py | 38 +++++++++---------- tests/www/views/test_views_log.py | 2 +- tests/www/views/test_views_pool.py | 2 +- tests/www/views/test_views_rendered.py | 4 +- 12 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index 139644f67a6da..7b60a2c11ecad 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -136,13 +136,13 @@ def acl_app(app): @pytest.fixture(scope="module") -def reset_dagruns(): +def _reset_dagruns(): """Clean up stray garbage from other tests.""" clear_db_runs() @pytest.fixture(autouse=True) -def init_dagruns(acl_app, reset_dagruns): +def _init_dagruns(acl_app, _reset_dagruns): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} acl_app.dag_bag.get_dag("example_bash_operator").create_dagrun( run_id=DEFAULT_RUN_ID, @@ -322,7 +322,7 @@ def test_dag_autocomplete_dag_display_name(client_all_dags): @pytest.fixture -def setup_paused_dag(): +def _setup_paused_dag(): """Pause a DAG so we can test filtering.""" dag_to_pause = "example_branch_operator" with create_session() as session: @@ -339,7 +339,7 @@ def setup_paused_dag(): ("paused", "example_branch_operator", "example_branch_labels"), ], ) -@pytest.mark.usefixtures("setup_paused_dag") +@pytest.mark.usefixtures("_setup_paused_dag") def test_dag_autocomplete_status(client_all_dags, status, expected, unexpected): with client_all_dags.session_transaction() as flask_session: flask_session[FILTER_STATUS_COOKIE] = status diff --git a/tests/www/views/test_views_cluster_activity.py b/tests/www/views/test_views_cluster_activity.py index cdcebc1c8cf34..543d563e8da03 100644 --- a/tests/www/views/test_views_cluster_activity.py +++ b/tests/www/views/test_views_cluster_activity.py @@ -39,20 +39,20 @@ def examples_dag_bag(): @pytest.fixture(autouse=True) -def clean(): +def _clean(): clear_db_runs() yield clear_db_runs() -# freeze time fixture so that it is applied before `make_dag_runs` is! +# freeze time fixture so that it is applied before `_make_dag_runs` is! @pytest.fixture -def freeze_time_for_dagruns(time_machine): +def _freeze_time_for_dagruns(time_machine): time_machine.move_to("2023-05-02T00:00:00+00:00", tick=False) @pytest.fixture -def make_dag_runs(dag_maker, session, time_machine): +def _make_dag_runs(dag_maker, session, time_machine): with dag_maker( dag_id="test_dag_id", serialized=True, @@ -99,7 +99,7 @@ def make_dag_runs(dag_maker, session, time_machine): session.flush() -@pytest.mark.usefixtures("freeze_time_for_dagruns", "make_dag_runs") +@pytest.mark.usefixtures("_freeze_time_for_dagruns", "_make_dag_runs") def test_historical_metrics_data(admin_client, session, time_machine): resp = admin_client.get( "/object/historical_metrics_data?start_date=2023-01-01T00:00&end_date=2023-08-02T00:00", @@ -127,7 +127,7 @@ def test_historical_metrics_data(admin_client, session, time_machine): } -@pytest.mark.usefixtures("freeze_time_for_dagruns", "make_dag_runs") +@pytest.mark.usefixtures("_freeze_time_for_dagruns", "_make_dag_runs") def test_historical_metrics_data_date_filters(admin_client, session): resp = admin_client.get( "/object/historical_metrics_data?start_date=2023-02-02T00:00&end_date=2023-06-02T00:00", diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index a209cdfc2be8a..7530a1f677f9b 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -47,7 +47,7 @@ def conn_with_extra() -> dict[str, Any]: @pytest.fixture(autouse=True) -def clear_connections(): +def _clear_connections(): with create_session() as session: session.query(Connection).delete() diff --git a/tests/www/views/test_views_dagrun.py b/tests/www/views/test_views_dagrun.py index d95955246ac78..9e45464c28aa1 100644 --- a/tests/www/views/test_views_dagrun.py +++ b/tests/www/views/test_views_dagrun.py @@ -93,7 +93,7 @@ def client_dr_without_dag_run_create(app): @pytest.fixture(scope="module", autouse=True) -def init_blank_dagrun(): +def _init_blank_dagrun(): """Make sure there are no runs before we test anything. This really shouldn't be needed, but tests elsewhere leave the db dirty. @@ -104,7 +104,7 @@ def init_blank_dagrun(): @pytest.fixture(autouse=True) -def reset_dagrun(): +def _reset_dagrun(): yield with create_session() as session: session.query(DagRun).delete() diff --git a/tests/www/views/test_views_dataset.py b/tests/www/views/test_views_dataset.py index 3d3351bb6493a..3db16a996862e 100644 --- a/tests/www/views/test_views_dataset.py +++ b/tests/www/views/test_views_dataset.py @@ -32,7 +32,7 @@ class TestDatasetEndpoint: @pytest.fixture(autouse=True) - def cleanup(self): + def _cleanup(self): clear_db_assets() yield clear_db_assets() diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py index e1bc1db0efc48..86bce2f3bbbc2 100644 --- a/tests/www/views/test_views_decorators.py +++ b/tests/www/views/test_views_decorators.py @@ -78,7 +78,7 @@ def dagruns(bash_dag, xcom_dag): @pytest.fixture(autouse=True) -def clean_db(): +def _clean_db(): clear_db_variables() yield clear_db_variables() diff --git a/tests/www/views/test_views_extra_links.py b/tests/www/views/test_views_extra_links.py index 55b2424f3a9f0..4fb0d2d40b017 100644 --- a/tests/www/views/test_views_extra_links.py +++ b/tests/www/views/test_views_extra_links.py @@ -101,7 +101,7 @@ def dag_run(create_dag_run, session): @pytest.fixture(scope="module", autouse=True) -def patched_app(app, dag): +def _patched_app(app, dag): with mock.patch.object(app, "dag_bag") as mock_dag_bag: mock_dag_bag.get_dag.return_value = dag yield @@ -123,7 +123,7 @@ def task_3(dag): @pytest.fixture(scope="module", autouse=True) -def init_blank_task_instances(): +def _init_blank_task_instances(): """Make sure there are no runs before we test anything. This really shouldn't be needed, but tests elsewhere leave the db dirty. @@ -132,7 +132,7 @@ def init_blank_task_instances(): @pytest.fixture(autouse=True) -def reset_task_instances(): +def _reset_task_instances(): yield clear_db_runs() diff --git a/tests/www/views/test_views_grid.py b/tests/www/views/test_views_grid.py index b4dd6f6082e57..8d5e4e22e04b8 100644 --- a/tests/www/views/test_views_grid.py +++ b/tests/www/views/test_views_grid.py @@ -54,7 +54,7 @@ def examples_dag_bag(): @pytest.fixture(autouse=True) -def clean(): +def _clean(): clear_db_runs() clear_db_assets() yield @@ -187,11 +187,11 @@ def test_grid_data_filtered_on_run_type_and_run_state(admin_client, dag_with_run # Create this as a fixture so that it is applied before the `dag_with_runs` fixture is! @pytest.fixture -def freeze_time_for_dagruns(time_machine): +def _freeze_time_for_dagruns(time_machine): time_machine.move_to("2022-01-02T00:00:00+00:00", tick=False) -@pytest.mark.usefixtures("freeze_time_for_dagruns") +@pytest.mark.usefixtures("_freeze_time_for_dagruns") def test_one_run(admin_client, dag_with_runs: list[DagRun], session): """ Test a DAG with complex interaction of states: diff --git a/tests/www/views/test_views_home.py b/tests/www/views/test_views_home.py index 44dda24feecbc..6a5bf3c4ad531 100644 --- a/tests/www/views/test_views_home.py +++ b/tests/www/views/test_views_home.py @@ -42,7 +42,7 @@ def clean_db(): @pytest.fixture(autouse=True) -def setup(): +def _setup(): clean_db() yield clean_db() @@ -72,7 +72,7 @@ def test_home(capture_templates, admin_client): @mock.patch("airflow.www.views.AirflowBaseView.render_template") -def test_home_dags_count(render_template_mock, admin_client, working_dags, session): +def test_home_dags_count(render_template_mock, admin_client, _working_dags, session): from sqlalchemy import update from airflow.models.dag import DagModel @@ -204,7 +204,7 @@ def _process_file(file_path): @pytest.fixture -def working_dags(tmp_path): +def _working_dags(tmp_path): dag_contents_template = "from airflow import DAG\ndag = DAG('{}', schedule=None, tags=['{}'])" for dag_id, tag in zip(TEST_FILTER_DAG_IDS, TEST_TAGS): path = tmp_path / f"{dag_id}.py" @@ -213,7 +213,7 @@ def working_dags(tmp_path): @pytest.fixture -def working_dags_with_read_perm(tmp_path): +def _working_dags_with_read_perm(tmp_path): dag_contents_template = "from airflow import DAG\ndag = DAG('{}', schedule=None, tags=['{}'])" dag_contents_template_with_read_perm = ( "from airflow import DAG\ndag = DAG('{}', schedule=None, tags=['{}'], " @@ -229,7 +229,7 @@ def working_dags_with_read_perm(tmp_path): @pytest.fixture -def working_dags_with_edit_perm(tmp_path): +def _working_dags_with_edit_perm(tmp_path): dag_contents_template = "from airflow import DAG\ndag = DAG('{}', schedule=None, tags=['{}'])" dag_contents_template_with_read_perm = ( "from airflow import DAG\ndag = DAG('{}', schedule=None, tags=['{}'], " @@ -245,7 +245,7 @@ def working_dags_with_edit_perm(tmp_path): @pytest.fixture -def broken_dags(tmp_path, working_dags): +def _broken_dags(tmp_path, _working_dags): for dag_id in TEST_FILTER_DAG_IDS: path = tmp_path / f"{dag_id}.py" path.write_text("airflow DAG") @@ -253,7 +253,7 @@ def broken_dags(tmp_path, working_dags): @pytest.fixture -def broken_dags_with_read_perm(tmp_path, working_dags_with_read_perm): +def _broken_dags_with_read_perm(tmp_path, _working_dags_with_read_perm): for dag_id in TEST_FILTER_DAG_IDS: path = tmp_path / f"{dag_id}.py" path.write_text("airflow DAG") @@ -261,7 +261,7 @@ def broken_dags_with_read_perm(tmp_path, working_dags_with_read_perm): @pytest.fixture -def broken_dags_after_working(tmp_path): +def _broken_dags_after_working(tmp_path): # First create and process a DAG file that works path = tmp_path / "all_in_one.py" contents = "from airflow import DAG\n" @@ -275,7 +275,7 @@ def broken_dags_after_working(tmp_path): _process_file(path) -def test_home_filter_tags(working_dags, admin_client): +def test_home_filter_tags(_working_dags, admin_client): with admin_client: admin_client.get("home?tags=example&tags=data", follow_redirects=True) assert "example,data" == flask.session[FILTER_TAGS_COOKIE] @@ -284,7 +284,7 @@ def test_home_filter_tags(working_dags, admin_client): assert flask.session[FILTER_TAGS_COOKIE] is None -def test_home_importerrors(broken_dags, user_client): +def test_home_importerrors(_broken_dags, user_client): # Users with "can read on DAGs" gets all DAG import errors resp = user_client.get("home", follow_redirects=True) check_content_in_response("Import Errors", resp) @@ -292,7 +292,7 @@ def test_home_importerrors(broken_dags, user_client): check_content_in_response(f"/{dag_id}.py", resp) -def test_home_no_importerrors_perm(broken_dags, client_no_importerror): +def test_home_no_importerrors_perm(_broken_dags, client_no_importerror): # Users without "can read on import errors" don't see any import errors resp = client_no_importerror.get("home", follow_redirects=True) check_content_not_in_response("Import Errors", resp) @@ -310,7 +310,7 @@ def test_home_no_importerrors_perm(broken_dags, client_no_importerror): "home?lastrun=all_states", ], ) -def test_home_importerrors_filtered_singledag_user(broken_dags_with_read_perm, client_single_dag, page): +def test_home_importerrors_filtered_singledag_user(_broken_dags_with_read_perm, client_single_dag, page): # Users that can only see certain DAGs get a filtered list of import errors resp = client_single_dag.get(page, follow_redirects=True) check_content_in_response("Import Errors", resp) @@ -322,7 +322,7 @@ def test_home_importerrors_filtered_singledag_user(broken_dags_with_read_perm, c check_content_not_in_response(f"/{dag_id}.py", resp) -def test_home_importerrors_missing_read_on_all_dags_in_file(broken_dags_after_working, client_single_dag): +def test_home_importerrors_missing_read_on_all_dags_in_file(_broken_dags_after_working, client_single_dag): # If a user doesn't have READ on all DAGs in a file, that files traceback is redacted resp = client_single_dag.get("home", follow_redirects=True) check_content_in_response("Import Errors", resp) @@ -333,14 +333,14 @@ def test_home_importerrors_missing_read_on_all_dags_in_file(broken_dags_after_wo check_content_in_response("REDACTED", resp) -def test_home_dag_list(working_dags, user_client): +def test_home_dag_list(_working_dags, user_client): # Users with "can read on DAGs" gets all DAGs resp = user_client.get("home", follow_redirects=True) for dag_id in TEST_FILTER_DAG_IDS: check_content_in_response(f"dag_id={dag_id}", resp) -def test_home_dag_list_filtered_singledag_user(working_dags_with_read_perm, client_single_dag): +def test_home_dag_list_filtered_singledag_user(_working_dags_with_read_perm, client_single_dag): # Users that can only see certain DAGs get a filtered list resp = client_single_dag.get("home", follow_redirects=True) # They can see the first DAG @@ -350,7 +350,7 @@ def test_home_dag_list_filtered_singledag_user(working_dags_with_read_perm, clie check_content_not_in_response(f"dag_id={dag_id}", resp) -def test_home_dag_list_search(working_dags, user_client): +def test_home_dag_list_search(_working_dags, user_client): resp = user_client.get("home?search=filter_test", follow_redirects=True) check_content_in_response("dag_id=filter_test_1", resp) check_content_in_response("dag_id=filter_test_2", resp) @@ -358,7 +358,7 @@ def test_home_dag_list_search(working_dags, user_client): check_content_not_in_response("dag_id=a_first_dag_id_asc", resp) -def test_home_dag_edit_permissions(capture_templates, working_dags_with_edit_perm, client_single_dag_edit): +def test_home_dag_edit_permissions(capture_templates, _working_dags_with_edit_perm, client_single_dag_edit): with capture_templates() as templates: client_single_dag_edit.get("home", follow_redirects=True) @@ -446,7 +446,7 @@ def test_dashboard_flash_messages_type(user_client): ], ids=["no_order_provided", "ascending_order_on_dag_id", "descending_order_on_dag_id"], ) -def test_sorting_home_view(url, lower_key, greater_key, user_client, working_dags): +def test_sorting_home_view(url, lower_key, greater_key, user_client, _working_dags): resp = user_client.get(url, follow_redirects=True) resp_html = resp.data.decode("utf-8") lower_index = resp_html.find(lower_key) @@ -489,7 +489,7 @@ def test_analytics_pixel(user_client, is_enabled, should_have_pixel): ], ) def test_filter_cookie_eval( - working_dags, + _working_dags, admin_client, url, filter_tags_cookie_val, diff --git a/tests/www/views/test_views_log.py b/tests/www/views/test_views_log.py index f59fe7418f395..72e282c7b3f8c 100644 --- a/tests/www/views/test_views_log.py +++ b/tests/www/views/test_views_log.py @@ -117,7 +117,7 @@ def factory(): @pytest.fixture(autouse=True) -def reset_modules_after_every_test(backup_modules): +def _reset_modules_after_every_test(backup_modules): yield # Remove any new modules imported during the test run. # This lets us import the same source files for more than one test. diff --git a/tests/www/views/test_views_pool.py b/tests/www/views/test_views_pool.py index 3fcacbbbf8bed..020c5353a5f34 100644 --- a/tests/www/views/test_views_pool.py +++ b/tests/www/views/test_views_pool.py @@ -36,7 +36,7 @@ @pytest.fixture(autouse=True) -def clear_pools(): +def _clear_pools(): with create_session() as session: session.query(Pool).delete() diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 2d1754af29f65..87e693d5400ab 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -119,7 +119,7 @@ def task_secret(dag): @pytest.fixture(scope="module", autouse=True) -def init_blank_db(): +def _init_blank_db(): """Make sure there are no runs before we test anything. This really shouldn't be needed, but tests elsewhere leave the db dirty. @@ -130,7 +130,7 @@ def init_blank_db(): @pytest.fixture(autouse=True) -def reset_db(dag, task1, task2, task3, task4, task_secret): +def _reset_db(dag, task1, task2, task3, task4, task_secret): yield clear_db_dags() clear_db_runs() From eb0a028b77e796ea6a30a9ed2b75d79064e7f0c7 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 9 Oct 2024 19:45:50 +0100 Subject: [PATCH 038/199] Revert Asset to Dataset for Core Extension Doc (#42867) This PR reverts Core Extension doc change (https://airflow.apache.org/docs/apache-airflow-providers/core-extensions/index.html) from https://github.com/apache/airflow/pull/41348 Since those docs only work on Airflow stable, we can only change this after 3.0 --- .../core-extensions/asset-schemes.rst | 11 +++++++---- .../howto/create-custom-providers.rst | 7 +++++-- docs/exts/operators_and_hooks_ref.py | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/apache-airflow-providers/core-extensions/asset-schemes.rst b/docs/apache-airflow-providers/core-extensions/asset-schemes.rst index 0a629b9f8a4fd..87bfd8ae67652 100644 --- a/docs/apache-airflow-providers/core-extensions/asset-schemes.rst +++ b/docs/apache-airflow-providers/core-extensions/asset-schemes.rst @@ -15,14 +15,17 @@ specific language governing permissions and limitations under the License. -asset URI Schemes +Dataset URI Schemes ------------------- -This is a summary of all Apache Airflow Community provided implementations of asset URI schemes +This is a summary of all Apache Airflow Community provided implementations of dataset URI schemes exposed via Airflow core and community-managed providers. -See :doc:`documentation on assets ` for details on how asset URIs work. +.. + TODO: Change this from Dataset to Asset in Airflow 3.0 -.. airflow-asset-schemes:: +See :doc:`documentation on Datasets ` for details on how dataset URIs work. + +.. airflow-dataset-schemes:: :tags: None :header-separator: " diff --git a/docs/apache-airflow-providers/howto/create-custom-providers.rst b/docs/apache-airflow-providers/howto/create-custom-providers.rst index d95719e38bc6c..ba6d8068c4281 100644 --- a/docs/apache-airflow-providers/howto/create-custom-providers.rst +++ b/docs/apache-airflow-providers/howto/create-custom-providers.rst @@ -96,9 +96,12 @@ Exposing customized functionality to the Airflow's core: * ``filesystems`` - this field should contain the list of all the filesystem module names. See :doc:`apache-airflow:core-concepts/objectstorage` for description of the filesystems. -* ``asset-uris`` - this field should contain the list of the URI schemes together with +.. + TODO: Change this from Dataset to Asset in Airflow 3.0 + +* ``dataset-uris`` - this field should contain the list of the URI schemes together with class names implementing normalization functions. - See :doc:`apache-airflow:authoring-and-scheduling/assets` for description of the asset URIs. + See :doc:`apache-airflow:authoring-and-scheduling/datasets` for description of the dataset URIs. .. note:: Deprecated values diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index fe6cd5d3300d2..25b9d1d779426 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -538,7 +538,7 @@ def setup(app): app.add_directive("airflow-executors", ExecutorsDirective) app.add_directive("airflow-deferrable-operators", DeferrableOperatorDirective) app.add_directive("airflow-deprecations", DeprecationsDirective) - app.add_directive("airflow-asset-schemes", DatasetSchemeDirective) + app.add_directive("airflow-dataset-schemes", DatasetSchemeDirective) return {"parallel_read_safe": True, "parallel_write_safe": True} From 66a3380adc0bca3b3edd9860ebeceed955bfca07 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Wed, 9 Oct 2024 20:24:53 +0100 Subject: [PATCH 039/199] Split providers out of the main "airflow/" tree into a UV workspace project (#42505) This is only a partial split so far. It moves all the code and tests, but leaves the creation of `core/` to a separate PR as this is already large enough. In addition to the straight file rename the other changes I had to make here are: - Some mypy/typing fixes. Mypy can be fragile about what it picks up when, so maybe some of those changes were caused by that. But the typing changes aren't large. - Improve typing in common.sql type stub Again, likely a mypy file oddity, but the types should be safe - Removed the `check-providers-init-file-missing` check This isn't needed now that airflow/providers shouldn't exist at all in the main tree. - Create a "dev.tests_common" package that contains helper files and common pytest fixtures Since the provider tests are no longer under tests/ they don't automatically share the fixtures from the parent `tests/conftest.py` so they needed extracted. Ditto for `tests.test_utils` -- they can't be easily imported in provider tests anymore, so they are moved to a more explicit shared location. In future we should switch how the CI image is built to make better use of UV caching than our own approach as that would remvoe a lot of custom code. --- .dockerignore | 1 + .github/boring-cyborg.yml | 500 +++--- .github/workflows/basic-tests.yml | 5 +- .pre-commit-config.yaml | 142 +- Dockerfile | 7 +- Dockerfile.ci | 8 +- airflow/models/dag.py | 4 +- airflow/providers/.gitignore | 1 - airflow/providers_manager.py | 6 +- airflow/settings.py | 3 +- contributing-docs/08_static_code_checks.rst | 2 - contributing-docs/testing/system_tests.rst | 2 +- contributing-docs/testing/unit_tests.rst | 13 +- dev/breeze/doc/images/output_build-docs.svg | 28 +- .../doc/images/output_compile-ui-assets.svg | 10 +- ...release-management_add-back-references.svg | 12 +- ...ement_generate-issue-content-providers.svg | 10 +- ...agement_prepare-provider-documentation.svg | 24 +- ...e-management_prepare-provider-packages.svg | 26 +- ...output_release-management_publish-docs.svg | 28 +- ...t_sbom_generate-providers-requirements.svg | 26 +- ...utput_setup_check-all-params-in-groups.svg | 8 +- ...output_setup_regenerate-command-images.svg | 14 +- .../doc/images/output_static-checks.svg | 6 +- .../doc/images/output_static-checks.txt | 2 +- .../commands/kubernetes_commands.py | 2 +- .../commands/testing_commands.py | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 1 - .../prepare_providers/provider_packages.py | 4 +- .../utils/docker_command_utils.py | 3 + .../src/airflow_breeze/utils/packages.py | 4 +- .../src/airflow_breeze/utils/path_utils.py | 8 +- .../utils/provider_dependencies.py | 4 +- .../utils/publish_docs_helpers.py | 24 +- .../src/airflow_breeze/utils/run_tests.py | 35 +- .../airflow_breeze/utils/selective_checks.py | 36 +- dev/breeze/tests/test_packages.py | 17 +- .../tests/test_provider_documentation.py | 2 +- .../tests/test_pytest_args_for_test_types.py | 28 +- dev/breeze/tests/test_run_test_args.py | 18 +- dev/breeze/tests/test_selective_checks.py | 86 +- dev/example_dags/update_example_dags_paths.py | 8 +- dev/perf/scheduler_dag_execution_timing.py | 2 +- .../tests_common/__init__.py | 19 +- .../tests_common/_internals}/__init__.py | 0 .../_internals/capture_warnings.py | 0 .../_internals/forbidden_warnings.py | 21 +- dev/tests_common/pyproject.toml | 38 + dev/tests_common/pytest_plugin.py | 1436 +++++++++++++++++ .../tests_common}/test_utils/README.md | 10 +- .../tests_common/test_utils}/__init__.py | 5 + .../test_utils/api_connexion_utils.py | 9 +- .../tests_common}/test_utils/asserts.py | 0 .../test_utils/azure_system_helpers.py | 5 +- .../tests_common}/test_utils/compat.py | 0 .../tests_common}/test_utils/config.py | 0 {tests => dev/tests_common}/test_utils/db.py | 31 +- .../tests_common}/test_utils/decorators.py | 0 .../tests_common}/test_utils/fake_datetime.py | 0 .../test_utils/gcp_system_helpers.py | 32 +- .../tests_common}/test_utils/get_all_tests.py | 0 .../tests_common}/test_utils/hdfs_utils.py | 0 .../test_utils/logging_command_executor.py | 0 .../tests_common}/test_utils/mapping.py | 0 .../tests_common}/test_utils/mock_executor.py | 0 .../test_utils/mock_operators.py | 3 +- .../tests_common}/test_utils/mock_plugins.py | 0 .../test_utils/mock_security_manager.py | 2 +- .../test_utils/operators}/__init__.py | 0 .../operators/postgres_local_executor.cfg | 0 .../tests_common/test_utils/perf}/__init__.py | 0 .../test_utils/perf/perf_kit/__init__.py | 0 .../test_utils/perf/perf_kit/memory.py | 0 .../test_utils/perf/perf_kit/python.py | 0 .../perf/perf_kit/repeat_and_time.py | 0 .../test_utils/perf/perf_kit/sqlalchemy.py | 0 .../tests_common}/test_utils/permissions.py | 0 .../tests_common}/test_utils/providers.py | 0 .../remote_user_api_auth_backend.py | 0 .../test_utils/reset_warning_registry.py | 0 .../test_utils/salesforce_system_helpers.py | 0 .../test_utils/sftp_system_helpers.py | 0 .../tests_common/test_utils/system_tests.py | 0 .../test_utils/system_tests_class.py | 5 +- .../tests_common}/test_utils/terraform.py | 2 +- .../tests_common}/test_utils/timetables.py | 0 .../tests_common/test_utils}/watcher.py | 0 {tests => dev/tests_common}/test_utils/www.py | 0 .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators/airbyte.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators/analyticdb_spark.rst | 2 +- .../operators/oss.rst | 2 +- .../changelog.rst | 2 +- .../example-dags.rst | 2 +- .../apache-airflow-providers-amazon/index.rst | 2 +- .../operators/appflow.rst | 12 +- .../operators/athena/athena_boto.rst | 6 +- .../operators/athena/athena_sql.rst | 4 +- .../operators/batch.rst | 10 +- .../operators/bedrock.rst | 34 +- .../operators/cloudformation.rst | 8 +- .../operators/comprehend.rst | 8 +- .../operators/datasync.rst | 6 +- .../operators/dms.rst | 12 +- .../operators/dynamodb.rst | 4 +- .../operators/ec2.rst | 14 +- .../operators/ecs.rst | 20 +- .../operators/eks.rst | 26 +- .../operators/emr/emr.rst | 20 +- .../operators/emr/emr_eks.rst | 8 +- .../operators/emr/emr_serverless.rst | 12 +- .../operators/eventbridge.rst | 8 +- .../operators/glue.rst | 20 +- .../operators/glue_databrew.rst | 2 +- .../operators/kinesis_analytics.rst | 10 +- .../operators/lambda.rst | 6 +- .../operators/neptune.rst | 4 +- .../operators/opensearchserverless.rst | 2 +- .../operators/quicksight.rst | 4 +- .../operators/rds.rst | 28 +- .../operators/redshift/redshift_cluster.rst | 14 +- .../operators/redshift/redshift_data.rst | 4 +- .../operators/redshift/redshift_sql.rst | 2 +- .../operators/s3/glacier.rst | 6 +- .../operators/s3/s3.rst | 40 +- .../operators/sagemaker.rst | 46 +- .../operators/sns.rst | 2 +- .../operators/sqs.rst | 4 +- .../operators/step_functions.rst | 6 +- .../transfer/azure_blob_to_s3.rst | 2 +- .../transfer/dynamodb_to_s3.rst | 8 +- .../transfer/ftp_to_s3.rst | 2 +- .../transfer/gcs_to_s3.rst | 2 +- .../transfer/glacier_to_gcs.rst | 2 +- .../transfer/google_api_to_s3.rst | 6 +- .../transfer/hive_to_dynamodb.rst | 2 +- .../transfer/http_to_s3.rst | 2 +- .../transfer/imap_attachment_to_s3.rst | 2 +- .../transfer/local_to_s3.rst | 2 +- .../transfer/mongo_to_s3.rst | 2 +- .../transfer/redshift_to_s3.rst | 2 +- .../transfer/s3_to_dynamodb.rst | 4 +- .../transfer/s3_to_ftp.rst | 2 +- .../transfer/s3_to_redshift.rst | 4 +- .../transfer/s3_to_sftp.rst | 2 +- .../transfer/s3_to_sql.rst | 4 +- .../transfer/salesforce_to_s3.rst | 2 +- .../transfer/sftp_to_s3.rst | 2 +- .../transfer/sql_to_s3.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 26 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators/index.rst | 4 +- .../sensors.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 4 +- .../changelog.rst | 2 +- .../decorators/pyspark.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 6 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../operators/index.rst | 8 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-asana/index.rst | 4 +- .../operators/asana.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../kubernetes_executor.rst | 6 +- .../operators.rst | 20 +- .../changelog.rst | 2 +- .../apache-airflow-providers-cohere/index.rst | 2 +- .../operators/embedding.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 6 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators/copy_into.rst | 2 +- .../operators/jobs_create.rst | 6 +- .../operators/notebook.rst | 4 +- .../operators/repos_create.rst | 2 +- .../operators/repos_delete.rst | 2 +- .../operators/repos_update.rst | 2 +- .../operators/sql.rst | 16 +- .../operators/submit_run.rst | 4 +- .../operators/task.rst | 4 +- .../operators/workflow.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 12 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 8 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../decorators/docker.rst | 2 +- .../apache-airflow-providers-docker/index.rst | 4 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../connections/elasticsearch.rst | 2 +- .../hooks/elasticsearch_python_hook.rst | 2 +- .../hooks/elasticsearch_sql_hook.rst | 2 +- .../index.rst | 2 +- .../changelog.rst | 2 +- .../auth-manager/access-control.rst | 8 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-ftp/index.rst | 6 +- .../operators/index.rst | 8 +- .../changelog.rst | 2 +- .../apache-airflow-providers-github/index.rst | 4 +- .../operators/index.rst | 8 +- .../changelog.rst | 2 +- .../example-dags.rst | 10 +- .../apache-airflow-providers-google/index.rst | 2 +- .../operators/ads.rst | 4 +- .../operators/cloud/automl.rst | 31 +- .../operators/cloud/bigquery.rst | 74 +- .../operators/cloud/bigquery_dts.rst | 10 +- .../operators/cloud/bigtable.rst | 14 +- .../operators/cloud/cloud_batch.rst | 12 +- .../operators/cloud/cloud_build.rst | 42 +- .../operators/cloud/cloud_composer.rst | 30 +- .../operators/cloud/cloud_memorystore.rst | 26 +- .../cloud/cloud_memorystore_memcached.rst | 14 +- .../operators/cloud/cloud_run.rst | 24 +- .../operators/cloud/cloud_sql.rst | 70 +- .../cloud/cloud_storage_transfer_service.rst | 50 +- .../operators/cloud/compute.rst | 66 +- .../operators/cloud/compute_ssh.rst | 4 +- .../operators/cloud/data_loss_prevention.rst | 22 +- .../operators/cloud/datacatalog.rst | 64 +- .../operators/cloud/dataflow.rst | 50 +- .../operators/cloud/dataform.rst | 34 +- .../operators/cloud/datafusion.rst | 26 +- .../operators/cloud/datapipeline.rst | 4 +- .../operators/cloud/dataplex.rst | 66 +- .../operators/cloud/dataprep.rst | 14 +- .../operators/cloud/dataproc.rst | 86 +- .../operators/cloud/dataproc_metastore.rst | 30 +- .../operators/cloud/datastore.rst | 26 +- .../operators/cloud/functions.rst | 16 +- .../operators/cloud/gcs.rst | 30 +- .../operators/cloud/index.rst | 2 +- .../operators/cloud/kubernetes_engine.rst | 38 +- .../operators/cloud/life_sciences.rst | 6 +- .../operators/cloud/looker.rst | 4 +- .../operators/cloud/mlengine.rst | 22 +- .../operators/cloud/natural_language.rst | 20 +- .../operators/cloud/pubsub.rst | 20 +- .../operators/cloud/spanner.rst | 26 +- .../operators/cloud/speech_to_text.rst | 8 +- .../operators/cloud/stackdriver.rst | 20 +- .../operators/cloud/tasks.rst | 26 +- .../operators/cloud/text_to_speech.rst | 8 +- .../operators/cloud/translate.rst | 6 +- .../operators/cloud/translate_speech.rst | 6 +- .../operators/cloud/vertex_ai.rst | 116 +- .../operators/cloud/video_intelligence.rst | 24 +- .../operators/cloud/vision.rst | 158 +- .../operators/cloud/workflows.rst | 22 +- .../operators/firebase/firestore.rst | 2 +- .../operators/leveldb/leveldb.rst | 2 +- .../marketing_platform/analytics_admin.rst | 14 +- .../marketing_platform/campaign_manager.rst | 14 +- .../marketing_platform/display_video.rst | 20 +- .../marketing_platform/search_ads.rst | 10 +- .../operators/suite/sheets.rst | 4 +- .../operators/transfer/azure_blob_to_gcs.rst | 2 +- .../transfer/azure_fileshare_to_gcs.rst | 2 +- .../transfer/bigquery_to_bigquery.rst | 2 +- .../operators/transfer/bigquery_to_gcs.rst | 2 +- .../operators/transfer/bigquery_to_mssql.rst | 2 +- .../operators/transfer/bigquery_to_mysql.rst | 2 +- .../transfer/bigquery_to_postgres.rst | 4 +- .../operators/transfer/calendar_to_gcs.rst | 2 +- .../transfer/facebook_ads_to_gcs.rst | 2 +- .../operators/transfer/gcs_to_bigquery.rst | 4 +- .../operators/transfer/gcs_to_gcs.rst | 20 +- .../operators/transfer/gcs_to_gdrive.rst | 8 +- .../operators/transfer/gcs_to_local.rst | 2 +- .../operators/transfer/gcs_to_sftp.rst | 8 +- .../operators/transfer/gcs_to_sheets.rst | 2 +- .../operators/transfer/gdrive_to_gcs.rst | 2 +- .../operators/transfer/gdrive_to_local.rst | 2 +- .../operators/transfer/local_to_drive.rst | 2 +- .../operators/transfer/local_to_gcs.rst | 2 +- .../operators/transfer/mssql_to_gcs.rst | 2 +- .../operators/transfer/mysql_to_gcs.rst | 2 +- .../operators/transfer/oracle_to_gcs.rst | 2 +- .../operators/transfer/postgres_to_gcs.rst | 2 +- .../operators/transfer/presto_to_gcs.rst | 10 +- .../operators/transfer/s3_to_gcs.rst | 4 +- .../operators/transfer/salesforce_to_gcs.rst | 2 +- .../operators/transfer/sftp_to_gcs.rst | 8 +- .../operators/transfer/sheets_to_gcs.rst | 2 +- .../operators/transfer/sql_to_sheets.rst | 2 +- .../operators/transfer/trino_to_gcs.rst | 10 +- .../sensors/google-cloud-tasks.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-http/index.rst | 4 +- .../operators.rst | 18 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators/index.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-jdbc/index.rst | 4 +- .../operators.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators/adf_run_pipeline.rst | 10 +- .../operators/adls.rst | 6 +- .../operators/asb.rst | 20 +- .../operators/azure_synapse.rst | 4 +- .../operators/batch.rst | 2 +- .../operators/msgraph.rst | 12 +- .../sensors/cosmos_document_sensor.rst | 2 +- .../sensors/msgraph.rst | 2 +- .../sensors/wasb_sensors.rst | 4 +- .../transfer/local_to_adls.rst | 2 +- .../transfer/local_to_wasb.rst | 2 +- .../transfer/s3_to_wasb.rst | 2 +- .../transfer/sftp_to_wasb.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 12 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 4 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-mysql/index.rst | 4 +- .../operators.rst | 4 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-neo4j/index.rst | 4 +- .../operators/neo4j.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../apache-airflow-providers-openai/index.rst | 2 +- .../operators/openai.rst | 4 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../guides/developer.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 2 +- .../operators/opensearch.rst | 6 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../notifications/opsgenie_notifier.rst | 2 +- .../operators/opsgenie_alert.rst | 6 +- .../changelog.rst | 2 +- .../operators/index.rst | 6 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 6 +- .../changelog.rst | 2 +- .../index.rst | 2 +- .../operators/pgvector.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 2 +- .../operators/pinecone.rst | 6 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../postgres_operator_howto_guide.rst | 6 +- .../changelog.rst | 2 +- .../apache-airflow-providers-presto/index.rst | 4 +- .../operators/transfer/gcs_to_presto.rst | 2 +- .../changelog.rst | 2 +- .../apache-airflow-providers-qdrant/index.rst | 2 +- .../operators/qdrant.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-redis/index.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators/bulk.rst | 8 +- .../operators/salesforce_apex_rest.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-samba/index.rst | 2 +- .../transfer/gcs_to_samba.rst | 8 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-sftp/index.rst | 2 +- .../sensors/sftp_sensor.rst | 6 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-slack/index.rst | 4 +- .../operators/slack_api.rst | 8 +- .../operators/slack_webhook.rst | 4 +- .../operators/sql_to_slack.rst | 2 +- .../operators/sql_to_slack_webhook.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../decorators/snowpark.rst | 2 +- .../index.rst | 4 +- .../operators/copy_into_snowflake.rst | 2 +- .../operators/snowflake.rst | 4 +- .../operators/snowpark.rst | 2 +- .../changelog.rst | 2 +- .../apache-airflow-providers-sqlite/index.rst | 4 +- .../operators.rst | 4 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 4 +- .../operators.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 2 +- .../operators/azure_blob_to_teradata.rst | 14 +- .../operators/compute_cluster.rst | 8 +- .../operators/s3_to_teradata.rst | 8 +- .../operators/teradata.rst | 26 +- .../operators/teradata_to_teradata.rst | 4 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-trino/index.rst | 4 +- .../operators/transfer/gcs_to_trino.rst | 2 +- .../operators/trino.rst | 2 +- .../changelog.rst | 2 +- .../changelog.rst | 2 +- .../index.rst | 2 +- .../operators/weaviate.rst | 8 +- .../changelog.rst | 2 +- .../apache-airflow-providers-yandex/index.rst | 4 +- .../operators/dataproc.rst | 2 +- .../operators/yq.rst | 2 +- .../changelog.rst | 2 +- docs/apache-airflow-providers-ydb/index.rst | 4 +- .../operators/ydb_operator_howto_guide.rst | 4 +- .../changelog.rst | 2 +- .../index.rst | 4 +- docs/apache-airflow/tutorial/taskflow.rst | 4 +- docs/build_docs.py | 5 +- docs/conf.py | 101 +- docs/exts/docs_build/code_utils.py | 1 - docs/exts/exampleinclude.py | 13 +- docs/exts/provider_init_hack.py | 56 - docs/exts/provider_yaml_utils.py | 16 +- docs/exts/providers_extensions.py | 27 +- generated/provider_dependencies.json | 7 +- providers/.gitignore | 7 + {tests/test_utils => providers}/__init__.py | 13 +- providers/pyproject.toml | 98 ++ providers/src/airflow/providers/.gitignore | 1 + .../MANAGING_PROVIDERS_LIFECYCLE.rst | 12 +- .../airbyte/.latest-doc-only-change.txt | 0 .../airflow}/providers/airbyte/CHANGELOG.rst | 0 .../airflow}/providers/airbyte/__init__.py | 0 .../providers/airbyte/hooks/__init__.py | 0 .../providers/airbyte/hooks/airbyte.py | 0 .../providers/airbyte/operators/__init__.py | 0 .../providers/airbyte/operators/airbyte.py | 0 .../airflow}/providers/airbyte/provider.yaml | 0 .../providers/airbyte/sensors}/__init__.py | 0 .../providers/airbyte/sensors/airbyte.py | 0 .../providers/airbyte/triggers}/__init__.py | 0 .../providers/airbyte/triggers/airbyte.py | 0 .../alibaba/.latest-doc-only-change.txt | 0 .../airflow}/providers/alibaba/CHANGELOG.rst | 0 .../airflow}/providers/alibaba/__init__.py | 0 .../providers/alibaba/cloud/__init__.py | 0 .../providers/alibaba/cloud/hooks/__init__.py | 0 .../alibaba/cloud/hooks/analyticdb_spark.py | 0 .../providers/alibaba/cloud/hooks/oss.py | 0 .../providers/alibaba/cloud/log}/__init__.py | 0 .../alibaba/cloud/log/oss_task_handler.py | 0 .../alibaba/cloud/operators}/__init__.py | 0 .../cloud/operators/analyticdb_spark.py | 0 .../providers/alibaba/cloud/operators/oss.py | 0 .../alibaba/cloud/sensors}/__init__.py | 0 .../alibaba/cloud/sensors/analyticdb_spark.py | 0 .../alibaba/cloud/sensors/oss_key.py | 0 .../airflow}/providers/alibaba/provider.yaml | 0 .../amazon/.latest-doc-only-change.txt | 0 .../airflow}/providers/amazon/CHANGELOG.rst | 0 .../src/airflow}/providers/amazon/__init__.py | 0 .../airflow/providers/amazon/aws}/__init__.py | 0 .../providers/amazon/aws/assets}/__init__.py | 0 .../providers/amazon/aws/assets/s3.py | 0 .../amazon/aws/auth_manager}/__init__.py | 0 .../amazon/aws/auth_manager/avp}/__init__.py | 0 .../amazon/aws/auth_manager/avp/entities.py | 0 .../amazon/aws/auth_manager/avp/facade.py | 0 .../amazon/aws/auth_manager/avp/schema.json | 0 .../aws/auth_manager/aws_auth_manager.py | 0 .../amazon/aws/auth_manager/cli}/__init__.py | 0 .../aws/auth_manager/cli/avp_commands.py | 0 .../amazon/aws/auth_manager/cli/definition.py | 0 .../amazon/aws/auth_manager/constants.py | 0 .../security_manager}/__init__.py | 0 .../aws_security_manager_override.py | 0 .../providers/amazon/aws/auth_manager/user.py | 0 .../aws/auth_manager/views}/__init__.py | 0 .../amazon/aws/auth_manager/views/auth.py | 0 .../providers/amazon/aws/exceptions.py | 0 .../providers/amazon/aws/executors/Dockerfile | 0 .../amazon/aws/executors}/__init__.py | 0 .../amazon/aws/executors/batch/__init__.py | 0 .../aws/executors/batch/batch_executor.py | 0 .../executors/batch/batch_executor_config.py | 0 .../amazon/aws/executors/batch/boto_schema.py | 0 .../amazon/aws/executors/batch/utils.py | 0 .../amazon/aws/executors/ecs/__init__.py | 0 .../amazon/aws/executors/ecs/boto_schema.py | 0 .../amazon/aws/executors/ecs/ecs_executor.py | 0 .../aws/executors/ecs/ecs_executor_config.py | 0 .../amazon/aws/executors/ecs/utils.py | 0 .../amazon/aws/executors/utils}/__init__.py | 0 .../aws/executors/utils/base_config_keys.py | 0 .../utils/exponential_backoff_retry.py | 0 .../providers/amazon/aws/fs}/__init__.py | 0 .../airflow}/providers/amazon/aws/fs/s3.py | 0 .../providers/amazon/aws/hooks}/__init__.py | 0 .../providers/amazon/aws/hooks/appflow.py | 0 .../providers/amazon/aws/hooks/athena.py | 0 .../providers/amazon/aws/hooks/athena_sql.py | 0 .../providers/amazon/aws/hooks/base_aws.py | 0 .../amazon/aws/hooks/batch_client.py | 0 .../amazon/aws/hooks/batch_waiters.json | 0 .../amazon/aws/hooks/batch_waiters.py | 0 .../providers/amazon/aws/hooks/bedrock.py | 0 .../providers/amazon/aws/hooks/chime.py | 0 .../amazon/aws/hooks/cloud_formation.py | 0 .../providers/amazon/aws/hooks/comprehend.py | 0 .../providers/amazon/aws/hooks/datasync.py | 0 .../providers/amazon/aws/hooks/dms.py | 0 .../providers/amazon/aws/hooks/dynamodb.py | 0 .../providers/amazon/aws/hooks/ec2.py | 0 .../providers/amazon/aws/hooks/ecr.py | 0 .../providers/amazon/aws/hooks/ecs.py | 0 .../providers/amazon/aws/hooks/eks.py | 0 .../hooks/elasticache_replication_group.py | 0 .../providers/amazon/aws/hooks/emr.py | 0 .../providers/amazon/aws/hooks/eventbridge.py | 0 .../providers/amazon/aws/hooks/glacier.py | 0 .../providers/amazon/aws/hooks/glue.py | 0 .../amazon/aws/hooks/glue_catalog.py | 0 .../amazon/aws/hooks/glue_crawler.py | 0 .../amazon/aws/hooks/glue_databrew.py | 0 .../providers/amazon/aws/hooks/kinesis.py | 0 .../amazon/aws/hooks/kinesis_analytics.py | 0 .../amazon/aws/hooks/lambda_function.py | 0 .../providers/amazon/aws/hooks/logs.py | 0 .../providers/amazon/aws/hooks/neptune.py | 0 .../amazon/aws/hooks/opensearch_serverless.py | 0 .../providers/amazon/aws/hooks/quicksight.py | 0 .../providers/amazon/aws/hooks/rds.py | 0 .../amazon/aws/hooks/redshift_cluster.py | 0 .../amazon/aws/hooks/redshift_data.py | 0 .../amazon/aws/hooks/redshift_sql.py | 0 .../airflow}/providers/amazon/aws/hooks/s3.py | 0 .../providers/amazon/aws/hooks/sagemaker.py | 0 .../amazon/aws/hooks/secrets_manager.py | 0 .../providers/amazon/aws/hooks/ses.py | 0 .../providers/amazon/aws/hooks/sns.py | 0 .../providers/amazon/aws/hooks/sqs.py | 0 .../providers/amazon/aws/hooks/ssm.py | 0 .../amazon/aws/hooks/step_function.py | 0 .../providers/amazon/aws/hooks/sts.py | 0 .../amazon/aws/hooks/verified_permissions.py | 0 .../providers/amazon/aws/links}/__init__.py | 0 .../providers/amazon/aws/links/athena.py | 0 .../providers/amazon/aws/links/base_aws.py | 0 .../providers/amazon/aws/links/batch.py | 0 .../providers/amazon/aws/links/emr.py | 0 .../providers/amazon/aws/links/glue.py | 0 .../providers/amazon/aws/links/logs.py | 0 .../amazon/aws/links/step_function.py | 0 .../providers/amazon/aws/log}/__init__.py | 0 .../amazon/aws/log/cloudwatch_task_handler.py | 0 .../amazon/aws/log/s3_task_handler.py | 0 .../amazon/aws/notifications}/__init__.py | 0 .../amazon/aws/notifications/chime.py | 0 .../providers/amazon/aws/notifications/sns.py | 0 .../providers/amazon/aws/notifications/sqs.py | 0 .../amazon/aws/operators}/__init__.py | 0 .../providers/amazon/aws/operators/appflow.py | 0 .../providers/amazon/aws/operators/athena.py | 0 .../amazon/aws/operators/base_aws.py | 0 .../providers/amazon/aws/operators/batch.py | 0 .../providers/amazon/aws/operators/bedrock.py | 0 .../amazon/aws/operators/cloud_formation.py | 0 .../amazon/aws/operators/comprehend.py | 0 .../amazon/aws/operators/datasync.py | 0 .../providers/amazon/aws/operators/dms.py | 0 .../providers/amazon/aws/operators/ec2.py | 0 .../providers/amazon/aws/operators/ecs.py | 0 .../providers/amazon/aws/operators/eks.py | 0 .../providers/amazon/aws/operators/emr.py | 0 .../amazon/aws/operators/eventbridge.py | 0 .../providers/amazon/aws/operators/glacier.py | 0 .../providers/amazon/aws/operators/glue.py | 0 .../amazon/aws/operators/glue_crawler.py | 0 .../amazon/aws/operators/glue_databrew.py | 0 .../amazon/aws/operators/kinesis_analytics.py | 0 .../amazon/aws/operators/lambda_function.py | 0 .../providers/amazon/aws/operators/neptune.py | 0 .../amazon/aws/operators/quicksight.py | 0 .../providers/amazon/aws/operators/rds.py | 0 .../amazon/aws/operators/redshift_cluster.py | 0 .../amazon/aws/operators/redshift_data.py | 0 .../providers/amazon/aws/operators/s3.py | 0 .../amazon/aws/operators/sagemaker.py | 0 .../providers/amazon/aws/operators/sns.py | 0 .../providers/amazon/aws/operators/sqs.py | 0 .../amazon/aws/operators/step_function.py | 0 .../providers/amazon/aws/secrets}/__init__.py | 0 .../amazon/aws/secrets/secrets_manager.py | 0 .../amazon/aws/secrets/systems_manager.py | 0 .../providers/amazon/aws/sensors}/__init__.py | 0 .../providers/amazon/aws/sensors/athena.py | 0 .../providers/amazon/aws/sensors/base_aws.py | 0 .../providers/amazon/aws/sensors/batch.py | 0 .../providers/amazon/aws/sensors/bedrock.py | 0 .../amazon/aws/sensors/cloud_formation.py | 0 .../amazon/aws/sensors/comprehend.py | 0 .../providers/amazon/aws/sensors/dms.py | 0 .../providers/amazon/aws/sensors/dynamodb.py | 0 .../providers/amazon/aws/sensors/ec2.py | 0 .../providers/amazon/aws/sensors/ecs.py | 0 .../providers/amazon/aws/sensors/eks.py | 0 .../providers/amazon/aws/sensors/emr.py | 0 .../providers/amazon/aws/sensors/glacier.py | 0 .../providers/amazon/aws/sensors/glue.py | 0 .../aws/sensors/glue_catalog_partition.py | 0 .../amazon/aws/sensors/glue_crawler.py | 0 .../amazon/aws/sensors/kinesis_analytics.py | 0 .../amazon/aws/sensors/lambda_function.py | 0 .../aws/sensors/opensearch_serverless.py | 0 .../amazon/aws/sensors/quicksight.py | 0 .../providers/amazon/aws/sensors/rds.py | 0 .../amazon/aws/sensors/redshift_cluster.py | 0 .../providers/amazon/aws/sensors/s3.py | 0 .../providers/amazon/aws/sensors/sagemaker.py | 0 .../providers/amazon/aws/sensors/sqs.py | 0 .../amazon/aws/sensors/step_function.py | 0 .../amazon/aws/transfers}/__init__.py | 0 .../amazon/aws/transfers/azure_blob_to_s3.py | 0 .../providers/amazon/aws/transfers/base.py | 0 .../amazon/aws/transfers/dynamodb_to_s3.py | 0 .../amazon/aws/transfers/exasol_to_s3.py | 0 .../amazon/aws/transfers/ftp_to_s3.py | 0 .../amazon/aws/transfers/gcs_to_s3.py | 0 .../amazon/aws/transfers/glacier_to_gcs.py | 0 .../amazon/aws/transfers/google_api_to_s3.py | 0 .../amazon/aws/transfers/hive_to_dynamodb.py | 0 .../amazon/aws/transfers/http_to_s3.py | 0 .../aws/transfers/imap_attachment_to_s3.py | 0 .../amazon/aws/transfers/local_to_s3.py | 0 .../amazon/aws/transfers/mongo_to_s3.py | 0 .../amazon/aws/transfers/redshift_to_s3.py | 0 .../amazon/aws/transfers/s3_to_dynamodb.py | 0 .../amazon/aws/transfers/s3_to_ftp.py | 0 .../amazon/aws/transfers/s3_to_redshift.py | 0 .../amazon/aws/transfers/s3_to_sftp.py | 0 .../amazon/aws/transfers/s3_to_sql.py | 0 .../amazon/aws/transfers/salesforce_to_s3.py | 0 .../amazon/aws/transfers/sftp_to_s3.py | 0 .../amazon/aws/transfers/sql_to_s3.py | 0 .../providers/amazon/aws/triggers/README.md | 0 .../providers/amazon/aws/triggers/__init__.py | 0 .../providers/amazon/aws/triggers/athena.py | 0 .../providers/amazon/aws/triggers/base.py | 0 .../providers/amazon/aws/triggers/batch.py | 0 .../providers/amazon/aws/triggers/bedrock.py | 0 .../amazon/aws/triggers/comprehend.py | 0 .../providers/amazon/aws/triggers/ec2.py | 0 .../providers/amazon/aws/triggers/ecs.py | 12 +- .../providers/amazon/aws/triggers/eks.py | 0 .../providers/amazon/aws/triggers/emr.py | 0 .../providers/amazon/aws/triggers/glue.py | 0 .../amazon/aws/triggers/glue_crawler.py | 0 .../amazon/aws/triggers/glue_databrew.py | 0 .../amazon/aws/triggers/kinesis_analytics.py | 0 .../amazon/aws/triggers/lambda_function.py | 0 .../providers/amazon/aws/triggers/neptune.py | 0 .../aws/triggers/opensearch_serverless.py | 0 .../providers/amazon/aws/triggers/rds.py | 0 .../amazon/aws/triggers/redshift_cluster.py | 0 .../amazon/aws/triggers/redshift_data.py | 0 .../providers/amazon/aws/triggers/s3.py | 0 .../amazon/aws/triggers/sagemaker.py | 0 .../providers/amazon/aws/triggers/sqs.py | 0 .../amazon/aws/triggers/step_function.py | 0 .../providers/amazon/aws/utils/__init__.py | 0 .../utils/asset_compat_lineage_collector.py | 0 .../amazon/aws/utils/connection_wrapper.py | 0 .../amazon/aws/utils/eks_get_token.py | 0 .../providers/amazon/aws/utils/emailer.py | 0 .../providers/amazon/aws/utils/identifiers.py | 0 .../providers/amazon/aws/utils/mixins.py | 0 .../providers/amazon/aws/utils/openlineage.py | 0 .../providers/amazon/aws/utils/rds.py | 0 .../providers/amazon/aws/utils/redshift.py | 0 .../providers/amazon/aws/utils/sagemaker.py | 0 .../providers/amazon/aws/utils/sqs.py | 0 .../providers/amazon/aws/utils/suppress.py | 0 .../providers/amazon/aws/utils/tags.py | 0 .../amazon/aws/utils/task_log_fetcher.py | 0 .../providers/amazon/aws/utils/waiter.py | 0 .../amazon/aws/utils/waiter_with_logging.py | 0 .../providers/amazon/aws/waiters/README.md | 0 .../providers/amazon/aws/waiters}/__init__.py | 0 .../providers/amazon/aws/waiters/appflow.json | 0 .../providers/amazon/aws/waiters/athena.json | 0 .../amazon/aws/waiters/base_waiter.py | 0 .../providers/amazon/aws/waiters/batch.json | 0 .../amazon/aws/waiters/bedrock-agent.json | 0 .../providers/amazon/aws/waiters/bedrock.json | 0 .../amazon/aws/waiters/comprehend.json | 0 .../amazon/aws/waiters/databrew.json | 0 .../amazon/aws/waiters/dynamodb.json | 0 .../providers/amazon/aws/waiters/ecs.json | 0 .../providers/amazon/aws/waiters/eks.json | 0 .../amazon/aws/waiters/emr-containers.json | 0 .../amazon/aws/waiters/emr-serverless.json | 0 .../providers/amazon/aws/waiters/emr.json | 0 .../providers/amazon/aws/waiters/glue.json | 0 .../aws/waiters/kinesisanalyticsv2.json | 0 .../providers/amazon/aws/waiters/neptune.json | 0 .../aws/waiters/opensearchserverless.json | 0 .../providers/amazon/aws/waiters/rds.json | 0 .../amazon/aws/waiters/redshift.json | 0 .../amazon/aws/waiters/sagemaker.json | 0 .../amazon/aws/waiters/stepfunctions.json | 0 .../airflow}/providers/amazon/provider.yaml | 0 .../apache/beam/.latest-doc-only-change.txt | 0 .../providers/apache/beam/CHANGELOG.rst | 0 .../airflow}/providers/apache/beam/README.md | 0 .../providers/apache/beam/__init__.py | 0 .../providers/apache/beam/hooks/__init__.py | 0 .../providers/apache/beam/hooks/beam.py | 0 .../apache/beam/operators/__init__.py | 0 .../providers/apache/beam/operators/beam.py | 0 .../providers/apache/beam/provider.yaml | 0 .../apache/beam/triggers}/__init__.py | 0 .../providers/apache/beam/triggers/beam.py | 0 .../cassandra/.latest-doc-only-change.txt | 0 .../providers/apache/cassandra/CHANGELOG.rst | 0 .../providers/apache/cassandra/__init__.py | 0 .../apache/cassandra/hooks}/__init__.py | 0 .../apache/cassandra/hooks/cassandra.py | 0 .../providers/apache/cassandra/provider.yaml | 0 .../apache/cassandra/sensors}/__init__.py | 0 .../apache/cassandra/sensors/record.py | 0 .../apache/cassandra/sensors/table.py | 0 .../apache/drill/.latest-doc-only-change.txt | 0 .../providers/apache/drill/CHANGELOG.rst | 0 .../providers/apache/drill/__init__.py | 0 .../providers/apache/drill/hooks/__init__.py | 0 .../providers/apache/drill/hooks/drill.py | 0 .../apache/drill/operators/__init__.py | 0 .../providers/apache/drill/operators/drill.py | 0 .../providers/apache/drill/provider.yaml | 0 .../apache/druid/.latest-doc-only-change.txt | 0 .../providers/apache/druid/CHANGELOG.rst | 0 .../providers/apache/druid/__init__.py | 0 .../providers/apache/druid/hooks/__init__.py | 0 .../providers/apache/druid/hooks/druid.py | 0 .../apache/druid/operators/__init__.py | 0 .../providers/apache/druid/operators/druid.py | 0 .../apache/druid/operators/druid_check.py | 0 .../providers/apache/druid/provider.yaml | 0 .../apache/druid}/transfers/__init__.py | 0 .../apache/druid/transfers/hive_to_druid.py | 0 .../apache/flink/.latest-doc-only-change.txt | 0 .../providers/apache/flink/CHANGELOG.rst | 0 .../providers/apache/flink/__init__.py | 0 .../providers/apache/flink/hooks/__init__.py | 0 .../apache/flink/operators/__init__.py | 0 .../flink/operators/flink_kubernetes.py | 0 .../providers/apache/flink/provider.yaml | 0 .../apache/flink/sensors/__init__.py | 0 .../apache/flink/sensors/flink_kubernetes.py | 0 .../apache/hdfs/.latest-doc-only-change.txt | 0 .../providers/apache/hdfs/CHANGELOG.rst | 0 .../providers/apache/hdfs/__init__.py | 0 .../providers/apache/hdfs/hooks/__init__.py | 0 .../providers/apache/hdfs/hooks/hdfs.py | 0 .../providers/apache/hdfs/hooks/webhdfs.py | 0 .../providers/apache/hdfs/log}/__init__.py | 0 .../apache/hdfs/log/hdfs_task_handler.py | 0 .../providers/apache/hdfs/provider.yaml | 0 .../providers/apache/hdfs/sensors/__init__.py | 0 .../providers/apache/hdfs/sensors/hdfs.py | 0 .../providers/apache/hdfs/sensors/web_hdfs.py | 0 .../apache/hive/.latest-doc-only-change.txt | 0 .../providers/apache/hive/CHANGELOG.rst | 0 .../providers/apache/hive/__init__.py | 0 .../providers/apache/hive/hooks/__init__.py | 0 .../providers/apache/hive/hooks/hive.py | 10 +- .../providers/apache/hive/macros/__init__.py | 0 .../providers/apache/hive/macros/hive.py | 0 .../apache/hive/operators/__init__.py | 0 .../providers/apache/hive/operators/hive.py | 0 .../apache/hive/operators/hive_stats.py | 0 .../providers/apache/hive/plugins/__init__.py | 0 .../providers/apache/hive/plugins/hive.py | 0 .../providers/apache/hive/provider.yaml | 0 .../providers/apache/hive/sensors/__init__.py | 0 .../apache/hive/sensors/hive_partition.py | 0 .../hive/sensors/metastore_partition.py | 0 .../hive/sensors/named_hive_partition.py | 0 .../apache/hive/transfers}/__init__.py | 0 .../apache/hive/transfers/hive_to_mysql.py | 0 .../apache/hive/transfers/hive_to_samba.py | 0 .../apache/hive/transfers/mssql_to_hive.py | 0 .../apache/hive/transfers/mysql_to_hive.py | 0 .../apache/hive/transfers/s3_to_hive.py | 10 +- .../apache/hive/transfers/vertica_to_hive.py | 0 .../iceberg/.latest-doc-only-change.txt | 0 .../providers/apache/iceberg/CHANGELOG.rst | 0 .../providers/apache/iceberg/__init__.py | 0 .../apache/iceberg/hooks/__init__.py | 0 .../providers/apache/iceberg/hooks/iceberg.py | 0 .../providers/apache/iceberg/provider.yaml | 0 .../apache/impala/.latest-doc-only-change.txt | 0 .../providers/apache/impala/CHANGELOG.rst | 0 .../providers/apache/impala/__init__.py | 0 .../providers/apache/impala/hooks/__init__.py | 0 .../providers/apache/impala/hooks/impala.py | 0 .../providers/apache/impala/provider.yaml | 0 .../apache/kafka/.latest-doc-only-change.txt | 0 .../providers/apache/kafka/CHANGELOG.rst | 0 .../providers/apache/kafka/__init__.py | 0 .../providers/apache/kafka/hooks}/__init__.py | 0 .../providers/apache/kafka/hooks/base.py | 0 .../providers/apache/kafka/hooks/client.py | 0 .../providers/apache/kafka/hooks/consume.py | 0 .../providers/apache/kafka/hooks/produce.py | 0 .../apache/kafka/operators}/__init__.py | 0 .../apache/kafka/operators/consume.py | 0 .../apache/kafka/operators/produce.py | 0 .../providers/apache/kafka/provider.yaml | 0 .../apache/kafka/sensors}/__init__.py | 0 .../providers/apache/kafka/sensors/kafka.py | 0 .../apache/kafka/triggers}/__init__.py | 0 .../apache/kafka/triggers/await_message.py | 0 .../apache/kylin/.latest-doc-only-change.txt | 0 .../providers/apache/kylin/CHANGELOG.rst | 0 .../providers/apache/kylin/__init__.py | 0 .../providers/apache/kylin}/hooks/__init__.py | 0 .../providers/apache/kylin/hooks/kylin.py | 0 .../apache/kylin}/operators/__init__.py | 0 .../apache/kylin/operators/kylin_cube.py | 0 .../providers/apache/kylin/provider.yaml | 0 .../apache/livy/.latest-doc-only-change.txt | 0 .../providers/apache/livy/CHANGELOG.rst | 0 .../providers/apache/livy/__init__.py | 0 .../providers/apache/livy/hooks}/__init__.py | 0 .../providers/apache/livy/hooks/livy.py | 0 .../apache/livy/operators}/__init__.py | 0 .../providers/apache/livy/operators/livy.py | 0 .../providers/apache/livy/provider.yaml | 0 .../apache/livy/sensors}/__init__.py | 0 .../providers/apache/livy/sensors/livy.py | 0 .../apache/livy/triggers}/__init__.py | 0 .../providers/apache/livy/triggers/livy.py | 0 .../apache/pig/.latest-doc-only-change.txt | 0 .../providers/apache/pig/CHANGELOG.rst | 0 .../airflow}/providers/apache/pig/__init__.py | 0 .../providers/apache/pig/hooks/__init__.py | 0 .../providers/apache/pig/hooks/pig.py | 0 .../apache/pig/operators/__init__.py | 0 .../providers/apache/pig/operators/pig.py | 0 .../providers/apache/pig/provider.yaml | 0 .../apache/pinot/.latest-doc-only-change.txt | 0 .../providers/apache/pinot/CHANGELOG.rst | 0 .../providers/apache/pinot/__init__.py | 0 .../providers/apache/pinot/hooks/__init__.py | 0 .../providers/apache/pinot/hooks/pinot.py | 0 .../providers/apache/pinot/provider.yaml | 0 .../apache/spark/.latest-doc-only-change.txt | 0 .../providers/apache/spark/CHANGELOG.rst | 0 .../providers/apache/spark/__init__.py | 0 .../apache/spark/decorators/__init__.py | 0 .../apache/spark/decorators/pyspark.py | 0 .../providers/apache/spark/hooks/__init__.py | 0 .../apache/spark/hooks/spark_connect.py | 0 .../apache/spark/hooks/spark_jdbc.py | 0 .../apache/spark/hooks/spark_jdbc_script.py | 0 .../providers/apache/spark/hooks/spark_sql.py | 0 .../apache/spark/hooks/spark_submit.py | 0 .../apache/spark/operators/__init__.py | 0 .../apache/spark/operators/spark_jdbc.py | 0 .../apache/spark/operators/spark_sql.py | 0 .../apache/spark/operators/spark_submit.py | 0 .../providers/apache/spark/provider.yaml | 0 .../apprise/.latest-doc-only-change.txt | 0 .../airflow}/providers/apprise/CHANGELOG.rst | 0 .../airflow}/providers/apprise/__init__.py | 0 .../providers/apprise/hooks/__init__.py | 0 .../providers/apprise/hooks/apprise.py | 0 .../apprise/notifications/__init__.py | 0 .../apprise/notifications/apprise.py | 0 .../airflow}/providers/apprise/provider.yaml | 0 .../arangodb/.latest-doc-only-change.txt | 0 .../airflow}/providers/arangodb/CHANGELOG.rst | 0 .../airflow}/providers/arangodb/__init__.py | 0 .../arangodb/example_dags}/__init__.py | 0 .../arangodb/example_dags/example_arangodb.py | 0 .../providers/arangodb/hooks}/__init__.py | 0 .../providers/arangodb/hooks/arangodb.py | 0 .../providers/arangodb/operators}/__init__.py | 0 .../providers/arangodb/operators/arangodb.py | 0 .../airflow}/providers/arangodb/provider.yaml | 0 .../providers/arangodb/sensors}/__init__.py | 0 .../providers/arangodb/sensors/arangodb.py | 0 .../asana/.latest-doc-only-change.txt | 0 .../airflow}/providers/asana/CHANGELOG.rst | 0 .../src/airflow}/providers/asana/README.md | 0 .../src/airflow}/providers/asana/__init__.py | 0 .../providers/asana/hooks/__init__.py | 0 .../airflow}/providers/asana/hooks/asana.py | 0 .../providers/asana/operators/__init__.py | 0 .../providers/asana/operators/asana_tasks.py | 0 .../airflow}/providers/asana/provider.yaml | 0 .../jira/.latest-doc-only-change.txt | 0 .../providers/atlassian/jira/CHANGELOG.rst | 0 .../providers/atlassian/jira/__init__.py | 0 .../atlassian/jira/hooks}/__init__.py | 0 .../providers/atlassian/jira/hooks/jira.py | 0 .../atlassian/jira/notifications}/__init__.py | 0 .../atlassian/jira/notifications/jira.py | 0 .../atlassian/jira/operators}/__init__.py | 0 .../atlassian/jira/operators/jira.py | 0 .../providers/atlassian/jira/provider.yaml | 0 .../atlassian/jira/sensors}/__init__.py | 0 .../providers/atlassian/jira/sensors/jira.py | 0 .../celery/.latest-doc-only-change.txt | 0 .../airflow}/providers/celery/CHANGELOG.rst | 0 .../src/airflow}/providers/celery/__init__.py | 0 .../airflow/providers/celery/cli}/__init__.py | 0 .../providers/celery/cli/celery_command.py | 0 .../providers/celery/executors/__init__.py | 0 .../celery/executors/celery_executor.py | 0 .../celery/executors/celery_executor_utils.py | 0 .../executors/celery_kubernetes_executor.py | 0 .../celery/executors/default_celery.py | 0 .../airflow}/providers/celery/provider.yaml | 0 .../providers/celery/sensors/__init__.py | 0 .../providers/celery/sensors/celery_queue.py | 0 .../cloudant/.latest-doc-only-change.txt | 0 .../airflow}/providers/cloudant/CHANGELOG.rst | 0 .../airflow}/providers/cloudant/__init__.py | 0 .../providers/cloudant/cloudant_fake.py | 0 .../providers/cloudant/hooks/__init__.py | 0 .../providers/cloudant/hooks/cloudant.py | 0 .../airflow}/providers/cloudant/provider.yaml | 0 .../kubernetes/.latest-doc-only-change.txt | 0 .../providers/cncf/kubernetes/CHANGELOG.rst | 0 .../providers/cncf/kubernetes/__init__.py | 0 .../cncf/kubernetes/backcompat}/__init__.py | 0 .../backcompat/backwards_compat_converters.py | 0 .../providers/cncf/kubernetes/callbacks.py | 0 .../cncf/kubernetes/cli}/__init__.py | 0 .../cncf/kubernetes/cli/kubernetes_command.py | 0 .../cncf/kubernetes/decorators}/__init__.py | 0 .../cncf/kubernetes/decorators/kubernetes.py | 0 .../cncf/kubernetes/executors}/__init__.py | 0 .../executors/kubernetes_executor.py | 0 .../executors/kubernetes_executor_types.py | 0 .../executors/kubernetes_executor_utils.py | 0 .../executors/local_kubernetes_executor.py | 0 .../cncf/kubernetes/hooks}/__init__.py | 0 .../cncf/kubernetes/hooks/kubernetes.py | 0 .../providers/cncf/kubernetes/k8s_model.py | 0 .../providers/cncf/kubernetes/kube_client.py | 0 .../providers/cncf/kubernetes/kube_config.py | 0 .../__init__.py | 0 .../basic_template.yaml | 0 .../kubernetes/kubernetes_helper_functions.py | 0 .../cncf/kubernetes/operators}/__init__.py | 0 .../operators/custom_object_launcher.py | 0 .../cncf/kubernetes/operators/job.py | 0 .../kubernetes/operators/kubernetes_pod.py | 0 .../cncf/kubernetes/operators/pod.py | 0 .../cncf/kubernetes/operators/resource.py | 0 .../kubernetes/operators/spark_kubernetes.py | 0 .../cncf/kubernetes/pod_generator.py | 0 .../kubernetes/pod_generator_deprecated.py | 0 .../kubernetes/pod_launcher_deprecated.py | 0 .../pod_template_file_examples}/__init__.py | 0 .../dags_in_image_template.yaml | 0 .../dags_in_volume_template.yaml | 0 .../git_sync_template.yaml | 0 .../providers/cncf/kubernetes/provider.yaml | 0 .../python_kubernetes_script.jinja2 | 0 .../kubernetes/python_kubernetes_script.py | 0 .../kubernetes/resource_convert}/__init__.py | 0 .../kubernetes/resource_convert/configmap.py | 0 .../resource_convert/env_variable.py | 0 .../kubernetes/resource_convert/secret.py | 0 .../providers/cncf/kubernetes/secret.py | 0 .../cncf/kubernetes/sensors}/__init__.py | 0 .../kubernetes/sensors/spark_kubernetes.py | 0 .../cncf/kubernetes/template_rendering.py | 0 .../cncf/kubernetes/triggers}/__init__.py | 0 .../providers/cncf/kubernetes/triggers/job.py | 0 .../kubernetes/triggers/kubernetes_pod.py | 0 .../providers/cncf/kubernetes/triggers/pod.py | 0 .../cncf/kubernetes/utils/__init__.py | 0 .../cncf/kubernetes/utils/delete_from.py | 0 .../kubernetes/utils/k8s_resource_iterator.py | 0 .../cncf/kubernetes/utils/pod_manager.py | 0 .../cncf/kubernetes/utils/xcom_sidecar.py | 0 .../cohere/.latest-doc-only-change.txt | 0 .../airflow}/providers/cohere/CHANGELOG.rst | 0 .../src/airflow}/providers/cohere/__init__.py | 0 .../providers/cohere/hooks/__init__.py | 0 .../airflow}/providers/cohere/hooks/cohere.py | 0 .../providers/cohere/operators}/__init__.py | 0 .../providers/cohere/operators/embedding.py | 0 .../airflow}/providers/cohere/provider.yaml | 0 .../common/compat/.latest-doc-only-change.txt | 0 .../providers/common/compat/CHANGELOG.rst | 0 .../providers/common/compat/__init__.py | 0 .../common/compat/assets/__init__.py | 0 .../common/compat/lineage/__init__.py | 0 .../providers/common/compat/lineage/hook.py | 0 .../common/compat/openlineage/__init__.py | 0 .../common/compat/openlineage/facet.py | 0 .../compat/openlineage/utils/__init__.py | 0 .../common/compat/openlineage/utils/utils.py | 0 .../providers/common/compat/provider.yaml | 0 .../common/compat/security/__init__.py | 0 .../common/compat/security/permissions.py | 0 .../common/io/.latest-doc-only-change.txt | 0 .../providers/common/io/CHANGELOG.rst | 0 .../airflow}/providers/common/io/__init__.py | 0 .../providers/common/io/assets/__init__.py | 0 .../common/io/assets/assets}/__init__.py | 0 .../providers/common/io/assets/file.py | 0 .../common/io/operators}/__init__.py | 0 .../common/io/operators/file_transfer.py | 0 .../providers/common/io/provider.yaml | 0 .../providers/common/io/xcom/__init__.py | 0 .../providers/common/io/xcom/backend.py | 0 .../common/sql/.latest-doc-only-change.txt | 0 .../providers/common/sql/CHANGELOG.rst | 0 .../providers/common/sql/README_API.md | 0 .../airflow}/providers/common/sql/__init__.py | 0 .../adr/0001-record-architecture-decisions.md | 0 ...-structure-from-dbapihook-derived-hooks.md | 0 .../providers/common/sql/hooks}/__init__.py | 0 .../providers/common/sql/hooks/sql.py | 8 +- .../providers/common/sql/hooks/sql.pyi | 17 +- .../common/sql/operators}/__init__.py | 0 .../providers/common/sql/operators/sql.py | 0 .../providers/common/sql/operators/sql.pyi | 18 +- .../providers/common/sql/provider.yaml | 0 .../providers/common/sql/sensors}/__init__.py | 0 .../providers/common/sql/sensors/sql.py | 0 .../providers/common/sql/sensors/sql.pyi | 5 +- .../databricks/.latest-doc-only-change.txt | 0 .../providers/databricks/CHANGELOG.rst | 0 .../airflow}/providers/databricks/__init__.py | 0 .../providers/databricks/exceptions.py | 0 .../providers/databricks/hooks}/__init__.py | 0 .../providers/databricks/hooks/databricks.py | 0 .../databricks/hooks/databricks_base.py | 0 .../databricks/hooks/databricks_sql.py | 0 .../databricks/operators}/__init__.py | 0 .../databricks/operators/databricks.py | 0 .../databricks/operators/databricks_repos.py | 0 .../databricks/operators/databricks_sql.py | 0 .../operators/databricks_workflow.py | 0 .../providers/databricks/plugins}/__init__.py | 0 .../databricks/plugins/databricks_workflow.py | 0 .../providers/databricks/provider.yaml | 0 .../providers/databricks/sensors}/__init__.py | 0 .../sensors/databricks_partition.py | 0 .../databricks/sensors/databricks_sql.py | 0 .../databricks/triggers}/__init__.py | 0 .../databricks/triggers/databricks.py | 0 .../providers/databricks/utils}/__init__.py | 0 .../providers/databricks/utils/databricks.py | 0 .../datadog/.latest-doc-only-change.txt | 0 .../airflow}/providers/datadog/CHANGELOG.rst | 0 .../airflow}/providers/datadog/__init__.py | 0 .../providers/datadog/hooks}/__init__.py | 0 .../providers/datadog/hooks/datadog.py | 0 .../airflow}/providers/datadog/provider.yaml | 0 .../providers/datadog/sensors}/__init__.py | 0 .../providers/datadog/sensors/datadog.py | 0 .../dbt/cloud/.latest-doc-only-change.txt | 0 .../providers/dbt/cloud/CHANGELOG.rst | 0 .../airflow}/providers/dbt/cloud/__init__.py | 0 .../providers/dbt/cloud/hooks/__init__.py | 0 .../airflow}/providers/dbt/cloud/hooks/dbt.py | 5 +- .../providers/dbt/cloud/operators/__init__.py | 0 .../providers/dbt/cloud/operators/dbt.py | 0 .../providers/dbt/cloud/provider.yaml | 0 .../providers/dbt/cloud}/sensors/__init__.py | 0 .../providers/dbt/cloud/sensors/dbt.py | 0 .../providers/dbt/cloud/triggers/__init__.py | 0 .../providers/dbt/cloud/triggers/dbt.py | 0 .../providers/dbt/cloud/utils/__init__.py | 0 .../providers/dbt/cloud/utils/openlineage.py | 0 .../dingding/.latest-doc-only-change.txt | 0 .../airflow}/providers/dingding/CHANGELOG.rst | 0 .../airflow}/providers/dingding/__init__.py | 0 .../providers/dingding/hooks}/__init__.py | 0 .../providers/dingding/hooks/dingding.py | 0 .../providers/dingding/operators}/__init__.py | 0 .../providers/dingding/operators/dingding.py | 0 .../airflow}/providers/dingding/provider.yaml | 0 .../discord/.latest-doc-only-change.txt | 0 .../airflow}/providers/discord/CHANGELOG.rst | 0 .../airflow}/providers/discord/__init__.py | 0 .../providers/discord/hooks}/__init__.py | 0 .../discord/hooks/discord_webhook.py | 0 .../discord/notifications}/__init__.py | 0 .../discord/notifications/discord.py | 0 .../providers/discord/operators}/__init__.py | 0 .../discord/operators/discord_webhook.py | 0 .../airflow}/providers/discord/provider.yaml | 0 .../docker/.latest-doc-only-change.txt | 0 .../airflow}/providers/docker/CHANGELOG.rst | 0 .../src/airflow}/providers/docker/__init__.py | 0 .../providers/docker/decorators}/__init__.py | 0 .../providers/docker/decorators/docker.py | 0 .../airflow}/providers/docker/exceptions.py | 0 .../providers/docker/hooks}/__init__.py | 0 .../airflow}/providers/docker/hooks/docker.py | 0 .../providers/docker/operators}/__init__.py | 0 .../providers/docker/operators/docker.py | 0 .../docker/operators/docker_swarm.py | 0 .../airflow}/providers/docker/provider.yaml | 0 .../src/airflow}/providers/edge/CHANGELOG.rst | 0 .../src/airflow}/providers/edge/__init__.py | 0 .../providers/edge/api_endpoints/__init__.py | 0 .../edge/api_endpoints/health_endpoint.py | 0 .../edge/api_endpoints/rpc_api_endpoint.py | 0 .../airflow}/providers/edge/cli/__init__.py | 0 .../providers/edge/cli/edge_command.py | 0 .../providers/edge/example_dags/__init__.py | 0 .../edge/example_dags/integration_test.py | 0 .../providers/edge/models/__init__.py | 0 .../providers/edge/models/edge_job.py | 0 .../providers/edge/models/edge_logs.py | 0 .../providers/edge/models/edge_worker.py | 0 .../providers/edge/openapi/__init__.py | 0 .../edge/openapi/edge_worker_api_v1.yaml | 0 .../providers/edge/plugins/__init__.py | 0 .../edge/plugins/edge_executor_plugin.py | 0 .../plugins/templates/edge_worker_hosts.html | 0 .../plugins/templates/edge_worker_jobs.html | 0 .../src/airflow}/providers/edge/provider.yaml | 0 .../elasticsearch/.latest-doc-only-change.txt | 0 .../providers/elasticsearch/CHANGELOG.rst | 0 .../providers/elasticsearch/__init__.py | 0 .../elasticsearch/hooks}/__init__.py | 0 .../elasticsearch/hooks/elasticsearch.py | 0 .../providers/elasticsearch/log/__init__.py | 0 .../elasticsearch/log/es_json_formatter.py | 0 .../elasticsearch/log/es_response.py | 0 .../elasticsearch/log/es_task_handler.py | 0 .../providers/elasticsearch/provider.yaml | 0 .../exasol/.latest-doc-only-change.txt | 0 .../airflow}/providers/exasol/CHANGELOG.rst | 0 .../src/airflow}/providers/exasol/__init__.py | 0 .../providers/exasol}/hooks/__init__.py | 0 .../airflow}/providers/exasol/hooks/exasol.py | 0 .../providers/exasol/operators}/__init__.py | 0 .../providers/exasol/operators/exasol.py | 0 .../airflow}/providers/exasol/provider.yaml | 0 .../src/airflow}/providers/fab/CHANGELOG.rst | 0 .../src/airflow}/providers/fab/__init__.py | 0 .../src/airflow}/providers/fab/alembic.ini | 0 .../providers/fab/auth_manager}/__init__.py | 0 .../fab/auth_manager/api}/__init__.py | 0 .../fab/auth_manager/api/auth}/__init__.py | 0 .../api/auth/backend}/__init__.py | 0 .../api/auth/backend/basic_auth.py | 0 .../api/auth/backend/kerberos_auth.py | 0 .../auth_manager/api_endpoints/__init__.py | 0 .../role_and_permission_endpoint.py | 0 .../api_endpoints/user_endpoint.py | 0 .../auth_manager/cli_commands}/__init__.py | 0 .../auth_manager/cli_commands/db_command.py | 0 .../auth_manager/cli_commands/definition.py | 0 .../auth_manager/cli_commands/role_command.py | 0 .../cli_commands/sync_perm_command.py | 0 .../auth_manager/cli_commands/user_command.py | 0 .../fab/auth_manager/cli_commands/utils.py | 0 .../fab/auth_manager/decorators}/__init__.py | 0 .../fab/auth_manager/decorators/auth.py | 0 .../fab/auth_manager/fab_auth_manager.py | 0 .../fab/auth_manager/models/__init__.py | 0 .../fab/auth_manager/models/anonymous_user.py | 0 .../providers/fab/auth_manager/models/db.py | 9 +- .../fab/auth_manager/openapi/__init__.py | 0 .../fab/auth_manager/openapi/v1.yaml | 0 .../security_manager}/__init__.py | 0 .../security_manager/constants.py | 0 .../auth_manager/security_manager/override.py | 0 .../fab/auth_manager/views/__init__.py | 0 .../fab/auth_manager/views/permissions.py | 0 .../fab/auth_manager/views/roles_list.py | 0 .../providers/fab/auth_manager/views/user.py | 0 .../fab/auth_manager/views/user_edit.py | 0 .../fab/auth_manager/views/user_stats.py | 0 .../airflow}/providers/fab/migrations/README | 0 .../providers/fab/migrations/__init__.py | 0 .../airflow}/providers/fab/migrations/env.py | 0 .../providers/fab/migrations/script.py.mako | 0 .../0001_1_4_0_placeholder_migration.py | 0 .../fab/migrations/versions/__init__.py | 0 .../src/airflow}/providers/fab/provider.yaml | 0 .../facebook/.latest-doc-only-change.txt | 0 .../airflow}/providers/facebook/CHANGELOG.rst | 0 .../airflow}/providers/facebook/__init__.py | 0 .../providers/facebook/ads/__init__.py | 0 .../providers/facebook/ads/hooks/__init__.py | 0 .../providers/facebook/ads/hooks/ads.py | 0 .../airflow}/providers/facebook/provider.yaml | 0 .../providers/ftp/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/ftp/CHANGELOG.rst | 0 .../src/airflow}/providers/ftp/__init__.py | 0 .../airflow/providers/ftp/hooks}/__init__.py | 0 .../src/airflow}/providers/ftp/hooks/ftp.py | 0 .../providers/ftp/operators/__init__.py | 0 .../airflow}/providers/ftp/operators/ftp.py | 0 .../src/airflow}/providers/ftp/provider.yaml | 0 .../providers/ftp/sensors}/__init__.py | 0 .../src/airflow}/providers/ftp/sensors/ftp.py | 0 .../github/.latest-doc-only-change.txt | 0 .../airflow}/providers/github/CHANGELOG.rst | 0 .../src/airflow}/providers/github/__init__.py | 0 .../providers/github/hooks/__init__.py | 0 .../airflow}/providers/github/hooks/github.py | 0 .../providers/github/operators/__init__.py | 0 .../providers/github/operators/github.py | 0 .../airflow}/providers/github/provider.yaml | 0 .../providers/github/sensors/__init__.py | 0 .../providers/github/sensors/github.py | 0 .../google/.latest-doc-only-change.txt | 0 .../airflow}/providers/google/CHANGELOG.rst | 0 .../src/airflow}/providers/google/__init__.py | 0 .../airflow}/providers/google/ads/.gitignore | 0 .../airflow}/providers/google/ads/__init__.py | 0 .../providers/google/ads/hooks/__init__.py | 0 .../providers/google/ads/hooks/ads.py | 0 .../google/ads/operators/__init__.py | 0 .../providers/google/ads/operators/ads.py | 0 .../google/ads/transfers/__init__.py | 0 .../google/ads/transfers/ads_to_gcs.py | 0 .../providers/google/cloud/__init__.py | 0 .../google/cloud/_internal_client/__init__.py | 0 .../_internal_client/secret_manager_client.py | 0 .../google/cloud/example_dags/__init__.py | 0 .../cloud/example_dags/example_cloud_task.py | 0 .../example_facebook_ads_to_gcs.py | 0 .../cloud/example_dags/example_looker.py | 0 .../example_dags/example_presto_to_gcs.py | 0 .../example_dags/example_salesforce_to_gcs.py | 0 .../providers/google/cloud/fs/__init__.py | 0 .../airflow}/providers/google/cloud/fs/gcs.py | 0 .../providers/google/cloud/hooks/__init__.py | 0 .../providers/google/cloud/hooks/automl.py | 0 .../providers/google/cloud/hooks/bigquery.py | 0 .../google/cloud/hooks/bigquery_dts.py | 0 .../providers/google/cloud/hooks/bigtable.py | 0 .../google/cloud/hooks/cloud_batch.py | 0 .../google/cloud/hooks/cloud_build.py | 0 .../google/cloud/hooks/cloud_composer.py | 0 .../google/cloud/hooks/cloud_memorystore.py | 0 .../providers/google/cloud/hooks/cloud_run.py | 0 .../providers/google/cloud/hooks/cloud_sql.py | 0 .../hooks/cloud_storage_transfer_service.py | 0 .../providers/google/cloud/hooks/compute.py | 0 .../google/cloud/hooks/compute_ssh.py | 0 .../google/cloud/hooks/datacatalog.py | 0 .../providers/google/cloud/hooks/dataflow.py | 0 .../providers/google/cloud/hooks/dataform.py | 0 .../google/cloud/hooks/datafusion.py | 0 .../google/cloud/hooks/datapipeline.py | 0 .../providers/google/cloud/hooks/dataplex.py | 0 .../providers/google/cloud/hooks/dataprep.py | 0 .../providers/google/cloud/hooks/dataproc.py | 0 .../google/cloud/hooks/dataproc_metastore.py | 0 .../providers/google/cloud/hooks/datastore.py | 0 .../providers/google/cloud/hooks/dlp.py | 0 .../providers/google/cloud/hooks/functions.py | 0 .../providers/google/cloud/hooks/gcs.py | 0 .../providers/google/cloud/hooks/gdm.py | 0 .../providers/google/cloud/hooks/kms.py | 0 .../google/cloud/hooks/kubernetes_engine.py | 0 .../google/cloud/hooks/life_sciences.py | 0 .../providers/google/cloud/hooks/looker.py | 0 .../providers/google/cloud/hooks/mlengine.py | 0 .../google/cloud/hooks/natural_language.py | 0 .../providers/google/cloud/hooks/os_login.py | 0 .../providers/google/cloud/hooks/pubsub.py | 0 .../google/cloud/hooks/secret_manager.py | 0 .../providers/google/cloud/hooks/spanner.py | 0 .../google/cloud/hooks/speech_to_text.py | 0 .../google/cloud/hooks/stackdriver.py | 0 .../providers/google/cloud/hooks/tasks.py | 0 .../google/cloud/hooks/text_to_speech.py | 0 .../providers/google/cloud/hooks/translate.py | 0 .../google/cloud/hooks/vertex_ai/__init__.py | 0 .../google/cloud/hooks/vertex_ai/auto_ml.py | 0 .../hooks/vertex_ai/batch_prediction_job.py | 0 .../cloud/hooks/vertex_ai/custom_job.py | 0 .../google/cloud/hooks/vertex_ai/dataset.py | 0 .../cloud/hooks/vertex_ai/endpoint_service.py | 0 .../cloud/hooks/vertex_ai/generative_model.py | 0 .../vertex_ai/hyperparameter_tuning_job.py | 0 .../cloud/hooks/vertex_ai/model_service.py | 0 .../cloud/hooks/vertex_ai/pipeline_job.py | 0 .../hooks/vertex_ai/prediction_service.py | 0 .../google/cloud/hooks/video_intelligence.py | 0 .../providers/google/cloud/hooks/vision.py | 0 .../providers/google/cloud/hooks/workflows.py | 0 .../providers/google/cloud/links/__init__.py | 0 .../providers/google/cloud/links/automl.py | 0 .../providers/google/cloud/links/base.py | 0 .../providers/google/cloud/links/bigquery.py | 0 .../google/cloud/links/bigquery_dts.py | 0 .../providers/google/cloud/links/bigtable.py | 0 .../google/cloud/links/cloud_build.py | 0 .../google/cloud/links/cloud_functions.py | 0 .../google/cloud/links/cloud_memorystore.py | 0 .../providers/google/cloud/links/cloud_sql.py | 0 .../cloud/links/cloud_storage_transfer.py | 0 .../google/cloud/links/cloud_tasks.py | 0 .../providers/google/cloud/links/compute.py | 0 .../cloud/links/data_loss_prevention.py | 0 .../google/cloud/links/datacatalog.py | 0 .../providers/google/cloud/links/dataflow.py | 0 .../providers/google/cloud/links/dataform.py | 0 .../google/cloud/links/datafusion.py | 0 .../providers/google/cloud/links/dataplex.py | 0 .../providers/google/cloud/links/dataprep.py | 0 .../providers/google/cloud/links/dataproc.py | 0 .../providers/google/cloud/links/datastore.py | 0 .../google/cloud/links/kubernetes_engine.py | 0 .../google/cloud/links/life_sciences.py | 0 .../providers/google/cloud/links/mlengine.py | 0 .../providers/google/cloud/links/pubsub.py | 0 .../providers/google/cloud/links/spanner.py | 0 .../google/cloud/links/stackdriver.py | 0 .../providers/google/cloud/links/translate.py | 0 .../providers/google/cloud/links/vertex_ai.py | 0 .../providers/google/cloud/links/workflows.py | 0 .../providers/google/cloud/log/__init__.py | 0 .../google/cloud/log/gcs_task_handler.py | 0 .../cloud/log/stackdriver_task_handler.py | 0 .../openlineage/BigQueryErrorRunFacet.json | 0 .../openlineage/BigQueryJobRunFacet.json | 0 .../google/cloud/openlineage/__init__.py | 0 .../google/cloud/openlineage/mixins.py | 0 .../google/cloud/openlineage/utils.py | 0 .../google/cloud/operators/__init__.py | 0 .../google/cloud/operators/automl.py | 0 .../google/cloud/operators/bigquery.py | 2 +- .../google/cloud/operators/bigquery_dts.py | 0 .../google/cloud/operators/bigtable.py | 0 .../google/cloud/operators/cloud_base.py | 0 .../google/cloud/operators/cloud_batch.py | 0 .../google/cloud/operators/cloud_build.py | 0 .../google/cloud/operators/cloud_composer.py | 0 .../cloud/operators/cloud_memorystore.py | 0 .../google/cloud/operators/cloud_run.py | 0 .../google/cloud/operators/cloud_sql.py | 0 .../cloud_storage_transfer_service.py | 0 .../google/cloud/operators/compute.py | 0 .../google/cloud/operators/datacatalog.py | 0 .../google/cloud/operators/dataflow.py | 0 .../google/cloud/operators/dataform.py | 0 .../google/cloud/operators/datafusion.py | 0 .../google/cloud/operators/datapipeline.py | 0 .../google/cloud/operators/dataplex.py | 0 .../google/cloud/operators/dataprep.py | 0 .../google/cloud/operators/dataproc.py | 0 .../cloud/operators/dataproc_metastore.py | 0 .../google/cloud/operators/datastore.py | 0 .../providers/google/cloud/operators/dlp.py | 0 .../google/cloud/operators/functions.py | 0 .../providers/google/cloud/operators/gcs.py | 0 .../cloud/operators/kubernetes_engine.py | 0 .../google/cloud/operators/life_sciences.py | 0 .../google/cloud/operators/looker.py | 0 .../google/cloud/operators/mlengine.py | 0 .../cloud/operators/natural_language.py | 0 .../google/cloud/operators/pubsub.py | 0 .../google/cloud/operators/spanner.py | 0 .../google/cloud/operators/speech_to_text.py | 0 .../google/cloud/operators/stackdriver.py | 0 .../providers/google/cloud/operators/tasks.py | 0 .../google/cloud/operators/text_to_speech.py | 0 .../google/cloud/operators/translate.py | 0 .../cloud/operators/translate_speech.py | 0 .../cloud/operators/vertex_ai/__init__.py | 0 .../cloud/operators/vertex_ai/auto_ml.py | 0 .../vertex_ai/batch_prediction_job.py | 0 .../cloud/operators/vertex_ai/custom_job.py | 0 .../cloud/operators/vertex_ai/dataset.py | 0 .../operators/vertex_ai/endpoint_service.py | 0 .../operators/vertex_ai/generative_model.py | 0 .../vertex_ai/hyperparameter_tuning_job.py | 0 .../operators/vertex_ai/model_service.py | 0 .../cloud/operators/vertex_ai/pipeline_job.py | 0 .../cloud/operators/video_intelligence.py | 0 .../google/cloud/operators/vision.py | 0 .../google/cloud/operators/workflows.py | 0 .../google/cloud/secrets/__init__.py | 0 .../google/cloud/secrets/secret_manager.py | 0 .../google/cloud/sensors/__init__.py | 0 .../google/cloud/sensors/bigquery.py | 0 .../google/cloud/sensors/bigquery_dts.py | 0 .../google/cloud/sensors/bigtable.py | 0 .../google/cloud/sensors/cloud_composer.py | 0 .../sensors/cloud_storage_transfer_service.py | 0 .../google/cloud/sensors/dataflow.py | 0 .../google/cloud/sensors/dataform.py | 0 .../google/cloud/sensors/datafusion.py | 0 .../google/cloud/sensors/dataplex.py | 0 .../google/cloud/sensors/dataprep.py | 0 .../google/cloud/sensors/dataproc.py | 0 .../cloud/sensors/dataproc_metastore.py | 0 .../providers/google/cloud/sensors/gcs.py | 0 .../providers/google/cloud/sensors/looker.py | 0 .../providers/google/cloud/sensors/pubsub.py | 0 .../providers/google/cloud/sensors/tasks.py | 0 .../google/cloud/sensors/workflows.py | 0 .../google/cloud/transfers/__init__.py | 0 .../google/cloud/transfers/adls_to_gcs.py | 0 .../cloud/transfers/azure_blob_to_gcs.py | 0 .../cloud/transfers/azure_fileshare_to_gcs.py | 0 .../cloud/transfers/bigquery_to_bigquery.py | 0 .../google/cloud/transfers/bigquery_to_gcs.py | 0 .../cloud/transfers/bigquery_to_mssql.py | 0 .../cloud/transfers/bigquery_to_mysql.py | 0 .../cloud/transfers/bigquery_to_postgres.py | 0 .../google/cloud/transfers/bigquery_to_sql.py | 0 .../google/cloud/transfers/calendar_to_gcs.py | 0 .../cloud/transfers/cassandra_to_gcs.py | 0 .../cloud/transfers/facebook_ads_to_gcs.py | 0 .../google/cloud/transfers/gcs_to_bigquery.py | 0 .../google/cloud/transfers/gcs_to_gcs.py | 0 .../google/cloud/transfers/gcs_to_local.py | 0 .../google/cloud/transfers/gcs_to_sftp.py | 0 .../google/cloud/transfers/gdrive_to_gcs.py | 0 .../google/cloud/transfers/gdrive_to_local.py | 0 .../google/cloud/transfers/local_to_gcs.py | 0 .../google/cloud/transfers/mssql_to_gcs.py | 0 .../google/cloud/transfers/mysql_to_gcs.py | 0 .../google/cloud/transfers/oracle_to_gcs.py | 0 .../google/cloud/transfers/postgres_to_gcs.py | 0 .../google/cloud/transfers/presto_to_gcs.py | 0 .../google/cloud/transfers/s3_to_gcs.py | 0 .../cloud/transfers/salesforce_to_gcs.py | 0 .../google/cloud/transfers/sftp_to_gcs.py | 0 .../google/cloud/transfers/sheets_to_gcs.py | 0 .../google/cloud/transfers/sql_to_gcs.py | 0 .../google/cloud/transfers/trino_to_gcs.py | 0 .../google/cloud/triggers/__init__.py | 0 .../google/cloud/triggers/bigquery.py | 0 .../google/cloud/triggers/bigquery_dts.py | 0 .../google/cloud/triggers/cloud_batch.py | 0 .../google/cloud/triggers/cloud_build.py | 0 .../google/cloud/triggers/cloud_composer.py | 0 .../google/cloud/triggers/cloud_run.py | 0 .../google/cloud/triggers/cloud_sql.py | 0 .../cloud_storage_transfer_service.py | 0 .../google/cloud/triggers/dataflow.py | 0 .../google/cloud/triggers/datafusion.py | 0 .../google/cloud/triggers/dataplex.py | 0 .../google/cloud/triggers/dataproc.py | 0 .../providers/google/cloud/triggers/gcs.py | 0 .../cloud/triggers/kubernetes_engine.py | 0 .../google/cloud/triggers/mlengine.py | 0 .../providers/google/cloud/triggers/pubsub.py | 0 .../google/cloud/triggers/vertex_ai.py | 0 .../providers/google/cloud/utils}/__init__.py | 0 .../providers/google/cloud/utils/bigquery.py | 0 .../google/cloud/utils/bigquery_get_data.py | 0 .../cloud/utils/credentials_provider.py | 0 .../providers/google/cloud/utils/dataform.py | 0 .../google/cloud/utils/datafusion.py | 0 .../providers/google/cloud/utils/dataproc.py | 0 .../cloud/utils/external_token_supplier.py | 0 .../google/cloud/utils/field_sanitizer.py | 0 .../google/cloud/utils/field_validator.py | 0 .../providers/google/cloud/utils/helpers.py | 0 .../cloud/utils/mlengine_operator_utils.py | 0 .../utils/mlengine_prediction_summary.py | 0 .../providers/google/common/__init__.py | 0 .../google/common/auth_backend/__init__.py | 0 .../common/auth_backend/google_openid.py | 0 .../providers/google/common/consts.py | 0 .../providers/google/common/deprecated.py | 0 .../providers/google/common/hooks/__init__.py | 0 .../google/common/hooks/base_google.py | 0 .../google/common/hooks/discovery_api.py | 0 .../providers/google/common/links/__init__.py | 0 .../providers/google/common/links/storage.py | 0 .../providers/google/common/utils/__init__.py | 0 .../common/utils/id_token_credentials.py | 0 .../providers/google/datasets/__init__.py | 0 .../providers/google/datasets/bigquery.py | 0 .../providers/google/firebase/__init__.py | 0 .../google/firebase/hooks/__init__.py | 0 .../google/firebase/hooks/firestore.py | 0 .../google/firebase/operators/__init__.py | 0 .../google/firebase/operators/firestore.py | 0 .../providers/google/go_module_utils.py | 0 .../providers/google/leveldb/__init__.py | 0 .../google/leveldb/hooks/__init__.py | 0 .../providers/google/leveldb/hooks/leveldb.py | 0 .../google/leveldb/operators/__init__.py | 0 .../google/leveldb/operators/leveldb.py | 0 .../google/marketing_platform/__init__.py | 0 .../example_dags/__init__.py | 0 .../example_dags/example_display_video.py | 0 .../marketing_platform/hooks/__init__.py | 0 .../marketing_platform/hooks/analytics.py | 0 .../hooks/analytics_admin.py | 0 .../hooks/campaign_manager.py | 0 .../marketing_platform/hooks/display_video.py | 0 .../marketing_platform/hooks/search_ads.py | 0 .../marketing_platform/links/__init__.py | 0 .../links/analytics_admin.py | 0 .../marketing_platform/operators/__init__.py | 0 .../marketing_platform/operators/analytics.py | 0 .../operators/analytics_admin.py | 0 .../operators/campaign_manager.py | 0 .../operators/display_video.py | 0 .../operators/search_ads.py | 0 .../marketing_platform/sensors/__init__.py | 0 .../sensors/campaign_manager.py | 0 .../sensors/display_video.py | 0 .../airflow}/providers/google/provider.yaml | 0 .../providers/google/suite}/__init__.py | 0 .../providers/google/suite/hooks}/__init__.py | 0 .../providers/google/suite/hooks/calendar.py | 0 .../providers/google/suite/hooks/drive.py | 0 .../providers/google/suite/hooks/sheets.py | 0 .../google/suite/operators}/__init__.py | 0 .../google/suite/operators/sheets.py | 0 .../google/suite/sensors/__init__.py | 0 .../providers/google/suite/sensors/drive.py | 0 .../google/suite/transfers/__init__.py | 0 .../google/suite/transfers/gcs_to_gdrive.py | 0 .../google/suite/transfers/gcs_to_sheets.py | 0 .../google/suite/transfers/local_to_drive.py | 0 .../google/suite/transfers/sql_to_sheets.py | 0 .../grpc/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/grpc/CHANGELOG.rst | 0 .../src/airflow}/providers/grpc/__init__.py | 0 .../airflow/providers/grpc/hooks}/__init__.py | 0 .../src/airflow}/providers/grpc/hooks/grpc.py | 0 .../providers/grpc/operators}/__init__.py | 0 .../airflow}/providers/grpc/operators/grpc.py | 0 .../src/airflow}/providers/grpc/provider.yaml | 0 .../hashicorp/.latest-doc-only-change.txt | 0 .../providers/hashicorp/CHANGELOG.rst | 0 .../airflow}/providers/hashicorp/__init__.py | 0 .../hashicorp/_internal_client/__init__.py | 0 .../_internal_client/vault_client.py | 0 .../providers/hashicorp/hooks/__init__.py | 0 .../providers/hashicorp/hooks/vault.py | 0 .../providers/hashicorp/provider.yaml | 0 .../providers/hashicorp/secrets/__init__.py | 0 .../providers/hashicorp/secrets/vault.py | 0 .../http/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/http/CHANGELOG.rst | 0 .../src/airflow}/providers/http/__init__.py | 0 .../airflow/providers/http/hooks}/__init__.py | 0 .../src/airflow}/providers/http/hooks/http.py | 0 .../providers/http/operators}/__init__.py | 0 .../airflow}/providers/http/operators/http.py | 0 .../src/airflow}/providers/http/provider.yaml | 0 .../providers/http/sensors}/__init__.py | 0 .../airflow}/providers/http/sensors/http.py | 0 .../providers/http/triggers}/__init__.py | 0 .../airflow}/providers/http/triggers/http.py | 0 .../imap/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/imap/CHANGELOG.rst | 0 .../src/airflow}/providers/imap/__init__.py | 0 .../airflow/providers/imap/hooks}/__init__.py | 0 .../src/airflow}/providers/imap/hooks/imap.py | 0 .../src/airflow}/providers/imap/provider.yaml | 0 .../providers/imap/sensors}/__init__.py | 0 .../providers/imap/sensors/imap_attachment.py | 0 .../influxdb/.latest-doc-only-change.txt | 0 .../airflow}/providers/influxdb/CHANGELOG.rst | 0 .../airflow}/providers/influxdb/__init__.py | 0 .../providers/influxdb/hooks/__init__.py | 0 .../providers/influxdb/hooks/influxdb.py | 0 .../providers/influxdb/operators/__init__.py | 0 .../providers/influxdb/operators/influxdb.py | 0 .../airflow}/providers/influxdb/provider.yaml | 0 .../jdbc/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/jdbc/CHANGELOG.rst | 0 .../src/airflow}/providers/jdbc/__init__.py | 0 .../airflow/providers/jdbc/hooks}/__init__.py | 0 .../src/airflow}/providers/jdbc/hooks/jdbc.py | 0 .../providers/jdbc/operators}/__init__.py | 0 .../airflow}/providers/jdbc/operators/jdbc.py | 0 .../src/airflow}/providers/jdbc/provider.yaml | 0 .../jenkins/.latest-doc-only-change.txt | 0 .../airflow}/providers/jenkins/CHANGELOG.rst | 0 .../airflow}/providers/jenkins/__init__.py | 0 .../providers/jenkins/hooks}/__init__.py | 0 .../providers/jenkins/hooks/jenkins.py | 0 .../providers/jenkins/operators}/__init__.py | 0 .../jenkins/operators/jenkins_job_trigger.py | 0 .../airflow}/providers/jenkins/provider.yaml | 0 .../providers/jenkins/sensors}/__init__.py | 0 .../providers/jenkins/sensors/jenkins.py | 0 .../azure/.latest-doc-only-change.txt | 0 .../providers/microsoft/azure/CHANGELOG.rst | 0 .../providers/microsoft/azure/__init__.py | 0 .../providers/microsoft/azure/fs/__init__.py | 0 .../providers/microsoft/azure/fs/adls.py | 0 .../microsoft/azure/hooks}/__init__.py | 0 .../providers/microsoft/azure/hooks/adx.py | 0 .../providers/microsoft/azure/hooks/asb.py | 26 +- .../microsoft/azure/hooks/base_azure.py | 0 .../providers/microsoft/azure/hooks/batch.py | 0 .../azure/hooks/container_instance.py | 0 .../azure/hooks/container_registry.py | 0 .../microsoft/azure/hooks/container_volume.py | 0 .../providers/microsoft/azure/hooks/cosmos.py | 0 .../microsoft/azure/hooks/data_factory.py | 0 .../microsoft/azure/hooks/data_lake.py | 0 .../microsoft/azure/hooks/fileshare.py | 0 .../microsoft/azure/hooks/msgraph.py | 0 .../microsoft/azure/hooks/powerbi.py | 0 .../microsoft/azure/hooks/synapse.py | 0 .../providers/microsoft/azure/hooks/wasb.py | 0 .../providers/microsoft/azure/log/__init__.py | 0 .../microsoft/azure/log/wasb_task_handler.py | 0 .../microsoft/azure/operators}/__init__.py | 0 .../microsoft/azure/operators/adls.py | 0 .../microsoft/azure/operators/adx.py | 0 .../microsoft/azure/operators/asb.py | 0 .../microsoft/azure/operators/batch.py | 0 .../azure/operators/container_instances.py | 0 .../microsoft/azure/operators/cosmos.py | 0 .../microsoft/azure/operators/data_factory.py | 0 .../microsoft/azure/operators/msgraph.py | 0 .../microsoft/azure/operators/powerbi.py | 0 .../microsoft/azure/operators/synapse.py | 0 .../azure/operators/wasb_delete_blob.py | 0 .../providers/microsoft/azure/provider.yaml | 0 .../microsoft/azure/secrets/__init__.py | 0 .../microsoft/azure/secrets/key_vault.py | 0 .../microsoft/azure/sensors}/__init__.py | 0 .../microsoft/azure/sensors/cosmos.py | 0 .../microsoft/azure/sensors/data_factory.py | 0 .../microsoft/azure/sensors/msgraph.py | 0 .../providers/microsoft/azure/sensors/wasb.py | 0 .../microsoft/azure/transfers/__init__.py | 0 .../azure/transfers/azure_blob_to_gcs.py | 0 .../azure/transfers/local_to_adls.py | 0 .../azure/transfers/local_to_wasb.py | 0 .../transfers/oracle_to_azure_data_lake.py | 0 .../microsoft/azure/transfers/s3_to_wasb.py | 0 .../microsoft/azure/transfers/sftp_to_wasb.py | 0 .../microsoft/azure/triggers/__init__.py | 0 .../microsoft/azure/triggers/data_factory.py | 0 .../microsoft/azure/triggers/msgraph.py | 0 .../microsoft/azure/triggers/powerbi.py | 0 .../microsoft/azure/triggers/wasb.py | 0 .../providers/microsoft/azure/utils.py | 0 .../mssql/.latest-doc-only-change.txt | 0 .../providers/microsoft/mssql/CHANGELOG.rst | 0 .../providers/microsoft/mssql/__init__.py | 0 .../microsoft/mssql/hooks}/__init__.py | 0 .../providers/microsoft/mssql/hooks/mssql.py | 0 .../microsoft/mssql/operators}/__init__.py | 0 .../microsoft/mssql/operators/mssql.py | 0 .../providers/microsoft/mssql/provider.yaml | 0 .../psrp/.latest-doc-only-change.txt | 0 .../providers/microsoft/psrp/CHANGELOG.rst | 0 .../providers/microsoft/psrp/__init__.py | 0 .../microsoft/psrp}/hooks/__init__.py | 0 .../providers/microsoft/psrp/hooks/psrp.py | 0 .../microsoft/psrp}/operators/__init__.py | 0 .../microsoft/psrp/operators/psrp.py | 20 +- .../providers/microsoft/psrp/provider.yaml | 0 .../winrm/.latest-doc-only-change.txt | 0 .../providers/microsoft/winrm/CHANGELOG.rst | 0 .../providers/microsoft/winrm/__init__.py | 0 .../microsoft/winrm}/hooks/__init__.py | 0 .../providers/microsoft/winrm/hooks/winrm.py | 0 .../microsoft/winrm}/operators/__init__.py | 0 .../microsoft/winrm/operators/winrm.py | 0 .../providers/microsoft/winrm/provider.yaml | 0 .../mongo/.latest-doc-only-change.txt | 0 .../airflow}/providers/mongo/CHANGELOG.rst | 0 .../src/airflow}/providers/mongo/__init__.py | 0 .../providers/mongo}/hooks/__init__.py | 0 .../airflow}/providers/mongo/hooks/mongo.py | 0 .../airflow}/providers/mongo/provider.yaml | 0 .../providers/mongo/sensors}/__init__.py | 0 .../airflow}/providers/mongo/sensors/mongo.py | 0 .../mysql/.latest-doc-only-change.txt | 0 .../airflow}/providers/mysql/CHANGELOG.rst | 0 .../src/airflow}/providers/mysql/__init__.py | 0 .../providers/mysql/assets/__init__.py | 0 .../airflow}/providers/mysql/assets/mysql.py | 0 .../providers/mysql}/hooks/__init__.py | 0 .../airflow}/providers/mysql/hooks/mysql.py | 0 .../providers/mysql/operators}/__init__.py | 0 .../providers/mysql/operators/mysql.py | 0 .../airflow}/providers/mysql/provider.yaml | 0 .../providers/mysql/transfers/__init__.py | 0 .../mysql/transfers/presto_to_mysql.py | 0 .../providers/mysql/transfers/s3_to_mysql.py | 0 .../mysql/transfers/trino_to_mysql.py | 0 .../mysql/transfers/vertica_to_mysql.py | 0 .../neo4j/.latest-doc-only-change.txt | 0 .../airflow}/providers/neo4j/CHANGELOG.rst | 0 .../src/airflow}/providers/neo4j/README.md | 0 .../src/airflow}/providers/neo4j/__init__.py | 0 .../providers/neo4j}/hooks/__init__.py | 0 .../airflow}/providers/neo4j/hooks/neo4j.py | 0 .../providers/neo4j}/operators/__init__.py | 0 .../providers/neo4j/operators/neo4j.py | 0 .../airflow}/providers/neo4j/provider.yaml | 0 .../odbc/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/odbc/CHANGELOG.rst | 0 .../src/airflow}/providers/odbc/__init__.py | 0 .../airflow}/providers/odbc/hooks/__init__.py | 0 .../src/airflow}/providers/odbc/hooks/odbc.py | 0 .../src/airflow}/providers/odbc/provider.yaml | 0 .../openai/.latest-doc-only-change.txt | 0 .../airflow}/providers/openai/CHANGELOG.rst | 0 .../src/airflow}/providers/openai/__init__.py | 0 .../airflow}/providers/openai/exceptions.py | 0 .../providers/openai/hooks/__init__.py | 0 .../airflow}/providers/openai/hooks/openai.py | 0 .../providers/openai/operators/__init__.py | 0 .../providers/openai/operators/openai.py | 0 .../airflow}/providers/openai/provider.yaml | 0 .../providers/openai/triggers/__init__.py | 0 .../providers/openai/triggers/openai.py | 0 .../openfaas/.latest-doc-only-change.txt | 0 .../airflow}/providers/openfaas/CHANGELOG.rst | 0 .../airflow}/providers/openfaas/__init__.py | 0 .../providers/openfaas}/hooks/__init__.py | 0 .../providers/openfaas/hooks/openfaas.py | 0 .../airflow}/providers/openfaas/provider.yaml | 0 .../openlineage/.latest-doc-only-change.txt | 0 .../providers/openlineage/CHANGELOG.rst | 0 .../providers/openlineage/__init__.py | 0 .../airflow}/providers/openlineage/conf.py | 0 .../openlineage/extractors/__init__.py | 0 .../providers/openlineage/extractors/base.py | 0 .../providers/openlineage/extractors/bash.py | 0 .../openlineage/extractors/manager.py | 0 .../openlineage/extractors/python.py | 0 .../facets/AirflowDagRunFacet.json | 0 .../facets/AirflowDebugRunFacet.json | 0 .../openlineage/facets/AirflowJobFacet.json | 0 .../openlineage/facets/AirflowRunFacet.json | 0 .../facets/AirflowStateRunFacet.json | 0 .../providers/openlineage/facets/__init__.py | 0 .../providers/openlineage/plugins/__init__.py | 0 .../providers/openlineage/plugins/adapter.py | 0 .../providers/openlineage/plugins/facets.py | 0 .../providers/openlineage/plugins/listener.py | 0 .../providers/openlineage/plugins/macros.py | 0 .../openlineage/plugins/openlineage.py | 0 .../providers/openlineage/provider.yaml | 0 .../providers/openlineage/sqlparser.py | 0 .../providers/openlineage/utils/__init__.py | 0 .../utils/asset_compat_lineage_collector.py | 0 .../openlineage/utils/selective_enable.py | 0 .../providers/openlineage/utils/sql.py | 0 .../providers/openlineage/utils/utils.py | 0 .../opensearch/.latest-doc-only-change.txt | 0 .../providers/opensearch/CHANGELOG.rst | 0 .../airflow}/providers/opensearch/__init__.py | 0 .../providers/opensearch/hooks/__init__.py | 0 .../providers/opensearch/hooks/opensearch.py | 0 .../providers/opensearch/log/__init__.py | 0 .../opensearch/log/os_json_formatter.py | 0 .../providers/opensearch/log/os_response.py | 0 .../opensearch/log/os_task_handler.py | 0 .../opensearch/operators/__init__.py | 0 .../opensearch/operators/opensearch.py | 0 .../providers/opensearch/provider.yaml | 0 .../opsgenie/.latest-doc-only-change.txt | 0 .../airflow}/providers/opsgenie/CHANGELOG.rst | 0 .../airflow}/providers/opsgenie/__init__.py | 0 .../providers/opsgenie/hooks}/__init__.py | 0 .../providers/opsgenie/hooks/opsgenie.py | 0 .../opsgenie/notifications/__init__.py | 0 .../opsgenie/notifications/opsgenie.py | 0 .../providers/opsgenie/operators}/__init__.py | 0 .../providers/opsgenie/operators/opsgenie.py | 0 .../airflow}/providers/opsgenie/provider.yaml | 0 .../providers/opsgenie/typing/__init__.py | 0 .../providers/opsgenie/typing/opsgenie.py | 0 .../oracle/.latest-doc-only-change.txt | 0 .../airflow}/providers/oracle/CHANGELOG.rst | 0 .../src/airflow}/providers/oracle/__init__.py | 0 .../providers/oracle/example_dags/__init__.py | 0 .../oracle/example_dags/example_oracle.py | 0 .../providers/oracle}/hooks/__init__.py | 0 .../airflow}/providers/oracle/hooks/oracle.py | 0 .../providers/oracle}/operators/__init__.py | 0 .../providers/oracle/operators/oracle.py | 0 .../airflow}/providers/oracle/provider.yaml | 0 .../providers/oracle/transfers/__init__.py | 0 .../oracle/transfers/oracle_to_oracle.py | 0 .../pagerduty/.latest-doc-only-change.txt | 0 .../providers/pagerduty/CHANGELOG.rst | 0 .../airflow}/providers/pagerduty/__init__.py | 0 .../providers/pagerduty}/hooks/__init__.py | 0 .../providers/pagerduty/hooks/pagerduty.py | 0 .../pagerduty/hooks/pagerduty_events.py | 0 .../pagerduty/notifications/__init__.py | 0 .../pagerduty/notifications/pagerduty.py | 0 .../providers/pagerduty/provider.yaml | 0 .../papermill/.latest-doc-only-change.txt | 0 .../providers/papermill/CHANGELOG.rst | 0 .../airflow}/providers/papermill/__init__.py | 0 .../providers/papermill/hooks}/__init__.py | 0 .../providers/papermill/hooks/kernel.py | 0 .../papermill/operators}/__init__.py | 0 .../papermill/operators/papermill.py | 0 .../providers/papermill/provider.yaml | 0 .../pgvector/.latest-doc-only-change.txt | 0 .../airflow}/providers/pgvector/CHANGELOG.rst | 0 .../airflow}/providers/pgvector/__init__.py | 0 .../providers/pgvector/hooks/__init__.py | 0 .../providers/pgvector/hooks/pgvector.py | 0 .../providers/pgvector/operators/__init__.py | 0 .../providers/pgvector/operators/pgvector.py | 0 .../airflow}/providers/pgvector/provider.yaml | 0 .../pinecone/.latest-doc-only-change.txt | 0 .../airflow}/providers/pinecone/CHANGELOG.rst | 0 .../airflow}/providers/pinecone/__init__.py | 0 .../providers/pinecone/hooks/__init__.py | 0 .../providers/pinecone/hooks/pinecone.py | 0 .../providers/pinecone/operators/__init__.py | 0 .../providers/pinecone/operators/pinecone.py | 0 .../airflow}/providers/pinecone/provider.yaml | 0 .../postgres/.latest-doc-only-change.txt | 0 .../airflow}/providers/postgres/CHANGELOG.rst | 0 .../airflow}/providers/postgres/__init__.py | 0 .../providers/postgres/assets/__init__.py | 0 .../providers/postgres/assets/postgres.py | 0 .../providers/postgres}/hooks/__init__.py | 0 .../providers/postgres/hooks/postgres.py | 0 .../providers/postgres}/operators/__init__.py | 0 .../providers/postgres/operators/postgres.py | 0 .../airflow}/providers/postgres/provider.yaml | 0 .../presto/.latest-doc-only-change.txt | 0 .../airflow}/providers/presto/CHANGELOG.rst | 0 .../src/airflow}/providers/presto/__init__.py | 0 .../providers/presto}/hooks/__init__.py | 0 .../airflow}/providers/presto/hooks/presto.py | 0 .../airflow}/providers/presto/provider.yaml | 0 .../providers/presto/transfers/__init__.py | 0 .../presto/transfers/gcs_to_presto.py | 0 .../qdrant/.latest-doc-only-change.txt | 0 .../airflow}/providers/qdrant/CHANGELOG.rst | 0 .../src/airflow}/providers/qdrant/__init__.py | 0 .../providers/qdrant/hooks/__init__.py | 0 .../airflow}/providers/qdrant/hooks/qdrant.py | 0 .../providers/qdrant/operators/__init__.py | 0 .../providers/qdrant/operators/qdrant.py | 0 .../airflow}/providers/qdrant/provider.yaml | 0 .../redis/.latest-doc-only-change.txt | 0 .../airflow}/providers/redis/CHANGELOG.rst | 0 .../src/airflow}/providers/redis/__init__.py | 0 .../providers/redis/hooks}/__init__.py | 0 .../airflow}/providers/redis/hooks/redis.py | 0 .../airflow/providers/redis/log}/__init__.py | 0 .../providers/redis/log/redis_task_handler.py | 0 .../providers/redis/operators}/__init__.py | 0 .../redis/operators/redis_publish.py | 0 .../airflow}/providers/redis/provider.yaml | 0 .../providers/redis/sensors}/__init__.py | 0 .../providers/redis/sensors/redis_key.py | 0 .../providers/redis/sensors/redis_pub_sub.py | 0 .../salesforce/.latest-doc-only-change.txt | 0 .../providers/salesforce/CHANGELOG.rst | 0 .../airflow}/providers/salesforce/__init__.py | 0 .../providers/salesforce/hooks}/__init__.py | 0 .../providers/salesforce/hooks/salesforce.py | 0 .../salesforce/operators/__init__.py | 0 .../providers/salesforce/operators/bulk.py | 0 .../operators/salesforce_apex_rest.py | 0 .../providers/salesforce/provider.yaml | 0 .../samba/.latest-doc-only-change.txt | 0 .../airflow}/providers/samba/CHANGELOG.rst | 0 .../src/airflow}/providers/samba/__init__.py | 0 .../providers/samba/hooks}/__init__.py | 0 .../airflow}/providers/samba/hooks/samba.py | 0 .../airflow}/providers/samba/provider.yaml | 0 .../providers/samba/transfers/__init__.py | 0 .../providers/samba/transfers/gcs_to_samba.py | 0 .../segment/.latest-doc-only-change.txt | 0 .../airflow}/providers/segment/CHANGELOG.rst | 0 .../airflow}/providers/segment/__init__.py | 0 .../providers/segment}/hooks/__init__.py | 0 .../providers/segment/hooks/segment.py | 0 .../providers/segment/operators}/__init__.py | 0 .../segment/operators/segment_track_event.py | 0 .../airflow}/providers/segment/provider.yaml | 0 .../sendgrid/.latest-doc-only-change.txt | 0 .../airflow}/providers/sendgrid/CHANGELOG.rst | 0 .../airflow}/providers/sendgrid/__init__.py | 0 .../airflow}/providers/sendgrid/provider.yaml | 0 .../providers/sendgrid/utils/__init__.py | 0 .../providers/sendgrid/utils/emailer.py | 0 .../sftp/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/sftp/CHANGELOG.rst | 0 .../src/airflow}/providers/sftp/__init__.py | 0 .../providers/sftp/decorators/__init__.py | 0 .../sftp/decorators/sensors/__init__.py | 0 .../providers/sftp/decorators/sensors/sftp.py | 0 .../airflow}/providers/sftp/hooks/__init__.py | 0 .../src/airflow}/providers/sftp/hooks/sftp.py | 0 .../providers/sftp/operators/__init__.py | 0 .../airflow}/providers/sftp/operators/sftp.py | 0 .../src/airflow}/providers/sftp/provider.yaml | 0 .../providers/sftp/sensors/__init__.py | 0 .../airflow}/providers/sftp/sensors/sftp.py | 0 .../providers/sftp/triggers/__init__.py | 0 .../airflow}/providers/sftp/triggers/sftp.py | 0 .../singularity/.latest-doc-only-change.txt | 0 .../providers/singularity/CHANGELOG.rst | 0 .../providers/singularity/__init__.py | 0 .../singularity/operators}/__init__.py | 0 .../singularity/operators/singularity.py | 0 .../providers/singularity/provider.yaml | 0 .../slack/.latest-doc-only-change.txt | 0 .../airflow}/providers/slack/CHANGELOG.rst | 0 .../src/airflow}/providers/slack/__init__.py | 0 .../providers/slack/hooks}/__init__.py | 0 .../airflow}/providers/slack/hooks/slack.py | 0 .../providers/slack/hooks/slack_webhook.py | 0 .../providers/slack/notifications/__init__.py | 0 .../providers/slack/notifications/slack.py | 0 .../slack/notifications/slack_notifier.py | 0 .../slack/notifications/slack_webhook.py | 0 .../providers/slack}/operators/__init__.py | 0 .../providers/slack/operators/slack.py | 0 .../slack/operators/slack_webhook.py | 0 .../airflow}/providers/slack/provider.yaml | 0 .../providers/slack/transfers/__init__.py | 0 .../slack/transfers/base_sql_to_slack.py | 0 .../providers/slack/transfers/sql_to_slack.py | 0 .../slack/transfers/sql_to_slack_webhook.py | 0 .../providers/slack/utils/__init__.py | 0 .../smtp/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/smtp/CHANGELOG.rst | 0 .../src/airflow}/providers/smtp/__init__.py | 0 .../airflow/providers/smtp}/hooks/__init__.py | 0 .../src/airflow}/providers/smtp/hooks/smtp.py | 0 .../providers/smtp/notifications/__init__.py | 0 .../providers/smtp/notifications/smtp.py | 0 .../smtp/notifications/templates/__init__.py | 0 .../smtp/notifications/templates/email.html | 0 .../templates/email_subject.jinja2 | 0 .../providers/smtp}/operators/__init__.py | 0 .../airflow}/providers/smtp/operators/smtp.py | 0 .../src/airflow}/providers/smtp/provider.yaml | 0 .../snowflake/.latest-doc-only-change.txt | 0 .../providers/snowflake/CHANGELOG.rst | 0 .../airflow}/providers/snowflake/__init__.py | 0 .../snowflake/decorators/__init__.py | 0 .../snowflake/decorators/snowpark.py | 0 .../providers/snowflake/hooks/__init__.py | 0 .../providers/snowflake/hooks/snowflake.py | 0 .../snowflake/hooks/snowflake_sql_api.py | 7 +- .../providers/snowflake/operators/__init__.py | 0 .../snowflake/operators/snowflake.py | 0 .../providers/snowflake/operators/snowpark.py | 0 .../providers/snowflake/provider.yaml | 0 .../providers/snowflake/transfers/__init__.py | 0 .../transfers/copy_into_snowflake.py | 0 .../providers/snowflake/triggers/__init__.py | 0 .../snowflake/triggers/snowflake_trigger.py | 0 .../providers/snowflake/utils/__init__.py | 0 .../providers/snowflake/utils/common.py | 0 .../providers/snowflake/utils/openlineage.py | 0 .../providers/snowflake/utils/snowpark.py | 0 .../snowflake/utils/sql_api_generate_jwt.py | 0 .../sqlite/.latest-doc-only-change.txt | 0 .../airflow}/providers/sqlite/CHANGELOG.rst | 0 .../src/airflow}/providers/sqlite/__init__.py | 0 .../providers/sqlite}/hooks/__init__.py | 0 .../airflow}/providers/sqlite/hooks/sqlite.py | 0 .../providers/sqlite}/operators/__init__.py | 0 .../providers/sqlite/operators/sqlite.py | 0 .../airflow}/providers/sqlite/provider.yaml | 0 .../providers/ssh/.latest-doc-only-change.txt | 0 .../src/airflow}/providers/ssh/CHANGELOG.rst | 0 .../src/airflow}/providers/ssh/__init__.py | 0 .../airflow/providers/ssh}/hooks/__init__.py | 0 .../src/airflow}/providers/ssh/hooks/ssh.py | 0 .../providers/ssh}/operators/__init__.py | 0 .../airflow}/providers/ssh/operators/ssh.py | 0 .../src/airflow}/providers/ssh/provider.yaml | 0 .../airflow}/providers/standard/CHANGELOG.rst | 0 .../airflow/providers/standard}/__init__.py | 0 .../providers/standard/operators/__init__.py | 0 .../providers/standard/operators/bash.py | 0 .../providers/standard/operators/datetime.py | 0 .../providers/standard/operators/weekday.py | 0 .../airflow}/providers/standard/provider.yaml | 0 .../providers/standard/sensors/__init__.py | 0 .../providers/standard/sensors/bash.py | 0 .../providers/standard/sensors/date_time.py | 0 .../providers/standard/sensors/time.py | 0 .../providers/standard/sensors/time_delta.py | 0 .../providers/standard/sensors/weekday.py | 0 .../tableau/.latest-doc-only-change.txt | 0 .../airflow}/providers/tableau/CHANGELOG.rst | 0 .../airflow}/providers/tableau/__init__.py | 0 .../providers/tableau/hooks}/__init__.py | 0 .../providers/tableau/hooks/tableau.py | 0 .../providers/tableau/operators/__init__.py | 0 .../providers/tableau/operators/tableau.py | 0 .../airflow}/providers/tableau/provider.yaml | 0 .../providers/tableau/sensors/__init__.py | 0 .../providers/tableau/sensors/tableau.py | 0 .../telegram/.latest-doc-only-change.txt | 0 .../airflow}/providers/telegram/CHANGELOG.rst | 0 .../airflow}/providers/telegram/__init__.py | 0 .../providers/telegram/hooks}/__init__.py | 0 .../providers/telegram/hooks/telegram.py | 0 .../providers/telegram/operators}/__init__.py | 0 .../providers/telegram/operators/telegram.py | 0 .../airflow}/providers/telegram/provider.yaml | 0 .../teradata/.latest-doc-only-change.txt | 0 .../airflow}/providers/teradata/CHANGELOG.rst | 0 .../airflow}/providers/teradata/__init__.py | 0 .../providers/teradata}/hooks/__init__.py | 0 .../providers/teradata/hooks/teradata.py | 0 .../providers/teradata}/operators/__init__.py | 0 .../providers/teradata/operators/teradata.py | 0 .../operators/teradata_compute_cluster.py | 0 .../airflow}/providers/teradata/provider.yaml | 0 .../providers/teradata/transfers/__init__.py | 0 .../transfers/azure_blob_to_teradata.py | 0 .../teradata/transfers/s3_to_teradata.py | 0 .../transfers/teradata_to_teradata.py | 0 .../providers/teradata/triggers/__init__.py | 0 .../triggers/teradata_compute_cluster.py | 0 .../providers/teradata/utils/__init__.py | 0 .../providers/teradata/utils/constants.py | 0 .../trino/.latest-doc-only-change.txt | 0 .../airflow}/providers/trino/CHANGELOG.rst | 0 .../src/airflow}/providers/trino/__init__.py | 0 .../providers/trino/assets/__init__.py | 0 .../airflow}/providers/trino/assets/trino.py | 0 .../providers/trino}/hooks/__init__.py | 0 .../airflow}/providers/trino/hooks/trino.py | 0 .../providers/trino/operators/__init__.py | 0 .../providers/trino/operators/trino.py | 0 .../airflow}/providers/trino/provider.yaml | 0 .../providers/trino/transfers}/__init__.py | 0 .../providers/trino/transfers/gcs_to_trino.py | 0 .../vertica/.latest-doc-only-change.txt | 0 .../airflow}/providers/vertica/CHANGELOG.rst | 0 .../airflow}/providers/vertica/__init__.py | 0 .../providers/vertica}/hooks/__init__.py | 0 .../providers/vertica/hooks/vertica.py | 0 .../providers/vertica/operators}/__init__.py | 0 .../providers/vertica/operators/vertica.py | 0 .../airflow}/providers/vertica/provider.yaml | 0 .../weaviate/.latest-doc-only-change.txt | 0 .../airflow}/providers/weaviate/CHANGELOG.rst | 0 .../airflow}/providers/weaviate/__init__.py | 0 .../providers/weaviate/hooks/__init__.py | 0 .../providers/weaviate/hooks/weaviate.py | 0 .../providers/weaviate/operators/__init__.py | 0 .../providers/weaviate/operators/weaviate.py | 0 .../airflow}/providers/weaviate/provider.yaml | 0 .../yandex/.latest-doc-only-change.txt | 0 .../airflow}/providers/yandex/CHANGELOG.rst | 0 .../src/airflow}/providers/yandex/__init__.py | 0 .../providers/yandex/hooks/__init__.py | 0 .../providers/yandex/hooks/dataproc.py | 0 .../airflow}/providers/yandex/hooks/yandex.py | 0 .../yandex/hooks/yandexcloud_dataproc.py | 0 .../src/airflow}/providers/yandex/hooks/yq.py | 0 .../providers/yandex/links/__init__.py | 0 .../src/airflow}/providers/yandex/links/yq.py | 0 .../providers/yandex/operators/__init__.py | 0 .../providers/yandex/operators/dataproc.py | 0 .../yandex/operators/yandexcloud_dataproc.py | 0 .../airflow}/providers/yandex/operators/yq.py | 0 .../airflow}/providers/yandex/provider.yaml | 0 .../providers/yandex/secrets/__init__.py | 0 .../providers/yandex/secrets/lockbox.py | 0 .../providers/yandex/utils/__init__.py | 0 .../providers/yandex/utils/credentials.py | 0 .../providers/yandex/utils/defaults.py | 0 .../airflow}/providers/yandex/utils/fields.py | 0 .../providers/yandex/utils/user_agent.py | 0 .../src/airflow}/providers/ydb/CHANGELOG.rst | 0 .../src/airflow}/providers/ydb/__init__.py | 0 .../airflow}/providers/ydb/hooks/__init__.py | 0 .../providers/ydb/hooks/_vendor/__init__.py | 0 .../ydb/hooks/_vendor/dbapi/__init__.py | 0 .../ydb/hooks/_vendor/dbapi/connection.py | 0 .../ydb/hooks/_vendor/dbapi/constants.py | 0 .../ydb/hooks/_vendor/dbapi/cursor.py | 0 .../ydb/hooks/_vendor/dbapi/errors.py | 0 .../providers/ydb/hooks/_vendor/readme.md | 0 .../src/airflow}/providers/ydb/hooks/ydb.py | 0 .../providers/ydb/operators/__init__.py | 0 .../airflow}/providers/ydb/operators/ydb.py | 0 .../src/airflow}/providers/ydb/provider.yaml | 0 .../airflow}/providers/ydb/utils/__init__.py | 0 .../providers/ydb/utils/credentials.py | 0 .../airflow}/providers/ydb/utils/defaults.py | 0 .../zendesk/.latest-doc-only-change.txt | 0 .../airflow}/providers/zendesk/CHANGELOG.rst | 0 .../airflow}/providers/zendesk/__init__.py | 0 .../providers/zendesk}/hooks/__init__.py | 0 .../providers/zendesk/hooks/zendesk.py | 0 .../airflow}/providers/zendesk/provider.yaml | 0 .../tests}/__init__.py | 0 .../tests/airbyte}/__init__.py | 0 .../tests/airbyte/hooks}/__init__.py | 0 .../tests}/airbyte/hooks/test_airbyte.py | 0 .../tests/airbyte/operators}/__init__.py | 0 .../tests}/airbyte/operators/test_airbyte.py | 0 .../tests/airbyte/sensors}/__init__.py | 0 .../tests}/airbyte/sensors/test_airbyte.py | 0 .../tests/airbyte/triggers}/__init__.py | 0 .../tests}/airbyte/triggers/test_airbyte.py | 0 .../tests/alibaba}/__init__.py | 0 .../tests/alibaba/cloud}/__init__.py | 0 .../tests/alibaba/cloud/hooks}/__init__.py | 0 .../cloud/hooks/test_analyticdb_spark.py | 3 +- .../tests}/alibaba/cloud/hooks/test_oss.py | 3 +- .../tests/alibaba/cloud/log}/__init__.py | 0 .../cloud/log/test_oss_task_handler.py | 5 +- .../alibaba/cloud/operators}/__init__.py | 0 .../cloud/operators/test_analyticdb_spark.py | 0 .../alibaba/cloud/operators/test_oss.py | 0 .../tests/alibaba/cloud/sensors}/__init__.py | 0 .../cloud/sensors/test_analyticdb_spark.py | 0 .../alibaba/cloud/sensors/test_oss_key.py | 0 .../tests/alibaba/cloud/utils}/__init__.py | 0 .../cloud/utils/analyticdb_spark_mock.py | 0 .../tests}/alibaba/cloud/utils/oss_mock.py | 0 .../tests}/alibaba/cloud/utils/test_utils.py | 0 .../tests/amazon}/__init__.py | 0 .../tests}/amazon/aws/.gitignore | 0 .../tests/amazon/aws}/__init__.py | 0 .../tests/amazon/aws/assets}/__init__.py | 0 .../tests}/amazon/aws/assets/test_s3.py | 0 .../amazon/aws/auth_manager}/__init__.py | 0 .../amazon/aws/auth_manager/avp}/__init__.py | 0 .../aws/auth_manager/avp/test_entities.py | 0 .../aws/auth_manager/avp/test_facade.py | 11 +- .../amazon/aws/auth_manager/cli}/__init__.py | 0 .../aws/auth_manager/cli/test_avp_commands.py | 5 +- .../aws/auth_manager/cli/test_definition.py | 0 .../security_manager}/__init__.py | 0 .../test_aws_security_manager_override.py | 2 +- .../aws/auth_manager/test_aws_auth_manager.py | 26 +- .../amazon/aws/auth_manager/test_constants.py | 0 .../amazon/aws/auth_manager/test_user.py | 0 .../aws/auth_manager/views}/__init__.py | 0 .../aws/auth_manager/views/test_auth.py | 54 +- .../amazon/aws/config_templates}/__init__.py | 0 .../amazon/aws/config_templates/args.json | 0 .../amazon/aws/config_templates/job.j2.json | 0 .../amazon/aws/config_templates/steps.j2.json | 0 .../amazon/aws/config_templates/steps.json | 0 .../tests/amazon/aws/deferrable}/__init__.py | 0 .../amazon/aws/deferrable/hooks}/__init__.py | 0 .../tests/amazon/aws/executors}/__init__.py | 0 .../amazon/aws/executors/batch}/__init__.py | 0 .../executors/batch/test_batch_executor.py | 11 +- .../amazon/aws/executors/ecs}/__init__.py | 0 .../aws/executors/ecs/test_ecs_executor.py | 13 +- .../amazon/aws/executors/utils}/__init__.py | 0 .../utils/test_exponential_backoff_retry.py | 0 .../tests/amazon/aws/fs}/__init__.py | 0 .../tests}/amazon/aws/fs/test_s3.py | 0 .../tests/amazon/aws/hooks}/__init__.py | 0 .../tests}/amazon/aws/hooks/test_appflow.py | 0 .../tests}/amazon/aws/hooks/test_athena.py | 0 .../amazon/aws/hooks/test_athena_sql.py | 0 .../tests}/amazon/aws/hooks/test_base_aws.py | 33 +- .../amazon/aws/hooks/test_batch_client.py | 12 +- .../amazon/aws/hooks/test_batch_waiters.py | 13 +- .../tests}/amazon/aws/hooks/test_bedrock.py | 0 .../tests}/amazon/aws/hooks/test_chime.py | 0 .../amazon/aws/hooks/test_cloud_formation.py | 0 .../amazon/aws/hooks/test_comprehend.py | 0 .../tests}/amazon/aws/hooks/test_datasync.py | 0 .../tests}/amazon/aws/hooks/test_dms.py | 0 .../tests}/amazon/aws/hooks/test_dynamodb.py | 0 .../tests}/amazon/aws/hooks/test_ec2.py | 0 .../tests}/amazon/aws/hooks/test_ecr.py | 0 .../tests}/amazon/aws/hooks/test_ecs.py | 0 .../tests}/amazon/aws/hooks/test_eks.py | 5 +- .../test_elasticache_replication_group.py | 0 .../tests}/amazon/aws/hooks/test_emr.py | 0 .../amazon/aws/hooks/test_emr_containers.py | 0 .../amazon/aws/hooks/test_emr_serverless.py | 0 .../amazon/aws/hooks/test_eventbridge.py | 0 .../tests}/amazon/aws/hooks/test_glacier.py | 0 .../tests}/amazon/aws/hooks/test_glue.py | 0 .../amazon/aws/hooks/test_glue_catalog.py | 0 .../amazon/aws/hooks/test_glue_crawler.py | 0 .../amazon/aws/hooks/test_glue_databrew.py | 0 .../amazon/aws/hooks/test_hooks_signature.py | 0 .../tests}/amazon/aws/hooks/test_kinesis.py | 0 .../aws/hooks/test_kinesis_analytics.py | 0 .../amazon/aws/hooks/test_lambda_function.py | 0 .../tests}/amazon/aws/hooks/test_logs.py | 0 .../tests}/amazon/aws/hooks/test_neptune.py | 0 .../aws/hooks/test_opensearch_serverless.py | 0 .../amazon/aws/hooks/test_quicksight.py | 0 .../tests}/amazon/aws/hooks/test_rds.py | 0 .../amazon/aws/hooks/test_redshift_cluster.py | 0 .../amazon/aws/hooks/test_redshift_data.py | 0 .../amazon/aws/hooks/test_redshift_sql.py | 0 .../tests}/amazon/aws/hooks/test_s3.py | 3 +- .../tests}/amazon/aws/hooks/test_sagemaker.py | 0 .../amazon/aws/hooks/test_secrets_manager.py | 0 .../tests}/amazon/aws/hooks/test_ses.py | 0 .../tests}/amazon/aws/hooks/test_sns.py | 0 .../tests}/amazon/aws/hooks/test_sqs.py | 0 .../tests}/amazon/aws/hooks/test_ssm.py | 0 .../amazon/aws/hooks/test_step_function.py | 0 .../tests}/amazon/aws/hooks/test_sts.py | 0 .../aws/hooks/test_verified_permissions.py | 0 .../amazon/aws/infrastructure}/__init__.py | 0 .../example_s3_to_redshift}/__init__.py | 0 .../example_s3_to_redshift/outputs.tf | 0 .../example_s3_to_redshift/resources.tf | 0 .../example_s3_to_redshift/variables.tf | 0 .../tests/amazon/aws/links}/__init__.py | 0 .../tests}/amazon/aws/links/test_athena.py | 3 +- .../tests}/amazon/aws/links/test_base_aws.py | 5 +- .../tests}/amazon/aws/links/test_batch.py | 3 +- .../tests}/amazon/aws/links/test_emr.py | 3 +- .../tests}/amazon/aws/links/test_glue.py | 3 +- .../tests}/amazon/aws/links/test_logs.py | 3 +- .../amazon/aws/links/test_step_function.py | 3 +- .../tests/amazon/aws/log}/__init__.py | 0 .../aws/log/test_cloudwatch_task_handler.py | 3 +- .../amazon/aws/log/test_s3_task_handler.py | 3 +- .../amazon/aws/notifications}/__init__.py | 0 .../amazon/aws/notifications/test_chime.py | 0 .../amazon/aws/notifications/test_sns.py | 0 .../amazon/aws/notifications/test_sqs.py | 0 .../tests/amazon/aws/operators}/__init__.py | 0 .../amazon/aws/operators/athena_metadata.json | 0 .../amazon/aws/operators/test_appflow.py | 0 .../amazon/aws/operators/test_athena.py | 6 +- .../amazon/aws/operators/test_base_aws.py | 0 .../tests}/amazon/aws/operators/test_batch.py | 0 .../amazon/aws/operators/test_bedrock.py | 3 +- .../aws/operators/test_cloud_formation.py | 3 +- .../amazon/aws/operators/test_comprehend.py | 3 +- .../amazon/aws/operators/test_datasync.py | 3 +- .../tests}/amazon/aws/operators/test_dms.py | 3 +- .../tests}/amazon/aws/operators/test_ec2.py | 3 +- .../tests}/amazon/aws/operators/test_ecs.py | 3 +- .../tests}/amazon/aws/operators/test_eks.py | 9 +- .../aws/operators/test_emr_add_steps.py | 10 +- .../aws/operators/test_emr_containers.py | 3 +- .../aws/operators/test_emr_create_job_flow.py | 12 +- .../aws/operators/test_emr_modify_cluster.py | 3 +- .../operators/test_emr_notebook_execution.py | 5 +- .../aws/operators/test_emr_serverless.py | 3 +- .../operators/test_emr_terminate_job_flow.py | 3 +- .../amazon/aws/operators/test_eventbridge.py | 3 +- .../amazon/aws/operators/test_glacier.py | 3 +- .../tests}/amazon/aws/operators/test_glue.py | 3 +- .../amazon/aws/operators/test_glue_crawler.py | 3 +- .../aws/operators/test_glue_databrew.py | 3 +- .../aws/operators/test_kinesis_analytics.py | 3 +- .../aws/operators/test_lambda_function.py | 3 +- .../amazon/aws/operators/test_neptune.py | 3 +- .../amazon/aws/operators/test_quicksight.py | 3 +- .../tests}/amazon/aws/operators/test_rds.py | 3 +- .../aws/operators/test_redshift_cluster.py | 3 +- .../aws/operators/test_redshift_data.py | 3 +- .../amazon/aws/operators/test_redshift_sql.py | 0 .../tests}/amazon/aws/operators/test_s3.py | 3 +- .../aws/operators/test_sagemaker_base.py | 3 +- .../aws/operators/test_sagemaker_endpoint.py | 3 +- .../test_sagemaker_endpoint_config.py | 3 +- .../aws/operators/test_sagemaker_model.py | 3 +- .../aws/operators/test_sagemaker_notebook.py | 3 +- .../aws/operators/test_sagemaker_pipeline.py | 3 +- .../operators/test_sagemaker_processing.py | 3 +- .../aws/operators/test_sagemaker_training.py | 3 +- .../aws/operators/test_sagemaker_transform.py | 3 +- .../aws/operators/test_sagemaker_tuning.py | 3 +- .../tests}/amazon/aws/operators/test_sns.py | 3 +- .../tests}/amazon/aws/operators/test_sqs.py | 3 +- .../aws/operators/test_step_function.py | 16 +- .../tests/amazon/aws/secrets}/__init__.py | 0 .../aws/secrets/test_secrets_manager.py | 0 .../aws/secrets/test_systems_manager.py | 3 +- .../tests/amazon/aws/sensors}/__init__.py | 0 .../tests}/amazon/aws/sensors/test_athena.py | 0 .../amazon/aws/sensors/test_base_aws.py | 0 .../tests}/amazon/aws/sensors/test_batch.py | 0 .../tests}/amazon/aws/sensors/test_bedrock.py | 0 .../aws/sensors/test_cloud_formation.py | 0 .../amazon/aws/sensors/test_comprehend.py | 0 .../tests}/amazon/aws/sensors/test_dms.py | 0 .../amazon/aws/sensors/test_dynamodb.py | 0 .../tests}/amazon/aws/sensors/test_ec2.py | 0 .../tests}/amazon/aws/sensors/test_ecs.py | 0 .../tests}/amazon/aws/sensors/test_eks.py | 0 .../amazon/aws/sensors/test_emr_base.py | 0 .../amazon/aws/sensors/test_emr_containers.py | 0 .../amazon/aws/sensors/test_emr_job_flow.py | 0 .../sensors/test_emr_notebook_execution.py | 0 .../test_emr_serverless_application.py | 0 .../aws/sensors/test_emr_serverless_job.py | 0 .../amazon/aws/sensors/test_emr_step.py | 0 .../tests}/amazon/aws/sensors/test_glacier.py | 0 .../tests}/amazon/aws/sensors/test_glue.py | 0 .../sensors/test_glue_catalog_partition.py | 0 .../amazon/aws/sensors/test_glue_crawler.py | 0 .../aws/sensors/test_glue_data_quality.py | 0 .../aws/sensors/test_kinesis_analytics.py | 0 .../aws/sensors/test_lambda_function.py | 0 .../aws/sensors/test_opensearch_serverless.py | 0 .../amazon/aws/sensors/test_quicksight.py | 0 .../tests}/amazon/aws/sensors/test_rds.py | 0 .../aws/sensors/test_redshift_cluster.py | 0 .../tests}/amazon/aws/sensors/test_s3.py | 0 .../aws/sensors/test_sagemaker_automl.py | 0 .../amazon/aws/sensors/test_sagemaker_base.py | 0 .../aws/sensors/test_sagemaker_endpoint.py | 0 .../aws/sensors/test_sagemaker_pipeline.py | 0 .../aws/sensors/test_sagemaker_training.py | 0 .../aws/sensors/test_sagemaker_transform.py | 0 .../aws/sensors/test_sagemaker_tuning.py | 0 .../tests}/amazon/aws/sensors/test_sqs.py | 0 .../amazon/aws/sensors/test_step_function.py | 0 .../tests/amazon/aws/system}/__init__.py | 0 .../amazon/aws/system/utils}/__init__.py | 0 .../amazon/aws/system/utils/test_helpers.py | 6 +- .../tests/amazon/aws/transfers}/__init__.py | 0 .../aws/transfers/test_azure_blob_to_s3.py | 0 .../tests}/amazon/aws/transfers/test_base.py | 0 .../aws/transfers/test_dynamodb_to_s3.py | 0 .../amazon/aws/transfers/test_exasol_to_s3.py | 0 .../amazon/aws/transfers/test_ftp_to_s3.py | 0 .../amazon/aws/transfers/test_gcs_to_s3.py | 0 .../aws/transfers/test_glacier_to_gcs.py | 0 .../aws/transfers/test_google_api_to_s3.py | 0 .../aws/transfers/test_hive_to_dynamodb.py | 0 .../amazon/aws/transfers/test_http_to_s3.py | 0 .../transfers/test_imap_attachment_to_s3.py | 0 .../amazon/aws/transfers/test_local_to_s3.py | 0 .../amazon/aws/transfers/test_mongo_to_s3.py | 0 .../aws/transfers/test_redshift_to_s3.py | 3 +- .../aws/transfers/test_s3_to_dynamodb.py | 0 .../amazon/aws/transfers/test_s3_to_ftp.py | 0 .../aws/transfers/test_s3_to_redshift.py | 3 +- .../amazon/aws/transfers/test_s3_to_sftp.py | 3 +- .../amazon/aws/transfers/test_s3_to_sql.py | 0 .../aws/transfers/test_salesforce_to_s3.py | 0 .../amazon/aws/transfers/test_sftp_to_s3.py | 3 +- .../amazon/aws/transfers/test_sql_to_s3.py | 0 .../tests/amazon/aws/triggers}/__init__.py | 0 .../tests}/amazon/aws/triggers/test_athena.py | 0 .../tests}/amazon/aws/triggers/test_base.py | 0 .../tests}/amazon/aws/triggers/test_batch.py | 0 .../amazon/aws/triggers/test_bedrock.py | 3 +- .../amazon/aws/triggers/test_comprehend.py | 3 +- .../tests}/amazon/aws/triggers/test_ec2.py | 0 .../tests}/amazon/aws/triggers/test_ecs.py | 0 .../tests}/amazon/aws/triggers/test_eks.py | 0 .../tests}/amazon/aws/triggers/test_emr.py | 0 .../tests}/amazon/aws/triggers/test_glue.py | 3 +- .../amazon/aws/triggers/test_glue_crawler.py | 3 +- .../amazon/aws/triggers/test_glue_databrew.py | 0 .../aws/triggers/test_kinesis_analytics.py | 3 +- .../aws/triggers/test_lambda_function.py | 0 .../amazon/aws/triggers/test_neptune.py | 0 .../triggers/test_opensearch_serverless.py | 3 +- .../tests}/amazon/aws/triggers/test_rds.py | 0 .../aws/triggers/test_redshift_cluster.py | 0 .../amazon/aws/triggers/test_redshift_data.py | 0 .../tests}/amazon/aws/triggers/test_s3.py | 0 .../amazon/aws/triggers/test_sagemaker.py | 0 .../amazon/aws/triggers/test_serialization.py | 0 .../tests}/amazon/aws/triggers/test_sqs.py | 0 .../tests/amazon/aws/utils}/__init__.py | 0 .../amazon/aws/utils/eks_test_constants.py | 0 .../tests}/amazon/aws/utils/eks_test_utils.py | 2 +- .../aws/utils/test_connection_wrapper.py | 0 .../amazon/aws/utils/test_eks_get_token.py | 4 +- .../tests}/amazon/aws/utils/test_emailer.py | 0 .../amazon/aws/utils/test_identifiers.py | 0 .../tests}/amazon/aws/utils/test_mixins.py | 0 .../amazon/aws/utils/test_openlineage.py | 0 .../tests}/amazon/aws/utils/test_redshift.py | 0 .../tests}/amazon/aws/utils/test_sqs.py | 0 .../tests}/amazon/aws/utils/test_suppress.py | 0 .../tests}/amazon/aws/utils/test_tags.py | 0 .../amazon/aws/utils/test_task_log_fetcher.py | 0 .../amazon/aws/utils/test_template_fields.py | 0 .../tests}/amazon/aws/utils/test_utils.py | 0 .../tests}/amazon/aws/utils/test_waiter.py | 0 .../aws/utils/test_waiter_with_logging.py | 0 .../tests/amazon/aws/waiters}/__init__.py | 0 .../tests}/amazon/aws/waiters/test.json | 0 .../tests}/amazon/aws/waiters/test_batch.py | 0 .../tests}/amazon/aws/waiters/test_bedrock.py | 0 .../amazon/aws/waiters/test_bedrock_agent.py | 0 .../amazon/aws/waiters/test_comprehend.py | 0 .../amazon/aws/waiters/test_custom_waiters.py | 0 .../tests}/amazon/aws/waiters/test_dynamo.py | 0 .../tests}/amazon/aws/waiters/test_ecs.py | 0 .../tests}/amazon/aws/waiters/test_eks.py | 3 +- .../tests}/amazon/aws/waiters/test_emr.py | 0 .../tests}/amazon/aws/waiters/test_glue.py | 0 .../amazon/aws/waiters/test_glue_databrew.py | 0 .../aws/waiters/test_kinesis_analytics.py | 0 .../tests}/amazon/aws/waiters/test_neptune.py | 0 .../aws/waiters/test_opensearch_serverless.py | 0 .../tests}/amazon/conftest.py | 0 .../tests/apache}/__init__.py | 0 .../tests/apache/beam}/__init__.py | 0 .../tests/apache/beam/hooks}/__init__.py | 0 .../tests}/apache/beam/hooks/test_beam.py | 0 .../tests/apache/beam/operators}/__init__.py | 0 .../tests}/apache/beam/operators/test_beam.py | 0 .../tests/apache/beam/triggers}/__init__.py | 0 .../tests}/apache/beam/triggers/test_beam.py | 0 .../tests/apache/cassandra}/__init__.py | 0 .../apache/cassandra}/sensors/__init__.py | 0 .../apache/cassandra/sensors/test_record.py | 0 .../apache/cassandra/sensors/test_table.py | 0 .../tests/apache/drill}/__init__.py | 0 .../tests/apache/drill}/hooks/__init__.py | 0 .../tests}/apache/drill/hooks/test_drill.py | 7 +- .../tests/apache/druid}/__init__.py | 0 .../tests/apache/druid}/hooks/__init__.py | 0 .../tests}/apache/druid/hooks/test_druid.py | 0 .../tests/apache/druid/operators}/__init__.py | 0 .../apache/druid/operators/test_druid.py | 0 .../tests/apache/druid/transfers}/__init__.py | 0 .../druid/transfers/test_hive_to_druid.py | 0 .../tests/apache/flink}/__init__.py | 0 .../tests/apache/flink}/operators/__init__.py | 0 .../flink/operators/test_flink_kubernetes.py | 0 .../tests/apache/flink/sensors}/__init__.py | 0 .../flink/sensors/test_flink_kubernetes.py | 0 .../tests/apache/hdfs}/__init__.py | 0 .../tests/apache/hdfs/hooks}/__init__.py | 0 .../tests}/apache/hdfs/hooks/test_webhdfs.py | 0 .../tests/apache/hdfs/sensors}/__init__.py | 0 .../apache/hdfs/sensors/test_web_hdfs.py | 0 .../tests}/apache/hive/__init__.py | 0 .../tests/apache/hive}/hooks/__init__.py | 0 .../apache/hive/hooks/query_results.csv | 0 .../tests}/apache/hive/hooks/test_hive.py | 27 +- .../tests/apache/hive/macros}/__init__.py | 0 .../tests}/apache/hive/macros/test_hive.py | 0 .../tests/apache/hive/operators}/__init__.py | 0 .../tests}/apache/hive/operators/test_hive.py | 3 +- .../apache/hive/operators/test_hive_stats.py | 3 +- .../tests/apache/hive/sensors}/__init__.py | 0 .../hive/sensors/test_hive_partition.py | 3 +- .../hive/sensors/test_metastore_partition.py | 3 +- .../hive/sensors/test_named_hive_partition.py | 3 +- .../tests/apache/hive/transfers}/__init__.py | 0 .../hive/transfers/test_hive_to_mysql.py | 3 +- .../hive/transfers/test_hive_to_samba.py | 3 +- .../hive/transfers/test_mssql_to_hive.py | 0 .../hive/transfers/test_mysql_to_hive.py | 0 .../apache/hive/transfers/test_s3_to_hive.py | 0 .../hive/transfers/test_vertica_to_hive.py | 0 .../tests/apache/iceberg}/__init__.py | 0 .../tests/apache/iceberg/hooks}/__init__.py | 0 .../apache/iceberg/hooks/test_iceberg.py | 0 .../tests/apache/impala}/__init__.py | 0 .../tests/apache/impala/hooks}/__init__.py | 0 .../tests}/apache/impala/hooks/test_impala.py | 0 .../tests/apache/kafka}/__init__.py | 0 .../tests/apache/kafka/hooks}/__init__.py | 0 .../tests}/apache/kafka/hooks/test_base.py | 0 .../tests}/apache/kafka/hooks/test_client.py | 0 .../tests}/apache/kafka/hooks/test_consume.py | 0 .../tests}/apache/kafka/hooks/test_produce.py | 0 .../tests/apache/kafka/operators}/__init__.py | 0 .../apache/kafka/operators/test_consume.py | 2 +- .../apache/kafka/operators/test_produce.py | 2 +- .../tests/apache/kafka/sensors}/__init__.py | 0 .../tests}/apache/kafka/sensors/test_kafka.py | 0 .../tests/apache/kafka/triggers}/__init__.py | 0 .../kafka/triggers/test_await_message.py | 4 +- .../tests/apache/kylin}/__init__.py | 0 .../tests/apache/kylin/hooks}/__init__.py | 0 .../tests}/apache/kylin/hooks/test_kylin.py | 0 .../tests/apache/kylin/operators}/__init__.py | 0 .../apache/kylin/operators/test_kylin_cube.py | 0 .../tests/apache/livy}/__init__.py | 0 .../tests/apache/livy/hooks}/__init__.py | 0 .../tests}/apache/livy/hooks/test_livy.py | 3 +- .../tests/apache/livy/operators}/__init__.py | 0 .../tests}/apache/livy/operators/test_livy.py | 0 .../tests/apache/livy/sensors}/__init__.py | 0 .../tests}/apache/livy/sensors/test_livy.py | 0 .../tests/apache/livy/triggers}/__init__.py | 0 .../tests}/apache/livy/triggers/test_livy.py | 0 .../tests/apache/pig}/__init__.py | 0 .../tests/apache/pig/hooks}/__init__.py | 0 .../tests}/apache/pig/hooks/test_pig.py | 0 .../tests/apache/pig/operators}/__init__.py | 0 .../tests}/apache/pig/operators/test_pig.py | 0 .../tests/apache/pinot}/__init__.py | 0 .../tests/apache/pinot}/hooks/__init__.py | 0 .../tests}/apache/pinot/hooks/test_pinot.py | 0 .../tests/apache/spark}/__init__.py | 0 .../apache/spark/decorators}/__init__.py | 0 .../apache/spark/decorators/test_pyspark.py | 0 .../tests/apache/spark}/hooks/__init__.py | 0 .../apache/spark/hooks/test_spark_connect.py | 0 .../apache/spark/hooks/test_spark_jdbc.py | 0 .../spark/hooks/test_spark_jdbc_script.py | 0 .../apache/spark/hooks/test_spark_sql.py | 3 +- .../apache/spark/hooks/test_spark_submit.py | 0 .../tests/apache/spark/operators}/__init__.py | 0 .../apache/spark/operators/test_spark_jdbc.py | 0 .../apache/spark/operators/test_spark_sql.py | 0 .../spark/operators/test_spark_submit.py | 0 .../tests/apprise}/__init__.py | 0 .../tests/apprise}/hooks/__init__.py | 0 .../tests}/apprise/hooks/test_apprise.py | 0 .../tests/apprise/notifications}/__init__.py | 0 .../apprise/notifications/test_apprise.py | 0 .../tests/arangodb}/__init__.py | 0 .../tests/arangodb/hooks}/__init__.py | 0 .../tests}/arangodb/hooks/test_arangodb.py | 0 .../tests/arangodb}/operators/__init__.py | 0 .../arangodb/operators/test_arangodb.py | 0 .../tests/arangodb/sensors}/__init__.py | 0 .../tests}/arangodb/sensors/test_arangodb.py | 0 .../tests/asana}/__init__.py | 0 .../tests/asana/hooks}/__init__.py | 0 .../tests}/asana/hooks/test_asana.py | 0 .../tests/asana/operators}/__init__.py | 0 .../asana/operators/test_asana_tasks.py | 0 .../tests/atlassian}/__init__.py | 0 .../tests/atlassian/jira}/__init__.py | 0 .../tests/atlassian/jira/hooks}/__init__.py | 0 .../tests}/atlassian/jira/hooks/test_jira.py | 3 +- .../atlassian/jira/notifications}/__init__.py | 0 .../atlassian/jira/notifications/test_jira.py | 0 .../atlassian/jira/operators}/__init__.py | 0 .../atlassian/jira/operators/test_jira.py | 3 +- .../tests/atlassian/jira/sensors}/__init__.py | 0 .../atlassian/jira/sensors/test_jira.py | 3 +- .../tests/celery}/__init__.py | 0 .../tests/celery/cli}/__init__.py | 0 .../tests}/celery/cli/test_celery_command.py | 5 +- .../tests/celery/executors}/__init__.py | 0 .../celery/executors/test_celery_executor.py | 21 +- .../test_celery_kubernetes_executor.py | 0 .../tests/celery/log_handlers}/__init__.py | 0 .../celery/log_handlers/test_log_handlers.py | 5 +- .../tests/celery/sensors}/__init__.py | 0 .../celery/sensors/test_celery_queue.py | 0 .../tests/cloudant}/__init__.py | 0 .../tests/cloudant/hooks}/__init__.py | 0 .../tests}/cloudant/hooks/test_cloudant.py | 0 .../beam => providers/tests/cncf}/__init__.py | 0 .../tests/cncf/kubernetes}/__init__.py | 0 .../tests/cncf/kubernetes/cli}/__init__.py | 0 .../kubernetes/cli/test_kubernetes_command.py | 3 +- .../tests}/cncf/kubernetes/conftest.py | 0 .../cncf/kubernetes/data_files}/__init__.py | 0 .../data_files/executor}/__init__.py | 0 .../data_files/executor/basic_template.yaml | 0 .../cncf/kubernetes/data_files/kube_config | 0 .../kubernetes/data_files/pods}/__init__.py | 0 .../data_files/pods/generator_base.yaml | 0 .../pods/generator_base_with_secrets.yaml | 0 .../kubernetes/data_files/pods/template.yaml | 0 .../kubernetes/data_files/spark}/__init__.py | 0 .../spark/application_template.yaml | 0 .../data_files/spark/application_test.json | 0 .../data_files/spark/application_test.yaml | 0 ...ication_test_with_no_name_from_config.json | 0 ...ication_test_with_no_name_from_config.yaml | 0 .../cncf/kubernetes/decorators}/__init__.py | 0 .../kubernetes/decorators/test_kubernetes.py | 0 .../cncf/kubernetes/executors}/__init__.py | 0 .../executors/test_kubernetes_executor.py | 20 +- .../test_local_kubernetes_executor.py | 0 .../tests/cncf/kubernetes/hooks}/__init__.py | 0 .../cncf/kubernetes/hooks/test_kubernetes.py | 10 +- .../cncf/kubernetes/log_handlers}/__init__.py | 0 .../log_handlers/test_log_handlers.py | 5 +- .../tests/cncf/kubernetes/models}/__init__.py | 0 .../cncf/kubernetes/models/test_secret.py | 0 .../cncf/kubernetes/operators}/__init__.py | 0 .../operators/test_custom_object_launcher.py | 0 .../cncf/kubernetes/operators/test_job.py | 0 .../cncf/kubernetes/operators/test_pod.py | 9 +- .../kubernetes/operators/test_resource.py | 0 .../operators/test_spark_kubernetes.py | 3 +- .../kubernetes/resource_convert}/__init__.py | 0 .../resource_convert/test_configmap.py | 0 .../resource_convert/test_env_variable.py | 0 .../resource_convert/test_secret.py | 0 .../cncf/kubernetes/sensors}/__init__.py | 0 .../sensors/test_spark_kubernetes.py | 0 .../tests}/cncf/kubernetes/test_callbacks.py | 0 .../tests}/cncf/kubernetes/test_client.py | 3 +- .../test_kubernetes_helper_functions.py | 0 .../cncf/kubernetes/test_pod_generator.py | 0 .../kubernetes/test_template_rendering.py | 10 +- .../cncf/kubernetes/triggers}/__init__.py | 0 .../cncf/kubernetes/triggers/test_job.py | 0 .../cncf/kubernetes/triggers/test_pod.py | 0 .../tests/cncf/kubernetes/utils}/__init__.py | 0 .../utils/test_k8s_resource_iterator.py | 0 .../cncf/kubernetes/utils/test_pod_manager.py | 3 +- .../tests/cohere}/__init__.py | 0 .../tests/cohere/hooks}/__init__.py | 0 .../tests}/cohere/hooks/test_cohere.py | 13 +- .../tests/cohere/operators}/__init__.py | 0 .../tests}/cohere/operators/test_embedding.py | 0 .../tests/common}/__init__.py | 0 .../tests/common/compat}/__init__.py | 0 .../tests/common/compat/lineage}/__init__.py | 0 .../tests}/common/compat/lineage/test_hook.py | 0 .../common/compat/openlineage}/__init__.py | 0 .../common/compat/openlineage/test_facet.py | 0 .../compat/openlineage/utils}/__init__.py | 0 .../compat/openlineage/utils/test_utils.py | 0 .../tests/common/compat/security}/__init__.py | 0 .../compat/security/test_permissions.py | 0 .../tests/common/io}/__init__.py | 0 .../tests/common/io/assets}/__init__.py | 0 .../tests}/common/io/assets/test_file.py | 0 .../tests/common/io/operators}/__init__.py | 0 .../common/io/operators/test_file_transfer.py | 3 +- .../tests/common/io/xcom}/__init__.py | 0 .../tests}/common/io/xcom/test_backend.py | 9 +- .../tests/common/sql}/__init__.py | 0 .../tests/common/sql/hooks}/__init__.py | 0 .../tests}/common/sql/hooks/test_dbapi.py | 3 +- .../tests}/common/sql/hooks/test_sql.py | 5 +- .../tests}/common/sql/hooks/test_sqlparse.py | 3 +- .../tests/common/sql/operators}/__init__.py | 0 .../tests}/common/sql/operators/test_sql.py | 3 +- .../common/sql/operators/test_sql_execute.py | 3 +- .../tests/common/sql/sensors}/__init__.py | 0 .../tests}/common/sql/sensors/test_sql.py | 3 +- .../tests}/common/sql/test_utils.py | 3 +- .../providers => providers/tests}/conftest.py | 13 +- .../tests/databricks}/__init__.py | 0 .../tests/databricks/hooks}/__init__.py | 0 .../databricks/hooks/test_databricks.py | 0 .../databricks/hooks/test_databricks_base.py | 0 .../databricks/hooks/test_databricks_sql.py | 35 + .../tests/databricks/operators}/__init__.py | 0 .../databricks/operators/test_databricks.py | 0 .../operators/test_databricks_copy.py | 0 .../operators/test_databricks_repos.py | 0 .../operators/test_databricks_sql.py | 0 .../operators/test_databricks_workflow.py | 0 .../tests/databricks/plugins}/__init__.py | 0 .../plugins/test_databricks_workflow.py | 5 + .../tests/databricks/sensors}/__init__.py | 0 .../sensors/test_databricks_partition.py | 0 .../databricks/sensors/test_databricks_sql.py | 0 .../tests}/databricks/test_exceptions.py | 0 .../tests/databricks/triggers}/__init__.py | 0 .../databricks/triggers/test_databricks.py | 0 .../tests/databricks/utils}/__init__.py | 0 .../databricks/utils/test_databricks.py | 0 .../tests/datadog}/__init__.py | 0 .../tests/datadog/hooks}/__init__.py | 0 .../tests}/datadog/hooks/test_datadog.py | 0 .../tests/datadog/sensors}/__init__.py | 0 .../tests}/datadog/sensors/test_datadog.py | 0 .../cli => providers/tests/dbt}/__init__.py | 0 .../tests/dbt/cloud}/__init__.py | 0 .../tests/dbt/cloud/hooks}/__init__.py | 0 .../tests}/dbt/cloud/hooks/test_dbt.py | 5 +- .../tests/dbt/cloud/operators}/__init__.py | 0 .../tests}/dbt/cloud/operators/test_dbt.py | 5 +- .../tests/dbt/cloud/sensors}/__init__.py | 0 .../tests}/dbt/cloud/sensors/test_dbt.py | 0 .../tests/dbt/cloud/test_data}/__init__.py | 0 .../tests}/dbt/cloud/test_data/catalog.json | 0 .../tests}/dbt/cloud/test_data/job_run.json | 0 .../tests}/dbt/cloud/test_data/manifest.json | 0 .../dbt/cloud/test_data/run_results.json | 0 .../tests/dbt/cloud/triggers}/__init__.py | 0 .../tests}/dbt/cloud/triggers/test_dbt.py | 0 .../tests/dbt/cloud/utils}/__init__.py | 0 .../dbt/cloud/utils/test_openlineage.py | 0 providers/tests/deprecations_ignore.yml | 128 ++ .../tests/dingding}/__init__.py | 0 .../tests/dingding/hooks}/__init__.py | 0 .../tests}/dingding/hooks/test_dingding.py | 0 .../tests/dingding/operators}/__init__.py | 0 .../dingding/operators/test_dingding.py | 0 .../tests/discord}/__init__.py | 0 .../tests/discord/hooks}/__init__.py | 0 .../discord/hooks/test_discord_webhook.py | 0 .../tests/discord/notifications}/__init__.py | 0 .../discord/notifications/test_discord.py | 0 .../tests/discord/operators}/__init__.py | 0 .../discord/operators/test_discord_webhook.py | 0 .../tests/docker}/__init__.py | 0 .../tests}/docker/conftest.py | 0 .../tests/docker/decorators}/__init__.py | 0 .../tests}/docker/decorators/test_docker.py | 0 .../tests/docker/hooks}/__init__.py | 0 .../tests}/docker/hooks/test_docker.py | 0 .../tests/docker/operators}/__init__.py | 0 .../tests}/docker/operators/test_docker.py | 5 +- .../docker/operators/test_docker_swarm.py | 0 .../tests}/docker/test_exceptions.py | 0 .../tests/edge}/__init__.py | 0 .../tests/edge/api_endpoints}/__init__.py | 0 .../api_endpoints/test_health_endpoint.py | 0 .../api_endpoints/test_rpc_api_endpoint.py | 5 +- .../tests/edge/cli}/__init__.py | 0 .../tests}/edge/cli/test_edge_command.py | 3 +- .../tests/edge/models}/__init__.py | 0 .../tests}/edge/models/test_edge_job.py | 0 .../tests}/edge/models/test_edge_logs.py | 0 .../tests}/edge/models/test_edge_worker.py | 0 .../tests/edge/plugins}/__init__.py | 0 .../edge/plugins/test_edge_executor_plugin.py | 3 +- .../tests/elasticsearch}/__init__.py | 0 .../tests/elasticsearch/hooks}/__init__.py | 0 .../elasticsearch/hooks/test_elasticsearch.py | 0 .../tests/elasticsearch/log}/__init__.py | 0 .../elasticsearch/log/elasticmock/__init__.py | 2 +- .../log/elasticmock/fake_elasticsearch.py | 2 +- .../log/elasticmock/utilities/__init__.py | 0 .../log/test_es_json_formatter.py | 0 .../elasticsearch/log/test_es_response.py | 0 .../elasticsearch/log/test_es_task_handler.py | 9 +- .../tests/email}/__init__.py | 0 .../tests/email/operators}/__init__.py | 0 .../tests/exasol}/__init__.py | 0 .../tests/exasol/hooks}/__init__.py | 0 .../tests}/exasol/hooks/test_exasol.py | 0 .../tests}/exasol/hooks/test_sql.py | 0 .../tests/exasol/operators}/__init__.py | 0 .../tests}/exasol/operators/test_exasol.py | 0 .../exasol/operators/test_exasol_sql.py | 0 .../hooks => providers/tests/fab}/__init__.py | 0 .../tests/fab/auth_manager}/__init__.py | 0 .../tests/fab/auth_manager/api}/__init__.py | 0 .../fab/auth_manager/api/auth}/__init__.py | 0 .../api/auth/backend}/__init__.py | 0 .../api/auth/backend/test_basic_auth.py | 3 +- .../api/auth/backend/test_kerberos_auth.py | 2 +- .../auth_manager/api_endpoints}/__init__.py | 0 .../api_endpoints/api_connexion_utils.py | 2 +- .../remote_user_api_auth_backend.py | 0 .../api_endpoints/test_asset_endpoint.py | 13 +- .../auth_manager/api_endpoints/test_auth.py | 10 +- .../api_endpoints/test_backfill_endpoint.py | 24 +- .../auth_manager/api_endpoints/test_cors.py | 6 +- .../api_endpoints/test_dag_endpoint.py | 9 +- .../api_endpoints/test_dag_run_endpoint.py | 15 +- .../api_endpoints/test_dag_source_endpoint.py | 7 +- .../test_dag_warning_endpoint.py | 7 +- .../api_endpoints/test_event_log_endpoint.py | 7 +- .../test_import_error_endpoint.py | 9 +- .../test_role_and_permission_endpoint.py | 11 +- .../test_role_and_permission_schema.py | 3 +- .../test_task_instance_endpoint.py | 7 +- .../api_endpoints/test_user_endpoint.py | 15 +- .../api_endpoints/test_user_schema.py | 7 +- .../api_endpoints/test_variable_endpoint.py | 7 +- .../api_endpoints/test_xcom_endpoint.py | 7 +- .../auth_manager/cli_commands}/__init__.py | 0 .../cli_commands/test_db_command.py | 0 .../cli_commands/test_definition.py | 2 +- .../cli_commands/test_role_command.py | 5 +- .../cli_commands/test_sync_perm_command.py | 3 +- .../cli_commands/test_user_command.py | 3 +- .../auth_manager/cli_commands/test_utils.py | 2 +- .../tests}/fab/auth_manager/conftest.py | 7 +- .../fab/auth_manager/decorators}/__init__.py | 0 .../fab/auth_manager/decorators/test_auth.py | 3 +- .../fab/auth_manager/models}/__init__.py | 0 .../models/test_anonymous_user.py | 2 +- .../tests}/fab/auth_manager/models/test_db.py | 13 +- .../security_manager}/__init__.py | 0 .../security_manager/test_constants.py | 2 +- .../security_manager/test_override.py | 2 +- .../fab/auth_manager/test_fab_auth_manager.py | 2 +- .../tests}/fab/auth_manager/test_models.py | 2 +- .../tests}/fab/auth_manager/test_security.py | 14 +- .../tests/fab/auth_manager/views}/__init__.py | 0 .../auth_manager/views/test_permissions.py | 7 +- .../fab/auth_manager/views/test_roles_list.py | 7 +- .../fab/auth_manager/views/test_user.py | 7 +- .../fab/auth_manager/views/test_user_edit.py | 7 +- .../fab/auth_manager/views/test_user_stats.py | 7 +- .../tests/facebook}/__init__.py | 0 .../tests/facebook/ads}/__init__.py | 0 .../tests/facebook/ads/hooks}/__init__.py | 0 .../tests}/facebook/ads/hooks/test_ads.py | 0 .../tests/ftp}/__init__.py | 0 .../tests/ftp/hooks}/__init__.py | 0 .../tests}/ftp/hooks/test_ftp.py | 0 .../tests/ftp/operators}/__init__.py | 0 .../tests}/ftp/operators/test_ftp.py | 0 .../tests/ftp/sensors}/__init__.py | 0 .../tests}/ftp/sensors/test_ftp.py | 0 .../tests/github}/__init__.py | 0 .../tests/github/hooks}/__init__.py | 0 .../tests}/github/hooks/test_github.py | 0 .../tests/github/operators}/__init__.py | 0 .../tests}/github/operators/test_github.py | 0 .../tests/github/sensors}/__init__.py | 0 .../tests}/github/sensors/test_github.py | 0 .../tests/google}/__init__.py | 0 .../tests/google/ads}/__init__.py | 0 .../tests/google/ads/hooks}/__init__.py | 0 .../tests}/google/ads/hooks/test_ads.py | 0 .../tests/google/ads}/operators/__init__.py | 0 .../tests}/google/ads/operators/test_ads.py | 0 .../tests/google/ads/transfers}/__init__.py | 0 .../google/ads/transfers/test_ads_to_gcs.py | 3 +- .../tests/google/assets}/__init__.py | 0 .../tests/google/assets}/test_bigquery.py | 0 .../tests/google/cloud}/__init__.py | 0 .../cloud/_internal_client}/__init__.py | 0 .../test_secret_manager_client.py | 0 .../tests/google/cloud/hooks}/__init__.py | 0 .../tests}/google/cloud/hooks/test_automl.py | 3 +- .../google/cloud/hooks/test_bigquery.py | 0 .../google/cloud/hooks/test_bigquery_dts.py | 3 +- .../cloud/hooks/test_bigquery_system.py | 5 +- .../google/cloud/hooks/test_bigtable.py | 3 +- .../google/cloud/hooks/test_cloud_batch.py | 3 +- .../google/cloud/hooks/test_cloud_build.py | 3 +- .../google/cloud/hooks/test_cloud_composer.py | 0 .../cloud/hooks/test_cloud_memorystore.py | 3 +- .../google/cloud/hooks/test_cloud_run.py | 3 +- .../google/cloud/hooks/test_cloud_sql.py | 3 +- .../test_cloud_storage_transfer_service.py | 3 +- ...st_cloud_storage_transfer_service_async.py | 3 +- .../tests}/google/cloud/hooks/test_compute.py | 3 +- .../google/cloud/hooks/test_compute_ssh.py | 0 .../google/cloud/hooks/test_datacatalog.py | 3 +- .../google/cloud/hooks/test_dataflow.py | 0 .../google/cloud/hooks/test_dataform.py | 3 +- .../google/cloud/hooks/test_datafusion.py | 3 +- .../google/cloud/hooks/test_datapipeline.py | 0 .../google/cloud/hooks/test_dataplex.py | 3 +- .../google/cloud/hooks/test_dataprep.py | 0 .../google/cloud/hooks/test_dataproc.py | 0 .../cloud/hooks/test_dataproc_metastore.py | 3 +- .../google/cloud/hooks/test_datastore.py | 0 .../tests}/google/cloud/hooks/test_dlp.py | 3 +- .../google/cloud/hooks/test_functions.py | 3 +- .../tests}/google/cloud/hooks/test_gcs.py | 3 +- .../tests}/google/cloud/hooks/test_gdm.py | 0 .../tests}/google/cloud/hooks/test_kms.py | 0 .../google/cloud/hooks/test_kms_system.py | 5 +- .../cloud/hooks/test_kubernetes_engine.py | 3 +- .../google/cloud/hooks/test_life_sciences.py | 3 +- .../tests}/google/cloud/hooks/test_looker.py | 0 .../google/cloud/hooks/test_mlengine.py | 3 +- .../cloud/hooks/test_natural_language.py | 3 +- .../google/cloud/hooks/test_os_login.py | 3 +- .../tests}/google/cloud/hooks/test_pubsub.py | 0 .../google/cloud/hooks/test_secret_manager.py | 3 +- .../cloud/hooks/test_secret_manager_system.py | 5 +- .../tests}/google/cloud/hooks/test_spanner.py | 3 +- .../google/cloud/hooks/test_speech_to_text.py | 3 +- .../google/cloud/hooks/test_stackdriver.py | 0 .../tests}/google/cloud/hooks/test_tasks.py | 3 +- .../google/cloud/hooks/test_text_to_speech.py | 3 +- .../google/cloud/hooks/test_translate.py | 3 +- .../cloud/hooks/test_video_intelligence.py | 3 +- .../tests}/google/cloud/hooks/test_vision.py | 3 +- .../google/cloud/hooks/test_workflows.py | 0 .../google/cloud/hooks/vertex_ai}/__init__.py | 0 .../cloud/hooks/vertex_ai/test_auto_ml.py | 3 +- .../vertex_ai/test_batch_prediction_job.py | 3 +- .../cloud/hooks/vertex_ai/test_custom_job.py | 3 +- .../cloud/hooks/vertex_ai/test_dataset.py | 3 +- .../hooks/vertex_ai/test_endpoint_service.py | 3 +- .../hooks/vertex_ai/test_generative_model.py | 3 +- .../test_hyperparameter_tuning_job.py | 3 +- .../hooks/vertex_ai/test_model_service.py | 3 +- .../hooks/vertex_ai/test_pipeline_job.py | 3 +- .../vertex_ai/test_prediction_service.py | 3 +- .../tests/google/cloud/links}/__init__.py | 0 .../google/cloud/links/test_translate.py | 0 .../tests/google/cloud/log}/__init__.py | 0 .../google/cloud/log/test_gcs_task_handler.py | 5 +- .../cloud/log/test_gcs_task_handler_system.py | 24 +- .../log/test_stackdriver_task_handler.py | 7 +- .../test_stackdriver_task_handler_system.py | 22 +- .../google/cloud/openlineage}/__init__.py | 0 .../google/cloud/openlineage/test_mixins.py | 15 +- .../google/cloud/openlineage/test_utils.py | 4 +- .../tests/google/cloud/operators}/__init__.py | 0 .../cloud/operators/source}/__init__.py | 0 .../source/source_prefix}/__init__.py | 0 .../google/cloud/operators/test_automl.py | 0 .../google/cloud/operators/test_bigquery.py | 11 +- .../cloud/operators/test_bigquery_dts.py | 3 +- .../google/cloud/operators/test_bigtable.py | 0 .../google/cloud/operators/test_cloud_base.py | 0 .../cloud/operators/test_cloud_batch.py | 0 .../cloud/operators/test_cloud_build.py | 0 .../cloud/operators/test_cloud_composer.py | 0 .../cloud/operators/test_cloud_memorystore.py | 0 .../google/cloud/operators/test_cloud_run.py | 0 .../google/cloud/operators/test_cloud_sql.py | 0 .../test_cloud_storage_transfer_service.py | 0 .../google/cloud/operators/test_compute.py | 0 .../cloud/operators/test_datacatalog.py | 0 .../google/cloud/operators/test_dataflow.py | 0 .../google/cloud/operators/test_dataform.py | 0 .../google/cloud/operators/test_datafusion.py | 0 .../cloud/operators/test_datapipeline.py | 0 .../google/cloud/operators/test_dataplex.py | 0 .../google/cloud/operators/test_dataprep.py | 0 .../cloud/operators/test_dataprep_system.py | 5 +- .../google/cloud/operators/test_dataproc.py | 5 +- .../operators/test_dataproc_metastore.py | 0 .../google/cloud/operators/test_datastore.py | 0 .../cloud/operators/test_datastore_system.py | 8 +- .../tests}/google/cloud/operators/test_dlp.py | 0 .../google/cloud/operators/test_functions.py | 0 .../tests}/google/cloud/operators/test_gcs.py | 0 .../cloud/operators/test_kubernetes_engine.py | 0 .../cloud/operators/test_life_sciences.py | 0 .../google/cloud/operators/test_looker.py | 3 +- .../google/cloud/operators/test_mlengine.py | 0 .../cloud/operators/test_natural_language.py | 0 .../google/cloud/operators/test_pubsub.py | 0 .../google/cloud/operators/test_spanner.py | 0 .../cloud/operators/test_speech_to_text.py | 0 .../cloud/operators/test_stackdriver.py | 0 .../google/cloud/operators/test_tasks.py | 0 .../cloud/operators/test_text_to_speech.py | 0 .../google/cloud/operators/test_translate.py | 0 .../cloud/operators/test_translate_speech.py | 0 .../google/cloud/operators/test_vertex_ai.py | 18 +- .../operators/test_video_intelligence.py | 0 .../google/cloud/operators/test_vision.py | 0 .../google/cloud/operators/test_workflows.py | 0 .../cloud/operators/vertex_ai}/__init__.py | 0 .../vertex_ai/test_generative_model.py | 0 .../tests/google/cloud/secrets}/__init__.py | 0 .../cloud/secrets/test_secret_manager.py | 0 .../secrets/test_secret_manager_system.py | 4 +- .../tests/google/cloud/sensors}/__init__.py | 0 .../google/cloud/sensors/test_bigquery.py | 0 .../google/cloud/sensors/test_bigquery_dts.py | 0 .../google/cloud/sensors/test_bigtable.py | 0 .../cloud/sensors/test_cloud_composer.py | 0 .../test_cloud_storage_transfer_service.py | 0 .../google/cloud/sensors/test_dataflow.py | 0 .../google/cloud/sensors/test_datafusion.py | 0 .../google/cloud/sensors/test_dataplex.py | 0 .../google/cloud/sensors/test_dataprep.py | 0 .../google/cloud/sensors/test_dataproc.py | 0 .../cloud/sensors/test_dataproc_metastore.py | 0 .../tests}/google/cloud/sensors/test_gcs.py | 0 .../google/cloud/sensors/test_looker.py | 0 .../google/cloud/sensors/test_pubsub.py | 0 .../tests}/google/cloud/sensors/test_tasks.py | 0 .../google/cloud/sensors/test_workflows.py | 0 .../tests/google/cloud/transfers}/__init__.py | 0 .../cloud/transfers/test_adls_to_gcs.py | 0 .../cloud/transfers/test_azure_blob_to_gcs.py | 0 .../transfers/test_azure_fileshare_to_gcs.py | 0 .../transfers/test_bigquery_to_bigquery.py | 0 .../cloud/transfers/test_bigquery_to_gcs.py | 0 .../cloud/transfers/test_bigquery_to_mssql.py | 0 .../cloud/transfers/test_bigquery_to_mysql.py | 0 .../transfers/test_bigquery_to_postgres.py | 0 .../cloud/transfers/test_calendar_to_gcs.py | 0 .../cloud/transfers/test_cassandra_to_gcs.py | 18 +- .../transfers/test_facebook_ads_to_gcs.py | 0 .../test_facebook_ads_to_gcs_system.py | 9 +- .../cloud/transfers/test_gcs_to_bigquery.py | 0 .../google/cloud/transfers/test_gcs_to_gcs.py | 0 .../cloud/transfers/test_gcs_to_local.py | 0 .../cloud/transfers/test_gcs_to_sftp.py | 0 .../cloud/transfers/test_gdrive_to_gcs.py | 0 .../cloud/transfers/test_gdrive_to_local.py | 0 .../cloud/transfers/test_local_to_gcs.py | 0 .../cloud/transfers/test_mssql_to_gcs.py | 0 .../cloud/transfers/test_mysql_to_gcs.py | 0 .../cloud/transfers/test_oracle_to_gcs.py | 0 .../cloud/transfers/test_postgres_to_gcs.py | 0 .../google/cloud/transfers/test_s3_to_gcs.py | 0 .../cloud/transfers/test_salesforce_to_gcs.py | 0 .../test_salesforce_to_gcs_system.py | 10 +- .../cloud/transfers/test_sftp_to_gcs.py | 0 .../cloud/transfers/test_sheets_to_gcs.py | 0 .../google/cloud/transfers/test_sql_to_gcs.py | 0 .../cloud/transfers/test_trino_to_gcs.py | 0 .../tests/google/cloud/triggers}/__init__.py | 0 .../google/cloud/triggers/test_bigquery.py | 0 .../cloud/triggers/test_bigquery_dts.py | 0 .../google/cloud/triggers/test_cloud_batch.py | 0 .../google/cloud/triggers/test_cloud_build.py | 0 .../cloud/triggers/test_cloud_composer.py | 0 .../google/cloud/triggers/test_cloud_run.py | 0 .../google/cloud/triggers/test_cloud_sql.py | 0 .../test_cloud_storage_transfer_service.py | 0 .../google/cloud/triggers/test_dataflow.py | 0 .../google/cloud/triggers/test_datafusion.py | 0 .../google/cloud/triggers/test_dataplex.py | 0 .../google/cloud/triggers/test_dataproc.py | 0 .../tests}/google/cloud/triggers/test_gcs.py | 0 .../cloud/triggers/test_kubernetes_engine.py | 0 .../google/cloud/triggers/test_mlengine.py | 0 .../google/cloud/triggers/test_pubsub.py | 0 .../google/cloud/triggers/test_vertex_ai.py | 3 +- .../tests/google/cloud/utils}/__init__.py | 0 .../tests}/google/cloud/utils/airflow_util.py | 0 .../google/cloud/utils/base_gcp_mock.py | 0 .../google/cloud/utils/gcp_authenticator.py | 5 +- .../google/cloud/utils/job_details.json | 0 .../google/cloud/utils/out_table_details.json | 0 .../cloud/utils/script_job_details.json | 0 .../google/cloud/utils/table_details.json | 0 .../cloud/utils/test_credentials_provider.py | 12 +- .../google/cloud/utils/test_datafusion.py | 0 .../google/cloud/utils/test_dataproc.py | 0 .../utils/test_external_token_supplier.py | 0 .../cloud/utils/test_field_sanitizer.py | 0 .../cloud/utils/test_field_validator.py | 0 .../tests}/google/cloud/utils/test_helpers.py | 0 .../utils/test_mlengine_operator_utils.py | 0 .../utils/test_mlengine_prediction_summary.py | 12 +- .../tests/google/common}/__init__.py | 0 .../google/common/auth_backend}/__init__.py | 0 .../common/auth_backend/test_google_openid.py | 9 +- .../tests/google/common/hooks}/__init__.py | 0 .../google/common/hooks/test_base_google.py | 3 +- .../google/common/hooks/test_discovery_api.py | 0 .../tests}/google/common/test_deprecated.py | 0 .../tests/google/common/utils}/__init__.py | 0 .../common/utils/test_id_token_credentials.py | 0 .../tests/google/firebase}/__init__.py | 0 .../tests/google/firebase/hooks}/__init__.py | 0 .../google/firebase/hooks/test_firestore.py | 3 +- .../google/firebase/operators}/__init__.py | 0 .../firebase/operators/test_firestore.py | 0 .../tests/google/leveldb}/__init__.py | 0 .../tests/google/leveldb/hooks}/__init__.py | 0 .../google/leveldb/hooks/test_leveldb.py | 0 .../google/leveldb/operators}/__init__.py | 0 .../google/leveldb/operators/test_leveldb.py | 0 .../google/marketing_platform}/__init__.py | 0 .../marketing_platform/hooks}/__init__.py | 0 .../hooks/test_analytics.py | 14 +- .../hooks/test_analytics_admin.py | 3 +- .../hooks/test_campaign_manager.py | 3 +- .../hooks/test_display_video.py | 3 +- .../hooks/test_search_ads.py | 3 +- .../marketing_platform/links}/__init__.py | 0 .../links/test_analytics_admin.py | 0 .../marketing_platform/operators}/__init__.py | 0 .../operators/test_analytics.py | 0 .../operators/test_analytics_admin.py | 0 .../operators/test_campaign_manager.py | 0 .../operators/test_display_video.py | 0 .../operators/test_display_video_system.py | 9 +- .../operators/test_search_ads.py | 0 .../marketing_platform/sensors}/__init__.py | 0 .../sensors/test_campaign_manager.py | 0 .../sensors/test_display_video.py | 0 .../tests/google/suite}/__init__.py | 0 .../tests/google/suite/hooks}/__init__.py | 0 .../google/suite/hooks/test_calendar.py | 3 +- .../tests}/google/suite/hooks/test_drive.py | 3 +- .../tests}/google/suite/hooks/test_sheets.py | 3 +- .../tests/google/suite/operators}/__init__.py | 0 .../google/suite/operators/test_sheets.py | 0 .../tests/google/suite/sensors}/__init__.py | 0 .../tests}/google/suite/sensors/test_drive.py | 0 .../tests/google/suite/transfers}/__init__.py | 0 .../suite/transfers/test_gcs_to_gdrive.py | 0 .../suite/transfers/test_gcs_to_sheets.py | 0 .../suite/transfers/test_local_to_drive.py | 0 .../suite/transfers/test_sql_to_sheets.py | 0 .../tests}/google/test_go_module.py | 0 .../tests/grpc}/__init__.py | 0 .../tests/grpc/hooks}/__init__.py | 0 .../tests}/grpc/hooks/test_grpc.py | 0 .../tests/grpc/operators}/__init__.py | 0 .../tests}/grpc/operators/test_grpc.py | 0 .../tests/hashicorp}/__init__.py | 0 .../hashicorp/_internal_client}/__init__.py | 0 .../_internal_client/test_vault_client.py | 0 .../tests/hashicorp/hooks}/__init__.py | 0 .../tests}/hashicorp/hooks/test_vault.py | 3 +- .../tests/hashicorp/secrets}/__init__.py | 0 .../tests}/hashicorp/secrets/test_vault.py | 0 .../tests/http}/__init__.py | 0 .../tests/http/hooks}/__init__.py | 0 .../tests}/http/hooks/test_http.py | 53 +- .../tests/http/operators}/__init__.py | 0 .../tests}/http/operators/test_http.py | 0 .../tests/http/sensors}/__init__.py | 0 .../tests}/http/sensors/test_http.py | 0 .../tests/http/triggers}/__init__.py | 0 .../tests}/http/triggers/test_http.py | 0 .../tests/imap}/__init__.py | 0 .../tests/imap/hooks}/__init__.py | 0 .../tests}/imap/hooks/test_imap.py | 3 +- .../tests/imap/sensors}/__init__.py | 0 .../imap/sensors/test_imap_attachment.py | 0 .../tests/influxdb}/__init__.py | 0 .../tests/influxdb/hooks}/__init__.py | 0 .../tests}/influxdb/hooks/test_influxdb.py | 0 .../tests/influxdb/operators}/__init__.py | 0 .../influxdb/operators/test_influxdb.py | 0 .../tests/integration}/__init__.py | 0 .../tests/integration/apache}/__init__.py | 0 .../integration/apache/cassandra}/__init__.py | 0 .../apache/cassandra/hooks}/__init__.py | 0 .../apache/cassandra/hooks/test_cassandra.py | 0 .../integration/apache/drill}/__init__.py | 0 .../apache/drill/hooks}/__init__.py | 0 .../apache/drill/hooks/test_drill.py | 0 .../apache/drill/operators}/__init__.py | 0 .../integration/apache/hive}/__init__.py | 0 .../apache/hive/transfers}/__init__.py | 0 .../hive/transfers/test_mssql_to_hive.py | 0 .../integration/apache/kafka}/__init__.py | 0 .../apache/kafka/hooks}/__init__.py | 0 .../apache/kafka/hooks/test_admin_client.py | 0 .../apache/kafka/hooks/test_consumer.py | 0 .../apache/kafka/hooks/test_producer.py | 0 .../apache/kafka/operators}/__init__.py | 0 .../apache/kafka/operators/test_consume.py | 0 .../apache/kafka/operators/test_produce.py | 0 .../apache/kafka/sensors}/__init__.py | 0 .../apache/kafka/triggers}/__init__.py | 0 .../kafka/triggers/test_await_message.py | 0 .../integration/apache/pinot}/__init__.py | 0 .../apache/pinot/hooks}/__init__.py | 0 .../apache/pinot/hooks/test_pinot.py | 0 .../tests/integration/google}/__init__.py | 0 .../integration/google/cloud}/__init__.py | 0 .../google/cloud/transfers}/__init__.py | 0 .../cloud/transfers/test_bigquery_to_mssql.py | 0 .../cloud/transfers/test_mssql_to_gcs.py | 0 .../cloud/transfers/test_trino_to_gcs.py | 0 .../tests/integration/microsoft}/__init__.py | 0 .../integration/microsoft/mssql}/__init__.py | 0 .../microsoft/mssql/hooks}/__init__.py | 0 .../microsoft/mssql/hooks/test_mssql.py | 0 .../tests/integration/mongo}/__init__.py | 0 .../integration/mongo/sensors}/__init__.py | 0 .../integration}/mongo/sensors/test_mongo.py | 0 .../integration/openlineage}/__init__.py | 0 .../openlineage/operators}/__init__.py | 0 .../tests/integration/qdrant}/__init__.py | 0 .../integration/qdrant/hooks}/__init__.py | 0 .../integration}/qdrant/hooks/test_qdrant.py | 0 .../integration/qdrant/operators}/__init__.py | 0 .../qdrant/operators/test_qdrant_ingest.py | 0 .../tests/integration/redis}/__init__.py | 0 .../integration/redis/hooks}/__init__.py | 0 .../integration}/redis/hooks/test_redis.py | 0 .../integration/redis/operators}/__init__.py | 0 .../redis/operators/test_redis_publish.py | 0 .../integration/redis/sensors}/__init__.py | 0 .../redis/sensors/test_redis_key.py | 0 .../redis/sensors/test_redis_pub_sub.py | 0 .../tests/integration/trino}/__init__.py | 0 .../integration/trino/hooks}/__init__.py | 0 .../integration}/trino/hooks/test_trino.py | 0 .../tests/integration/ydb}/__init__.py | 0 .../tests/integration/ydb/hooks}/__init__.py | 0 .../integration/ydb/operators}/__init__.py | 0 .../integration}/ydb/operators/test_ydb.py | 0 .../tests/jdbc}/__init__.py | 0 .../tests/jdbc/hooks}/__init__.py | 0 .../tests}/jdbc/hooks/test_jdbc.py | 7 +- .../tests/jdbc/operators}/__init__.py | 0 .../tests}/jdbc/operators/test_jdbc.py | 0 .../tests/jenkins}/__init__.py | 0 .../tests/jenkins/hooks}/__init__.py | 0 .../tests}/jenkins/hooks/test_jenkins.py | 0 .../tests/jenkins}/operators/__init__.py | 0 .../operators/test_jenkins_job_trigger.py | 85 +- .../tests/jenkins/sensors}/__init__.py | 0 .../tests}/jenkins/sensors/test_jenkins.py | 0 .../tests/microsoft}/__init__.py | 0 .../tests/microsoft/azure}/__init__.py | 0 .../tests}/microsoft/azure/base.py | 10 +- .../tests/microsoft/azure/fs}/__init__.py | 0 .../tests}/microsoft/azure/fs/test_adls.py | 0 .../tests/microsoft/azure/hooks}/__init__.py | 0 .../tests}/microsoft/azure/hooks/test_adx.py | 0 .../tests}/microsoft/azure/hooks/test_asb.py | 0 .../microsoft/azure/hooks/test_base_azure.py | 0 .../microsoft/azure/hooks/test_batch.py | 0 .../azure/hooks/test_container_instance.py | 11 +- .../azure/hooks/test_container_registry.py | 0 .../azure/hooks/test_container_volume.py | 0 .../microsoft/azure/hooks/test_cosmos.py | 0 .../azure/hooks/test_data_factory.py | 0 .../microsoft/azure/hooks/test_data_lake.py | 0 .../microsoft/azure/hooks/test_fileshare.py | 0 .../microsoft/azure/hooks/test_msgraph.py | 3 +- .../microsoft/azure/hooks/test_powerbi.py | 0 .../microsoft/azure/hooks/test_synapse.py | 0 .../azure/hooks/test_synapse_pipeline.py | 0 .../tests}/microsoft/azure/hooks/test_wasb.py | 0 .../tests/microsoft/azure/log}/__init__.py | 0 .../azure/log/test_wasb_task_handler.py | 5 +- .../microsoft/azure/operators}/__init__.py | 0 .../azure/operators/test_adls_create.py | 0 .../azure/operators/test_adls_delete.py | 0 .../azure/operators/test_adls_list.py | 0 .../microsoft/azure/operators/test_adx.py | 0 .../microsoft/azure/operators/test_asb.py | 0 .../microsoft/azure/operators/test_batch.py | 0 .../operators/test_container_instances.py | 0 .../microsoft/azure/operators/test_cosmos.py | 0 .../azure/operators/test_data_factory.py | 0 .../microsoft/azure/operators/test_msgraph.py | 5 +- .../microsoft/azure/operators/test_powerbi.py | 5 +- .../microsoft/azure/operators/test_synapse.py | 0 .../azure/operators/test_wasb_delete_blob.py | 0 .../microsoft/azure/resources}/__init__.py | 0 .../microsoft/azure/resources/dummy.pdf | Bin .../microsoft/azure/resources/next_users.json | 0 .../microsoft/azure/resources/status.json | 0 .../microsoft/azure/resources/users.json | 0 .../microsoft/azure/secrets}/__init__.py | 0 .../microsoft/azure/secrets/test_key_vault.py | 0 .../microsoft/azure/sensors}/__init__.py | 0 .../microsoft/azure/sensors/test_cosmos.py | 0 .../azure/sensors/test_data_factory.py | 0 .../microsoft/azure/sensors/test_msgraph.py | 5 +- .../microsoft/azure/sensors/test_wasb.py | 0 .../tests}/microsoft/azure/test_utils.py | 0 .../microsoft/azure/transfers}/__init__.py | 0 .../azure/transfers/test_local_to_adls.py | 0 .../azure/transfers/test_local_to_wasb.py | 0 .../test_oracle_to_azure_data_lake.py | 0 .../azure/transfers/test_s3_to_wasb.py | 0 .../azure/transfers/test_sftp_to_wasb.py | 0 .../microsoft/azure/triggers}/__init__.py | 0 .../azure/triggers/test_data_factory.py | 0 .../microsoft/azure/triggers/test_msgraph.py | 5 +- .../microsoft/azure/triggers/test_powerbi.py | 3 +- .../microsoft/azure/triggers/test_wasb.py | 0 .../tests}/microsoft/conftest.py | 0 .../tests/microsoft/mssql}/__init__.py | 0 .../tests/microsoft/mssql/hooks}/__init__.py | 0 .../microsoft/mssql/hooks/test_mssql.py | 3 +- .../microsoft/mssql/operators}/__init__.py | 0 .../microsoft/mssql/operators/test_mssql.py | 0 .../microsoft/mssql/resources}/__init__.py | 0 .../microsoft/mssql/resources/replace.sql | 0 .../tests/microsoft/psrp}/__init__.py | 0 .../tests/microsoft/psrp/hooks}/__init__.py | 0 .../tests}/microsoft/psrp/hooks/test_psrp.py | 17 +- .../microsoft/psrp/operators}/__init__.py | 0 .../microsoft/psrp/operators/test_psrp.py | 0 .../tests/microsoft/winrm}/__init__.py | 0 .../tests/microsoft/winrm/hooks}/__init__.py | 0 .../microsoft/winrm/hooks/test_winrm.py | 0 .../microsoft/winrm/operators}/__init__.py | 0 .../microsoft/winrm/operators/test_winrm.py | 0 .../tests/mongo}/__init__.py | 0 .../tests/mongo/hooks}/__init__.py | 0 .../tests}/mongo/hooks/test_mongo.py | 3 +- .../tests/mongo/sensors}/__init__.py | 0 .../tests}/mongo/sensors/test_mongo.py | 0 .../tests/mysql}/__init__.py | 0 .../tests/mysql/assets}/__init__.py | 0 .../tests}/mysql/assets/test_mysql.py | 0 .../tests/mysql/hooks}/__init__.py | 0 .../tests}/mysql/hooks/test_mysql.py | 3 +- .../hooks/test_mysql_connector_python.py | 0 .../tests/mysql/operators}/__init__.py | 0 .../tests}/mysql/operators/test_mysql.py | 3 +- .../tests/mysql/transfers}/__init__.py | 0 .../mysql/transfers/test_presto_to_mysql.py | 0 .../mysql/transfers/test_s3_to_mysql.py | 0 .../mysql/transfers/test_trino_to_mysql.py | 0 .../mysql/transfers/test_vertica_to_mysql.py | 0 .../tests/neo4j}/__init__.py | 0 .../tests/neo4j/hooks}/__init__.py | 0 .../tests}/neo4j/hooks/test_neo4j.py | 0 .../tests/neo4j/operators}/__init__.py | 0 .../tests}/neo4j/operators/test_neo4j.py | 0 .../tests/odbc}/__init__.py | 0 .../tests/odbc/hooks}/__init__.py | 0 .../tests}/odbc/hooks/test_odbc.py | 3 +- .../tests/openai}/__init__.py | 0 .../tests/openai/hooks}/__init__.py | 0 .../tests}/openai/hooks/test_openai.py | 0 .../tests/openai/operators}/__init__.py | 0 .../tests}/openai/operators/test_openai.py | 0 .../tests}/openai/test_exceptions.py | 0 .../tests/openai/triggers}/__init__.py | 0 .../tests}/openai/triggers/test_openai.py | 0 .../tests/openfaas}/__init__.py | 0 .../tests/openfaas/hooks}/__init__.py | 0 .../tests}/openfaas/hooks/test_openfaas.py | 0 .../tests/openlineage}/__init__.py | 0 .../tests/openlineage/extractors}/__init__.py | 0 .../openlineage/extractors/test_base.py | 2 +- .../openlineage/extractors/test_bash.py | 3 +- .../openlineage/extractors/test_manager.py | 3 +- .../openlineage/extractors/test_python.py | 3 +- .../tests}/openlineage/log_config.py | 0 .../tests/openlineage/plugins}/__init__.py | 0 .../plugins/openlineage_configs}/__init__.py | 0 .../plugins/openlineage_configs/http.yaml | 0 .../openlineage/plugins/test_adapter.py | 5 +- .../openlineage/plugins/test_execution.py | 5 +- .../tests}/openlineage/plugins/test_facets.py | 0 .../openlineage/plugins/test_listener.py | 5 +- .../tests}/openlineage/plugins/test_macros.py | 0 .../openlineage/plugins/test_openlineage.py | 4 +- .../tests}/openlineage/plugins/test_utils.py | 3 +- .../tests}/openlineage/test_conf.py | 3 +- .../tests}/openlineage/test_sqlparser.py | 0 .../tests/openlineage/utils}/__init__.py | 0 .../openlineage/utils/custom_facet_fixture.py | 0 .../utils/test_selective_enable.py | 0 .../tests}/openlineage/utils/test_sql.py | 0 .../tests}/openlineage/utils/test_utils.py | 31 +- .../tests/opensearch}/__init__.py | 0 .../tests}/opensearch/conftest.py | 0 .../tests/opensearch/hooks}/__init__.py | 0 .../opensearch/hooks/test_opensearch.py | 0 .../tests/opensearch/log}/__init__.py | 0 .../opensearch/log/test_os_json_formatter.py | 0 .../tests}/opensearch/log/test_os_response.py | 0 .../opensearch/log/test_os_task_handler.py | 7 +- .../tests/opensearch/operators}/__init__.py | 0 .../opensearch/operators/test_opensearch.py | 0 .../tests/opsgenie}/__init__.py | 0 .../tests/opsgenie}/hooks/__init__.py | 0 .../tests}/opsgenie/hooks/test_opsgenie.py | 0 .../tests/opsgenie/notifications}/__init__.py | 0 .../opsgenie/notifications/test_opsgenie.py | 0 .../tests/opsgenie/operators}/__init__.py | 0 .../opsgenie/operators/test_opsgenie.py | 0 .../tests/opsgenie/typing}/__init__.py | 0 .../tests}/opsgenie/typing/test_opsgenie.py | 0 .../tests/oracle}/__init__.py | 0 .../tests/oracle/hooks}/__init__.py | 0 .../tests}/oracle/hooks/test_oracle.py | 0 .../tests/oracle/operators}/__init__.py | 0 .../tests}/oracle/operators/test_oracle.py | 0 .../tests/oracle/transfers}/__init__.py | 0 .../oracle/transfers/test_oracle_to_oracle.py | 0 .../tests/pagerduty}/__init__.py | 0 .../tests/pagerduty/hooks}/__init__.py | 0 .../tests}/pagerduty/hooks/test_pagerduty.py | 0 .../pagerduty/hooks/test_pagerduty_events.py | 0 .../pagerduty/notifications}/__init__.py | 0 .../pagerduty/notifications/test_pagerduty.py | 0 .../tests/papermill}/__init__.py | 0 .../tests/papermill/hooks}/__init__.py | 0 .../tests}/papermill/hooks/test_kernel.py | 0 .../tests/papermill/operators}/__init__.py | 0 .../papermill/operators/test_papermill.py | 0 .../tests/pgvector}/__init__.py | 0 .../tests/pgvector/hooks}/__init__.py | 0 .../tests}/pgvector/hooks/test_pgvector.py | 0 .../tests/pgvector/operators}/__init__.py | 0 .../pgvector/operators/test_pgvector.py | 0 .../tests/pinecone}/__init__.py | 0 .../tests/pinecone/hooks}/__init__.py | 0 .../tests}/pinecone/hooks/test_pinecone.py | 0 .../tests/pinecone/operators}/__init__.py | 0 .../pinecone/operators/test_pinecone.py | 0 .../tests/postgres}/__init__.py | 0 .../tests/postgres/assets}/__init__.py | 0 .../tests}/postgres/assets/test_postgres.py | 0 .../tests/postgres/hooks}/__init__.py | 0 .../tests}/postgres/hooks/test_postgres.py | 0 .../tests/postgres/operators}/__init__.py | 0 .../postgres/operators/test_postgres.py | 0 .../tests/presto}/__init__.py | 0 .../tests/presto/hooks}/__init__.py | 0 .../tests}/presto/hooks/test_presto.py | 0 .../tests/presto/transfers}/__init__.py | 0 .../presto/transfers/test_gcs_to_presto.py | 0 .../tests/qdrant}/__init__.py | 0 .../tests/qdrant/hooks}/__init__.py | 0 .../tests}/qdrant/hooks/test_qdrant.py | 0 .../tests/qdrant}/operators/__init__.py | 0 .../tests}/qdrant/operators/test_qdrant.py | 0 .../tests/redis}/__init__.py | 0 .../tests/redis/hooks}/__init__.py | 0 .../tests}/redis/hooks/test_redis.py | 0 .../tests/redis/log}/__init__.py | 0 .../redis/log/test_redis_task_handler.py | 3 +- .../tests/redis/operators}/__init__.py | 0 .../redis/operators/test_redis_publish.py | 0 .../tests/redis/sensors}/__init__.py | 0 .../tests}/redis/sensors/test_redis_key.py | 0 .../redis/sensors/test_redis_pub_sub.py | 0 .../tests/salesforce}/__init__.py | 0 .../tests/salesforce/hooks}/__init__.py | 0 .../salesforce/hooks/test_salesforce.py | 0 .../tests/salesforce/operators}/__init__.py | 0 .../tests}/salesforce/operators/test_bulk.py | 0 .../operators/test_salesforce_apex_rest.py | 0 .../tests/samba}/__init__.py | 0 .../tests/samba}/hooks/__init__.py | 0 .../tests}/samba/hooks/test_samba.py | 0 .../tests/samba/transfers}/__init__.py | 0 .../samba/transfers/test_gcs_to_samba.py | 0 .../tests/segment}/__init__.py | 0 .../tests/segment/hooks}/__init__.py | 0 .../tests}/segment/hooks/test_segment.py | 0 .../tests/segment/operators}/__init__.py | 0 .../operators/test_segment_track_event.py | 0 .../tests/sendgrid}/__init__.py | 0 .../tests/sendgrid/utils}/__init__.py | 0 .../tests}/sendgrid/utils/test_emailer.py | 0 .../tests/sftp}/__init__.py | 0 .../tests/sftp/decorators}/__init__.py | 0 .../sftp/decorators/sensors}/__init__.py | 0 .../sftp/decorators/sensors/test_sftp.py | 0 .../tests/sftp/hooks}/__init__.py | 0 .../tests}/sftp/hooks/test_sftp.py | 5 +- .../tests/sftp/operators}/__init__.py | 0 .../tests}/sftp/operators/test_sftp.py | 20 +- .../tests/sftp/sensors}/__init__.py | 0 .../tests}/sftp/sensors/test_sftp.py | 0 .../tests/sftp/triggers}/__init__.py | 0 .../tests}/sftp/triggers/test_sftp.py | 0 .../tests/singularity}/__init__.py | 0 .../tests/singularity}/operators/__init__.py | 0 .../singularity/operators/test_singularity.py | 0 .../tests/slack}/__init__.py | 0 .../tests/slack/hooks}/__init__.py | 0 .../tests}/slack/hooks/test_slack.py | 0 .../tests}/slack/hooks/test_slack_webhook.py | 0 .../tests/slack/notifications}/__init__.py | 0 .../tests}/slack/notifications/test_slack.py | 0 .../slack/notifications/test_slack_webhook.py | 0 .../tests/slack/operators}/__init__.py | 0 .../tests}/slack/operators/test_slack.py | 5 +- .../slack/operators/test_slack_webhook.py | 0 .../tests/slack/transfers}/__init__.py | 0 .../tests}/slack/transfers/conftest.py | 0 .../slack/transfers/test_base_sql_to_slack.py | 0 .../slack/transfers/test_sql_to_slack.py | 0 .../transfers/test_sql_to_slack_webhook.py | 10 +- .../tests/slack/utils}/__init__.py | 0 .../tests}/slack/utils/test_utils.py | 0 .../tests/smtp}/__init__.py | 0 .../tests/smtp/hooks}/__init__.py | 0 .../tests}/smtp/hooks/test_smtp.py | 3 +- .../tests/smtp/notifications}/__init__.py | 0 .../tests}/smtp/notifications/test_smtp.py | 12 +- .../tests/smtp/operators}/__init__.py | 0 .../tests}/smtp/operators/test_smtp.py | 0 .../tests/snowflake}/__init__.py | 0 .../tests/snowflake/decorators}/__init__.py | 0 .../snowflake/decorators/test_snowpark.py | 0 .../tests/snowflake/hooks}/__init__.py | 0 .../tests}/snowflake/hooks/test_snowflake.py | 53 +- .../snowflake/hooks/test_snowflake_sql_api.py | 24 +- .../tests}/snowflake/hooks/test_sql.py | 0 .../tests/snowflake/operators}/__init__.py | 0 .../snowflake/operators/test_snowflake.py | 0 .../snowflake/operators/test_snowflake_sql.py | 0 .../snowflake/operators/test_snowpark.py | 0 .../tests/snowflake/transfers}/__init__.py | 0 .../transfers/test_copy_into_snowflake.py | 0 .../tests/snowflake/triggers}/__init__.py | 0 .../snowflake/triggers/test_snowflake.py | 0 .../tests/snowflake/utils}/__init__.py | 0 .../tests}/snowflake/utils/test_common.py | 0 .../snowflake/utils/test_openlineage.py | 0 .../tests}/snowflake/utils/test_snowpark.py | 0 .../utils/test_sql_api_generate_jwt.py | 0 .../tests/sqlite}/__init__.py | 0 .../tests/sqlite/hooks}/__init__.py | 0 .../tests}/sqlite/hooks/test_sqlite.py | 0 .../tests/sqlite/operators}/__init__.py | 0 .../tests}/sqlite/operators/test_sqlite.py | 0 .../hooks => providers/tests/ssh}/__init__.py | 0 .../tests/ssh/hooks}/__init__.py | 0 .../tests}/ssh/hooks/test_ssh.py | 5 +- .../tests/ssh/operators}/__init__.py | 0 .../tests}/ssh/operators/test_ssh.py | 3 +- .../tests/standard}/__init__.py | 0 .../tests/standard/operators}/__init__.py | 0 .../tests}/standard/operators/test_bash.py | 3 +- .../standard/operators/test_datetime.py | 3 +- .../tests}/standard/operators/test_weekday.py | 3 +- .../tests/standard/sensors}/__init__.py | 0 .../tests}/standard/sensors/test_bash.py | 0 .../tests}/standard/sensors/test_date_time.py | 0 .../tests}/standard/sensors/test_time.py | 0 .../standard/sensors/test_time_delta.py | 0 .../tests}/standard/sensors/test_weekday.py | 3 +- .../tests/system}/__init__.py | 0 .../tests/system/airbyte}/__init__.py | 0 .../airbyte/example_airbyte_trigger_job.py | 2 +- .../tests/system/alibaba}/__init__.py | 0 .../alibaba/example_adb_spark_batch.py | 4 +- .../system}/alibaba/example_adb_spark_sql.py | 4 +- .../system}/alibaba/example_oss_bucket.py | 4 +- .../system}/alibaba/example_oss_object.py | 4 +- .../tests/system}/amazon/CONTRIBUTING.md | 2 +- .../tests/system}/amazon/README.md | 0 .../tests/system/amazon}/__init__.py | 0 .../tests/system/amazon/aws}/__init__.py | 0 .../system}/amazon/aws/example_appflow.py | 7 +- .../system}/amazon/aws/example_appflow_run.py | 7 +- .../system}/amazon/aws/example_athena.py | 7 +- .../amazon/aws/example_azure_blob_to_s3.py | 7 +- .../tests/system}/amazon/aws/example_batch.py | 7 +- .../system}/amazon/aws/example_bedrock.py | 7 +- .../example_bedrock_retrieve_and_generate.py | 7 +- .../amazon/aws/example_cloudformation.py | 7 +- .../system}/amazon/aws/example_comprehend.py | 7 +- .../example_comprehend_document_classifier.py | 7 +- .../system}/amazon/aws/example_datasync.py | 7 +- .../tests/system}/amazon/aws/example_dms.py | 9 +- .../system}/amazon/aws/example_dynamodb.py | 7 +- .../amazon/aws/example_dynamodb_to_s3.py | 7 +- .../tests/system}/amazon/aws/example_ec2.py | 7 +- .../tests/system}/amazon/aws/example_ecs.py | 7 +- .../system}/amazon/aws/example_ecs_fargate.py | 7 +- .../amazon/aws/example_eks_templated.py | 7 +- .../example_eks_with_fargate_in_one_step.py | 9 +- .../aws/example_eks_with_fargate_profile.py | 9 +- .../example_eks_with_nodegroup_in_one_step.py | 9 +- .../amazon/aws/example_eks_with_nodegroups.py | 9 +- .../tests/system}/amazon/aws/example_emr.py | 7 +- .../system}/amazon/aws/example_emr_eks.py | 7 +- .../aws/example_emr_notebook_execution.py | 7 +- .../amazon/aws/example_emr_serverless.py | 7 +- .../system}/amazon/aws/example_eventbridge.py | 5 +- .../system}/amazon/aws/example_ftp_to_s3.py | 7 +- .../system}/amazon/aws/example_gcs_to_s3.py | 7 +- .../amazon/aws/example_glacier_to_gcs.py | 7 +- .../tests/system}/amazon/aws/example_glue.py | 7 +- .../amazon/aws/example_glue_data_quality.py | 7 +- ...e_glue_data_quality_with_recommendation.py | 7 +- .../amazon/aws/example_glue_databrew.py | 7 +- .../aws/example_google_api_sheets_to_s3.py | 7 +- .../aws/example_google_api_youtube_to_s3.py | 7 +- .../amazon/aws/example_hive_to_dynamodb.py | 7 +- .../system}/amazon/aws/example_http_to_s3.py | 7 +- .../aws/example_imap_attachment_to_s3.py | 7 +- .../amazon/aws/example_kinesis_analytics.py | 7 +- .../system}/amazon/aws/example_lambda.py | 7 +- .../system}/amazon/aws/example_local_to_s3.py | 7 +- .../system}/amazon/aws/example_mongo_to_s3.py | 7 +- .../system}/amazon/aws/example_neptune.py | 7 +- .../system}/amazon/aws/example_quicksight.py | 7 +- .../system}/amazon/aws/example_rds_event.py | 7 +- .../system}/amazon/aws/example_rds_export.py | 7 +- .../amazon/aws/example_rds_instance.py | 7 +- .../amazon/aws/example_rds_snapshot.py | 7 +- .../system}/amazon/aws/example_redshift.py | 7 +- .../aws/example_redshift_s3_transfers.py | 7 +- .../tests/system}/amazon/aws/example_s3.py | 7 +- .../amazon/aws/example_s3_to_dynamodb.py | 7 +- .../system}/amazon/aws/example_s3_to_ftp.py | 7 +- .../system}/amazon/aws/example_s3_to_sftp.py | 7 +- .../system}/amazon/aws/example_s3_to_sql.py | 7 +- .../system}/amazon/aws/example_sagemaker.py | 7 +- .../amazon/aws/example_sagemaker_endpoint.py | 9 +- .../amazon/aws/example_sagemaker_notebook.py | 7 +- .../amazon/aws/example_sagemaker_pipeline.py | 9 +- .../amazon/aws/example_salesforce_to_s3.py | 7 +- .../system}/amazon/aws/example_sftp_to_s3.py | 7 +- .../tests/system}/amazon/aws/example_sns.py | 7 +- .../system}/amazon/aws/example_sql_to_s3.py | 7 +- .../tests/system}/amazon/aws/example_sqs.py | 7 +- .../amazon/aws/example_step_functions.py | 7 +- .../system/amazon/aws/tests}/__init__.py | 0 .../amazon/aws/tests/test_aws_auth_manager.py | 20 +- .../system}/amazon/aws/utils/__init__.py | 2 +- .../tests/system}/amazon/aws/utils/ec2.py | 0 .../tests/system}/amazon/aws/utils/k8s.py | 0 .../tests/system/apache}/__init__.py | 0 .../tests/system/apache/beam}/__init__.py | 0 .../tests/system}/apache/beam/example_beam.py | 5 +- .../apache/beam/example_beam_java_flink.py | 5 +- .../apache/beam/example_beam_java_spark.py | 5 +- .../tests/system}/apache/beam/example_go.py | 5 +- .../apache/beam/example_go_dataflow.py | 5 +- .../apache/beam/example_java_dataflow.py | 5 +- .../system}/apache/beam/example_python.py | 5 +- .../apache/beam/example_python_async.py | 5 +- .../apache/beam/example_python_dataflow.py | 5 +- .../tests/system}/apache/beam/utils.py | 0 .../system/apache/cassandra}/__init__.py | 0 .../apache/cassandra/example_cassandra_dag.py | 2 +- .../tests/system/apache/drill}/__init__.py | 0 .../system}/apache/drill/example_drill_dag.py | 2 +- .../tests/system/apache/druid}/__init__.py | 0 .../system}/apache/druid/example_druid_dag.py | 2 +- .../tests/system/apache/hive}/__init__.py | 0 .../apache/hive/example_twitter_README.md | 0 .../apache/hive/example_twitter_dag.py | 4 +- .../tests/system/apache/iceberg}/__init__.py | 0 .../system}/apache/iceberg/example_iceberg.py | 2 +- .../tests/system/apache/kafka}/__init__.py | 0 .../kafka/example_dag_event_listener.py | 2 +- .../apache/kafka/example_dag_hello_kafka.py | 2 +- .../tests/system/apache/kylin}/__init__.py | 0 .../system}/apache/kylin/example_kylin_dag.py | 4 +- .../tests/system/apache/livy}/__init__.py | 0 .../tests/system}/apache/livy/example_livy.py | 4 +- .../tests/system/apache/pig}/__init__.py | 0 .../tests/system}/apache/pig/example_pig.py | 2 +- .../tests/system/apache/pinot}/__init__.py | 0 .../system}/apache/pinot/example_pinot_dag.py | 2 +- .../tests/system/apache/spark}/__init__.py | 0 .../system}/apache/spark/example_pyspark.py | 2 +- .../system}/apache/spark/example_spark_dag.py | 2 +- .../tests/system/asana}/__init__.py | 0 .../tests/system}/asana/example_asana.py | 4 +- .../tests/system/cncf}/__init__.py | 0 .../tests/system/cncf/kubernetes}/__init__.py | 0 .../cncf/kubernetes/example_kubernetes.py | 4 +- .../kubernetes/example_kubernetes_async.py | 4 +- .../example_kubernetes_decorator.py | 2 +- .../cncf/kubernetes/example_kubernetes_job.py | 4 +- .../kubernetes/example_kubernetes_resource.py | 4 +- .../kubernetes/example_spark_kubernetes.py | 4 +- .../example_spark_kubernetes_spark_pi.yaml | 0 .../cncf/kubernetes/spark_job_template.yaml | 0 .../tests/system/cohere}/__init__.py | 0 .../example_cohere_embedding_operator.py | 2 +- .../tests/system/common}/__init__.py | 0 .../tests/system/common/io}/__init__.py | 0 .../io/example_file_transfer_local_to_s3.py | 4 +- .../tests/system/common/sql}/__init__.py | 0 .../sql/example_sql_column_table_check.py | 2 +- .../common/sql/example_sql_execute_query.py | 2 +- .../tests/system/databricks}/__init__.py | 0 .../system}/databricks/example_databricks.py | 4 +- .../databricks/example_databricks_repos.py | 4 +- .../databricks/example_databricks_sensors.py | 4 +- .../databricks/example_databricks_sql.py | 4 +- .../databricks/example_databricks_workflow.py | 4 +- .../tests/system/dbt}/__init__.py | 0 .../tests/system/dbt/cloud}/__init__.py | 0 .../system}/dbt/cloud/example_dbt_cloud.py | 7 +- .../tests/system/dingding}/__init__.py | 0 .../system}/dingding/example_dingding.py | 4 +- .../tests/system/docker}/__init__.py | 0 .../tests/system}/docker/example_docker.py | 2 +- .../docker/example_docker_copy_data.py | 2 +- .../system}/docker/example_docker_swarm.py | 2 +- .../example_taskflow_api_docker_virtualenv.py | 2 +- .../tests/system/elasticsearch}/__init__.py | 0 .../example_elasticsearch_query.py | 2 +- .../tests/system/ftp}/__init__.py | 0 .../tests/system}/ftp/example_ftp.py | 4 +- .../tests/system/github}/__init__.py | 0 .../tests/system}/github/example_github.py | 2 +- .../tests/system}/google/README.md | 0 .../tests/system}/google/__init__.py | 0 .../tests/system/google/ads}/__init__.py | 0 .../tests/system}/google/ads/example_ads.py | 7 +- .../tests/system/google/cloud}/__init__.py | 0 .../system/google/cloud/automl}/__init__.py | 0 .../cloud/automl/example_automl_dataset.py | 4 +- .../automl/example_automl_translation.py | 4 +- .../example_automl_video_classification.py | 4 +- .../automl/example_automl_video_tracking.py | 4 +- .../example_automl_vision_classification.py | 4 +- .../example_automl_vision_object_detection.py | 4 +- .../cloud/automl/resources}/__init__.py | 0 .../system/google/cloud/azure}/__init__.py | 0 .../cloud/azure/example_azure_blob_to_gcs.py | 4 +- .../azure/example_azure_fileshare_to_gcs.py | 7 +- .../system/google/cloud/bigquery}/__init__.py | 0 .../bigquery/example_bigquery_dataset.py | 4 +- .../cloud/bigquery/example_bigquery_dts.py | 7 +- .../bigquery/example_bigquery_operations.py | 4 +- .../example_bigquery_operations_location.py | 4 +- .../bigquery/example_bigquery_queries.py | 7 +- .../example_bigquery_queries_async.py | 4 +- .../bigquery/example_bigquery_sensors.py | 4 +- .../cloud/bigquery/example_bigquery_tables.py | 7 +- .../bigquery/example_bigquery_to_bigquery.py | 4 +- .../cloud/bigquery/example_bigquery_to_gcs.py | 7 +- .../bigquery/example_bigquery_to_gcs_async.py | 7 +- .../bigquery/example_bigquery_to_mssql.py | 4 +- .../bigquery/example_bigquery_to_mysql.py | 4 +- .../bigquery/example_bigquery_to_postgres.py | 4 +- .../bigquery/example_bigquery_transfer.py | 7 +- .../bigquery/example_bigquery_value_check.py | 6 +- .../cloud/bigquery/resources}/__init__.py | 0 .../resources/example_bigquery_query.sql | 0 .../resources/update_table_schema.json | 0 .../cloud/bigquery/resources/us-states.csv | 0 .../system/google/cloud/bigtable}/__init__.py | 0 .../google/cloud/bigtable/example_bigtable.py | 7 +- .../google/cloud/cloud_batch}/__init__.py | 0 .../cloud/cloud_batch/example_cloud_batch.py | 7 +- .../google/cloud/cloud_build}/__init__.py | 0 .../cloud/cloud_build/example_cloud_build.py | 9 +- .../example_cloud_build_trigger.py | 9 +- .../cloud/cloud_build/resources}/__init__.py | 0 .../resources/example_cloud_build.yaml | 0 .../google/cloud/cloud_functions}/__init__.py | 0 .../cloud_functions/example_functions.py | 7 +- .../cloud/cloud_memorystore}/__init__.py | 0 .../example_cloud_memorystore_memcached.py | 4 +- .../example_cloud_memorystore_redis.py | 7 +- .../google/cloud/cloud_run}/__init__.py | 0 .../cloud/cloud_run/example_cloud_run.py | 4 +- .../cloud_run/example_cloud_run_service.py | 4 +- .../google/cloud/cloud_sql}/__init__.py | 0 .../cloud/cloud_sql/example_cloud_sql.py | 7 +- .../cloud_sql/example_cloud_sql_query.py | 7 +- .../cloud_sql/example_cloud_sql_query_ssl.py | 7 +- .../system/google/cloud/composer}/__init__.py | 0 .../cloud/composer/example_cloud_composer.py | 4 +- .../system/google/cloud/compute}/__init__.py | 0 .../google/cloud/compute/example_compute.py | 7 +- .../cloud/compute/example_compute_igm.py | 7 +- .../cloud/compute/example_compute_ssh.py | 7 +- .../compute/example_compute_ssh_os_login.py | 7 +- .../compute/example_compute_ssh_parallel.py | 7 +- .../cloud/data_loss_prevention}/__init__.py | 0 .../example_dlp_deidentify_content.py | 7 +- .../example_dlp_info_types.py | 7 +- .../example_dlp_inspect_template.py | 7 +- .../data_loss_prevention/example_dlp_job.py | 7 +- .../example_dlp_job_trigger.py | 7 +- .../resources}/__init__.py | 0 .../resources/dictionary.txt | 0 .../system/google/cloud/dataflow}/__init__.py | 0 .../cloud/dataflow/example_dataflow_go.py | 2 +- .../dataflow/example_dataflow_native_java.py | 4 +- .../example_dataflow_native_python.py | 4 +- .../example_dataflow_native_python_async.py | 4 +- .../dataflow/example_dataflow_pipeline.py | 4 +- .../example_dataflow_sensors_deferrable.py | 4 +- .../cloud/dataflow/example_dataflow_sql.py | 7 +- .../example_dataflow_streaming_python.py | 4 +- .../dataflow/example_dataflow_template.py | 7 +- .../cloud/dataflow/example_dataflow_yaml.py | 7 +- .../cloud/dataflow}/resources/__init__.py | 0 .../google/cloud/dataflow/resources/input.csv | 0 .../cloud/dataflow/resources/schema.json | 0 .../google/cloud/dataflow/resources/text.txt | 0 .../cloud/dataflow/resources/wordcount.go | 0 .../system/google/cloud/dataform}/__init__.py | 0 .../google/cloud/dataform/example_dataform.py | 7 +- .../google/cloud/datafusion}/__init__.py | 0 .../cloud/datafusion/example_datafusion.py | 7 +- .../google/cloud/datapipelines}/__init__.py | 0 .../datapipelines/example_datapipeline.py | 7 +- .../system/google/cloud/dataplex}/__init__.py | 0 .../google/cloud/dataplex/example_dataplex.py | 7 +- .../cloud/dataplex/example_dataplex_dp.py | 7 +- .../cloud/dataplex/example_dataplex_dq.py | 7 +- .../system/google/cloud/dataprep}/__init__.py | 0 .../google/cloud/dataprep/example_dataprep.py | 7 +- .../system/google/cloud/dataproc}/__init__.py | 0 .../cloud/dataproc/example_dataproc_batch.py | 7 +- .../example_dataproc_batch_deferrable.py | 7 +- .../example_dataproc_batch_persistent.py | 7 +- ...cluster_create_existing_stopped_cluster.py | 7 +- .../example_dataproc_cluster_deferrable.py | 7 +- .../example_dataproc_cluster_diagnose.py | 7 +- .../example_dataproc_cluster_generator.py | 7 +- .../example_dataproc_cluster_start_stop.py | 7 +- .../example_dataproc_cluster_update.py | 7 +- .../cloud/dataproc/example_dataproc_flink.py | 7 +- .../cloud/dataproc/example_dataproc_gke.py | 7 +- .../cloud/dataproc/example_dataproc_hadoop.py | 7 +- .../cloud/dataproc/example_dataproc_hive.py | 7 +- .../cloud/dataproc/example_dataproc_pig.py | 7 +- .../cloud/dataproc/example_dataproc_presto.py | 7 +- .../dataproc/example_dataproc_pyspark.py | 7 +- .../cloud/dataproc/example_dataproc_spark.py | 7 +- .../dataproc/example_dataproc_spark_async.py | 7 +- .../example_dataproc_spark_deferrable.py | 7 +- .../dataproc/example_dataproc_spark_sql.py | 7 +- .../cloud/dataproc/example_dataproc_sparkr.py | 7 +- .../cloud/dataproc/example_dataproc_trino.py | 7 +- .../dataproc/example_dataproc_workflow.py | 7 +- .../example_dataproc_workflow_deferrable.py | 7 +- .../cloud/dataproc_metastore}/__init__.py | 0 .../example_dataproc_metastore.py | 7 +- .../example_dataproc_metastore_backup.py | 7 +- ...ataproc_metastore_hive_partition_sensor.py | 7 +- .../google/cloud/datastore}/__init__.py | 0 .../datastore/example_datastore_commit.py | 7 +- .../datastore/example_datastore_query.py | 7 +- .../datastore/example_datastore_rollback.py | 7 +- .../system/google/cloud/gcs}/__init__.py | 0 .../cloud/gcs/example_calendar_to_gcs.py | 4 +- .../google/cloud/gcs/example_firestore.py | 4 +- .../google/cloud/gcs/example_gcs_acl.py | 7 +- .../cloud/gcs/example_gcs_copy_delete.py | 7 +- .../google/cloud/gcs/example_gcs_sensor.py | 7 +- .../cloud/gcs/example_gcs_to_bigquery.py | 7 +- .../gcs/example_gcs_to_bigquery_async.py | 7 +- .../google/cloud/gcs/example_gcs_to_gcs.py | 7 +- .../google/cloud/gcs/example_gcs_to_gdrive.py | 7 +- .../google/cloud/gcs/example_gcs_to_sheets.py | 4 +- .../google/cloud/gcs/example_gcs_transform.py | 7 +- .../gcs/example_gcs_transform_timespan.py | 7 +- .../cloud/gcs/example_gcs_upload_download.py | 7 +- .../google/cloud/gcs/example_gdrive_to_gcs.py | 7 +- .../google/cloud/gcs/example_mssql_to_gcs.py | 7 +- .../google/cloud/gcs/example_mysql_to_gcs.py | 4 +- .../google/cloud/gcs/example_oracle_to_gcs.py | 7 +- .../google/cloud/gcs/example_s3_to_gcs.py | 7 +- .../google/cloud/gcs/example_sftp_to_gcs.py | 7 +- .../google/cloud/gcs/example_sheets.py | 4 +- .../google/cloud/gcs/example_sheets_to_gcs.py | 4 +- .../google/cloud/gcs/example_trino_to_gcs.py | 4 +- .../google/cloud/gcs/resources}/__init__.py | 0 .../cloud/gcs/resources/example_upload.txt | 0 .../google/cloud/gcs/resources/tmp.tar.gz | Bin .../cloud/gcs/resources/transform_script.py | 0 .../cloud/gcs/resources/transform_timespan.py | 0 .../google/cloud/gcs/resources/us-states.csv | 0 .../cloud/kubernetes_engine}/__init__.py | 0 .../example_kubernetes_engine.py | 7 +- .../example_kubernetes_engine_async.py | 7 +- .../example_kubernetes_engine_job.py | 4 +- .../example_kubernetes_engine_kueue.py | 4 +- .../example_kubernetes_engine_resource.py | 4 +- .../google/cloud/life_sciences}/__init__.py | 0 .../life_sciences/example_life_sciences.py | 7 +- .../life_sciences/resources}/__init__.py | 0 .../google/cloud/life_sciences/resources/file | 0 .../google/cloud/ml_engine}/__init__.py | 0 .../cloud/ml_engine/example_mlengine.py | 4 +- .../cloud/natural_language}/__init__.py | 0 .../example_natural_language.py | 4 +- .../system/google/cloud/pubsub}/__init__.py | 0 .../google/cloud/pubsub/example_pubsub.py | 4 +- .../cloud/pubsub/example_pubsub_deferrable.py | 4 +- .../system/google/cloud/spanner}/__init__.py | 0 .../google/cloud/spanner/example_spanner.py | 7 +- .../google/cloud/speech_to_text}/__init__.py | 0 .../speech_to_text/example_speech_to_text.py | 7 +- .../google/cloud/sql_to_sheets}/__init__.py | 0 .../sql_to_sheets/example_sql_to_sheets.py | 4 +- .../google/cloud/stackdriver}/__init__.py | 0 .../cloud/stackdriver/example_stackdriver.py | 7 +- .../cloud/storage_transfer}/__init__.py | 0 ...mple_cloud_storage_transfer_service_aws.py | 7 +- ...mple_cloud_storage_transfer_service_gcp.py | 4 +- ...oud_storage_transfer_service_gcs_to_gcs.py | 4 +- .../storage_transfer/resources}/__init__.py | 0 .../resources/transfer_service_gcp_file | 0 .../transfer_service_gcs_to_gcs_file | 0 .../system/google/cloud/tasks}/__init__.py | 0 .../google/cloud/tasks/example_queue.py | 4 +- .../google/cloud/tasks/example_tasks.py | 4 +- .../google/cloud/text_to_speech}/__init__.py | 0 .../text_to_speech/example_text_to_speech.py | 7 +- .../google/cloud/transfers}/__init__.py | 0 .../cloud/transfers/example_gcs_to_sftp.py | 7 +- .../transfers/example_gdrive_to_local.py | 7 +- .../transfers/example_postgres_to_gcs.py | 4 +- .../cloud/transfers/resources}/__init__.py | 0 .../cloud/transfers/resources/empty.txt | 0 .../google/cloud/translate}/__init__.py | 0 .../cloud/translate/example_translate.py | 4 +- .../cloud/translate_speech}/__init__.py | 0 .../example_translate_speech.py | 4 +- .../google/cloud/vertex_ai}/__init__.py | 0 ..._vertex_ai_auto_ml_forecasting_training.py | 4 +- ...xample_vertex_ai_auto_ml_image_training.py | 4 +- ...example_vertex_ai_auto_ml_list_training.py | 4 +- ...mple_vertex_ai_auto_ml_tabular_training.py | 4 +- ...xample_vertex_ai_auto_ml_video_training.py | 4 +- .../example_vertex_ai_batch_prediction_job.py | 4 +- .../example_vertex_ai_custom_container.py | 4 +- .../vertex_ai/example_vertex_ai_custom_job.py | 4 +- ...ple_vertex_ai_custom_job_python_package.py | 4 +- .../vertex_ai/example_vertex_ai_dataset.py | 4 +- .../vertex_ai/example_vertex_ai_endpoint.py | 4 +- .../example_vertex_ai_generative_model.py | 4 +- ...ample_vertex_ai_generative_model_tuning.py | 4 +- ...ple_vertex_ai_hyperparameter_tuning_job.py | 4 +- .../example_vertex_ai_list_custom_jobs.py | 4 +- .../example_vertex_ai_model_service.py | 4 +- .../example_vertex_ai_pipeline_job.py | 4 +- .../cloud/video_intelligence}/__init__.py | 0 .../example_video_intelligence.py | 4 +- .../system/google/cloud/vision}/__init__.py | 0 .../vision/example_vision_annotate_image.py | 6 +- .../vision/example_vision_autogenerated.py | 6 +- .../cloud/vision/example_vision_explicit.py | 6 +- .../google/cloud/workflows}/__init__.py | 0 .../cloud/workflows/example_workflows.py | 4 +- .../tests/system}/google/conftest.py | 0 .../system/google/datacatalog}/__init__.py | 0 .../example_datacatalog_entries.py | 7 +- .../example_datacatalog_search_catalog.py | 7 +- .../example_datacatalog_tag_templates.py | 7 +- .../datacatalog/example_datacatalog_tags.py | 7 +- .../tests/system/google/firebase}/__init__.py | 0 .../tests/system/google/leveldb}/__init__.py | 0 .../system}/google/leveldb/example_leveldb.py | 4 +- .../google/marketing_platform}/__init__.py | 0 .../example_analytics_admin.py | 4 +- .../example_campaign_manager.py | 7 +- .../marketing_platform/example_search_ads.py | 2 +- .../tests/system/google/suite}/__init__.py | 0 .../google/suite/example_local_to_drive.py | 4 +- .../google/suite/resources}/__init__.py | 0 .../system}/google/suite/resources/test1 | 0 .../system}/google/suite/resources/test2 | 0 .../system/google/workplace}/__init__.py | 0 .../tests/system/http}/__init__.py | 0 .../tests/system}/http/example_http.py | 2 +- .../tests/system/influxdb}/__init__.py | 0 .../system}/influxdb/example_influxdb.py | 4 +- .../influxdb/example_influxdb_query.py | 2 +- .../tests/system/jdbc}/__init__.py | 0 .../system}/jdbc/example_jdbc_queries.py | 4 +- .../tests/system/jenkins}/__init__.py | 0 .../jenkins/example_jenkins_job_trigger.py | 2 +- .../tests/system/microsoft}/__init__.py | 0 .../tests/system/microsoft/azure}/__init__.py | 0 .../azure/example_adf_run_pipeline.py | 4 +- .../microsoft/azure/example_adls_create.py | 4 +- .../microsoft/azure/example_adls_delete.py | 4 +- .../microsoft/azure/example_adls_list.py | 4 +- .../azure/example_azure_batch_operator.py | 2 +- .../example_azure_container_instances.py | 2 +- .../microsoft/azure/example_azure_cosmosdb.py | 4 +- .../azure/example_azure_service_bus.py | 4 +- .../microsoft/azure/example_azure_synapse.py | 2 +- .../microsoft/azure/example_fileshare.py | 4 +- .../microsoft/azure/example_local_to_adls.py | 4 +- .../microsoft/azure/example_local_to_wasb.py | 4 +- .../microsoft/azure/example_msfabric.py | 4 +- .../microsoft/azure/example_msgraph.py | 4 +- .../microsoft/azure/example_powerbi.py | 4 +- .../azure/example_powerbi_dataset_refresh.py | 4 +- .../microsoft/azure/example_s3_to_wasb.py | 7 +- .../microsoft/azure/example_sftp_to_wasb.py | 4 +- .../azure/example_synapse_run_pipeline.py | 4 +- .../microsoft/azure/example_wasb_sensors.py | 2 +- .../tests/system/microsoft/mssql}/__init__.py | 0 .../system}/microsoft/mssql/create_table.sql | 0 .../system}/microsoft/mssql/example_mssql.py | 4 +- .../tests/system/microsoft/winrm}/__init__.py | 0 .../system}/microsoft/winrm/example_winrm.py | 4 +- .../tests/system/mysql}/__init__.py | 0 .../tests/system}/mysql/example_mysql.py | 2 +- .../tests/system/neo4j}/__init__.py | 0 .../tests/system}/neo4j/example_neo4j.py | 2 +- .../tests/system/openai}/__init__.py | 0 .../tests/system}/openai/example_openai.py | 2 +- .../openai/example_trigger_batch_operator.py | 2 +- .../tests/system/opensearch}/__init__.py | 0 .../system}/opensearch/example_opensearch.py | 4 +- .../tests/system/opsgenie}/__init__.py | 0 .../opsgenie/example_opsgenie_alert.py | 2 +- .../opsgenie/example_opsgenie_notifier.py | 2 +- .../tests/system/papermill}/__init__.py | 0 .../tests/system}/papermill/conftest.py | 0 .../system}/papermill/example_papermill.py | 2 +- .../example_papermill_remote_verify.py | 2 +- .../papermill/example_papermill_verify.py | 2 +- .../system}/papermill/input_notebook.ipynb | 0 .../tests/system/pgvector}/__init__.py | 0 .../system}/pgvector/example_pgvector.py | 2 +- .../pgvector/example_pgvector_openai.py | 2 +- .../tests/system/pinecone}/__init__.py | 0 .../pinecone/example_create_pod_index.py | 2 +- .../example_create_serverless_index.py | 2 +- .../system}/pinecone/example_dag_pinecone.py | 2 +- .../pinecone/example_pinecone_cohere.py | 2 +- .../pinecone/example_pinecone_openai.py | 2 +- .../tests/system/postgres}/__init__.py | 0 .../system}/postgres/example_postgres.py | 4 +- .../tests/system/presto}/__init__.py | 0 .../system}/presto/example_gcs_to_presto.py | 2 +- .../tests/system/qdrant}/__init__.py | 0 .../system}/qdrant/example_dag_qdrant.py | 2 +- .../tests/system/redis}/__init__.py | 0 .../system}/redis/example_redis_publish.py | 4 +- .../tests/system/salesforce}/__init__.py | 0 .../tests/system}/salesforce/example_bulk.py | 2 +- .../example_salesforce_apex_rest.py | 2 +- .../tests/system/samba}/__init__.py | 0 .../system}/samba/example_gcs_to_samba.py | 7 +- .../tests/system/sftp}/__init__.py | 0 .../tests/system}/sftp/example_sftp_sensor.py | 4 +- .../tests/system/singularity}/__init__.py | 0 .../singularity/example_singularity.py | 2 +- .../tests/system/slack}/__init__.py | 0 .../tests/system}/slack/example_slack.py | 2 +- .../system}/slack/example_slack_webhook.py | 2 +- .../system}/slack/example_sql_to_slack.py | 2 +- .../slack/example_sql_to_slack_webhook.py | 2 +- .../tests/system/snowflake}/__init__.py | 0 .../snowflake/example_copy_into_snowflake.py | 2 +- .../system}/snowflake/example_snowflake.py | 2 +- ...e_snowflake_snowflake_op_template_file.sql | 0 .../snowflake/example_snowpark_decorator.py | 3 +- .../snowflake/example_snowpark_operator.py | 3 +- .../tests/system/sqlite}/__init__.py | 0 .../tests/system}/sqlite/create_table.sql | 0 .../tests/system}/sqlite/example_sqlite.py | 4 +- .../tests/system/tableau}/__init__.py | 0 .../tests/system}/tableau/example_tableau.py | 2 +- .../tests/system/telegram}/__init__.py | 0 .../system}/telegram/example_telegram.py | 2 +- .../tests/system/teradata}/__init__.py | 0 .../system}/teradata/create_ssl_table.sql | 0 .../tests/system}/teradata/create_table.sql | 0 ...example_azure_blob_to_teradata_transfer.py | 4 +- .../example_s3_to_teradata_transfer.py | 4 +- .../system}/teradata/example_ssl_teradata.py | 4 +- .../system}/teradata/example_teradata.py | 4 +- .../teradata/example_teradata_call_sp.py | 4 +- .../example_teradata_compute_cluster.py | 4 +- .../example_teradata_to_teradata_transfer.py | 4 +- .../tests/system/trino}/__init__.py | 0 .../system}/trino/example_gcs_to_trino.py | 2 +- .../tests/system}/trino/example_trino.py | 2 +- .../tests/system/weaviate}/__init__.py | 0 .../weaviate/example_weaviate_cohere.py | 2 +- .../example_weaviate_dynamic_mapping_dag.py | 2 +- .../weaviate/example_weaviate_openai.py | 2 +- .../weaviate/example_weaviate_operator.py | 2 +- .../weaviate/example_weaviate_using_hook.py | 2 +- .../example_weaviate_vectorizer_dag.py | 2 +- ...example_weaviate_without_vectorizer_dag.py | 2 +- .../weaviate/jeopardy_data_with_vectors.json | 0 .../jeopardy_data_without_vectors.json | 0 .../jeopardy_doc_data_without_vectors.json | 0 .../tests/system/yandex}/__init__.py | 0 .../system}/yandex/example_yandexcloud.py | 7 +- .../yandex/example_yandexcloud_dataproc.py | 7 +- ...xample_yandexcloud_dataproc_lightweight.py | 7 +- .../system}/yandex/example_yandexcloud_yq.py | 7 +- .../tests/system/ydb}/__init__.py | 0 .../tests/system}/ydb/example_ydb.py | 4 +- .../tests/system/zendesk}/__init__.py | 0 .../zendesk/example_zendesk_custom_get.py | 2 +- .../tests/tableau}/__init__.py | 0 .../tests/tableau/hooks}/__init__.py | 0 .../tests}/tableau/hooks/test_tableau.py | 22 +- .../tests/tableau/operators}/__init__.py | 0 .../tests}/tableau/operators/test_tableau.py | 0 .../tests/tableau/sensors}/__init__.py | 0 .../tests}/tableau/sensors/test_tableau.py | 0 .../tests/telegram}/__init__.py | 0 .../tests/telegram/hooks}/__init__.py | 0 .../tests}/telegram/hooks/test_telegram.py | 0 .../tests/telegram/operators}/__init__.py | 0 .../telegram/operators/test_telegram.py | 0 .../tests/teradata}/__init__.py | 0 .../tests/teradata/hooks}/__init__.py | 0 .../tests}/teradata/hooks/test_teradata.py | 9 +- .../tests/teradata/operators}/__init__.py | 0 .../teradata/operators/test_teradata.py | 0 .../test_teradata_compute_cluster.py | 0 .../tests/teradata/transfers}/__init__.py | 0 .../transfers/test_azure_blob_to_teradata.py | 0 .../teradata/transfers/test_s3_to_teradata.py | 0 .../transfers/test_teradata_to_teradata.py | 0 .../tests/teradata/triggers}/__init__.py | 0 .../triggers/test_teradata_compute_cluster.py | 0 .../tests/teradata/utils}/__init__.py | 0 .../tests}/teradata/utils/test_constants.py | 0 .../tests/trino}/__init__.py | 0 .../tests/trino/assets}/__init__.py | 0 .../tests}/trino/assets/test_trino.py | 0 .../tests/trino/hooks}/__init__.py | 0 .../tests}/trino/hooks/test_trino.py | 0 .../tests/trino/operators}/__init__.py | 0 .../tests}/trino/operators/test_trino.py | 0 .../tests/trino/transfers}/__init__.py | 0 .../trino/transfers/test_gcs_to_trino.py | 0 .../tests/vertica}/__init__.py | 0 .../tests/vertica/hooks}/__init__.py | 0 .../tests}/vertica/hooks/test_vertica.py | 0 .../tests/vertica/operators}/__init__.py | 0 .../tests}/vertica/operators/test_vertica.py | 0 .../tests/weaviate}/__init__.py | 0 .../tests/weaviate/hooks}/__init__.py | 0 .../tests}/weaviate/hooks/test_weaviate.py | 0 .../tests/weaviate/operators}/__init__.py | 0 .../weaviate/operators/test_weaviate.py | 0 .../tests/yandex}/__init__.py | 0 .../tests/yandex/hooks}/__init__.py | 0 .../tests}/yandex/hooks/test_dataproc.py | 0 .../tests}/yandex/hooks/test_yandex.py | 21 +- .../tests}/yandex/hooks/test_yq.py | 0 .../tests/yandex/links}/__init__.py | 0 .../tests}/yandex/links/test_yq.py | 5 +- .../tests/yandex/operators}/__init__.py | 0 .../tests}/yandex/operators/test_dataproc.py | 0 .../tests}/yandex/operators/test_yq.py | 2 +- .../tests/yandex/secrets}/__init__.py | 0 .../tests}/yandex/secrets/test_lockbox.py | 0 .../tests/yandex/utils}/__init__.py | 0 .../tests}/yandex/utils/test_credentials.py | 0 .../tests}/yandex/utils/test_defaults.py | 0 .../tests}/yandex/utils/test_fields.py | 0 .../tests}/yandex/utils/test_user_agent.py | 0 .../tests/ydb}/__init__.py | 0 .../tests/ydb/hooks}/__init__.py | 0 .../tests}/ydb/hooks/test_ydb.py | 0 .../tests/ydb/operators}/__init__.py | 0 .../tests}/ydb/operators/test_ydb.py | 0 .../tests/ydb/utils/__init__.py | 0 .../tests}/ydb/utils/test_credentials.py | 0 .../tests/ydb/utils/test_defaults.py | 1 - .../tests/zendesk}/__init__.py | 1 - .../tests/zendesk/hooks}/__init__.py | 1 - .../tests}/zendesk/hooks/test_zendesk.py | 0 pyproject.toml | 93 +- scripts/ci/docker-compose/local.yml | 3 + scripts/ci/kubernetes/k8s_requirements.txt | 4 +- ...eck_providers_subpackages_all_have_init.py | 38 +- scripts/ci/pre_commit/check_system_tests.py | 8 +- .../check_system_tests_hidden_in_index.py | 4 +- scripts/ci/pre_commit/mypy_folder.py | 16 +- .../pre_commit/update_common_sql_api_stubs.py | 8 +- .../pre_commit/update_example_dags_paths.py | 13 +- .../update_providers_dependencies.py | 26 +- scripts/ci/pre_commit/version_heads_map.py | 5 +- scripts/docker/install_airflow.sh | 7 +- scripts/in_container/install_devel_deps.py | 3 +- scripts/in_container/run_mypy.sh | 4 +- .../run_provider_yaml_files_check.py | 36 +- tests/always/test_connection.py | 3 +- tests/always/test_example_dags.py | 31 +- tests/always/test_project_structure.py | 263 ++- tests/always/test_secrets.py | 5 +- tests/always/test_secrets_backends.py | 3 +- tests/always/test_secrets_local_filesystem.py | 3 +- tests/api_connexion/conftest.py | 7 +- .../endpoints/test_asset_endpoint.py | 11 +- .../endpoints/test_backfill_endpoint.py | 10 +- .../endpoints/test_config_endpoint.py | 4 +- .../endpoints/test_connection_endpoint.py | 9 +- .../endpoints/test_dag_endpoint.py | 9 +- .../endpoints/test_dag_parsing.py | 5 +- .../endpoints/test_dag_run_endpoint.py | 11 +- .../endpoints/test_dag_source_endpoint.py | 5 +- .../endpoints/test_dag_stats_endpoint.py | 5 +- .../endpoints/test_dag_warning_endpoint.py | 5 +- .../endpoints/test_event_log_endpoint.py | 7 +- .../endpoints/test_extra_link_endpoint.py | 11 +- .../endpoints/test_import_error_endpoint.py | 9 +- .../endpoints/test_log_endpoint.py | 5 +- .../test_mapped_task_instance_endpoint.py | 7 +- .../endpoints/test_plugin_endpoint.py | 9 +- .../endpoints/test_pool_endpoint.py | 9 +- .../endpoints/test_provider_endpoint.py | 3 +- .../endpoints/test_task_endpoint.py | 5 +- .../endpoints/test_task_instance_endpoint.py | 7 +- .../endpoints/test_variable_endpoint.py | 9 +- .../endpoints/test_xcom_endpoint.py | 7 +- .../schemas/test_connection_schema.py | 3 +- .../schemas/test_dag_run_schema.py | 5 +- .../schemas/test_dataset_schema.py | 3 +- .../schemas/test_error_schema.py | 5 +- .../schemas/test_plugin_schema.py | 3 +- .../schemas/test_pool_schemas.py | 3 +- .../api_connexion/schemas/test_xcom_schema.py | 3 +- tests/api_connexion/test_auth.py | 7 +- tests/api_connexion/test_parameters.py | 3 +- tests/api_connexion/test_security.py | 2 +- .../views/public/test_connections.py | 3 +- tests/api_fastapi/views/public/test_dags.py | 3 +- .../views/public/test_variables.py | 3 +- tests/api_fastapi/views/ui/test_assets.py | 3 +- tests/api_fastapi/views/ui/test_dashboard.py | 3 +- .../endpoints/test_rpc_api_endpoint.py | 5 +- tests/api_internal/test_internal_api_call.py | 3 +- tests/assets/test_manager.py | 2 +- tests/assets/tests_asset.py | 2 +- tests/auth/managers/simple/views/test_auth.py | 3 +- tests/cli/commands/test_celery_command.py | 3 +- tests/cli/commands/test_config_command.py | 3 +- tests/cli/commands/test_connection_command.py | 3 +- tests/cli/commands/test_dag_command.py | 5 +- .../commands/test_dag_processor_command.py | 3 +- tests/cli/commands/test_info_command.py | 3 +- .../cli/commands/test_internal_api_command.py | 3 +- tests/cli/commands/test_jobs_command.py | 3 +- tests/cli/commands/test_kerberos_command.py | 3 +- tests/cli/commands/test_kubernetes_command.py | 3 +- tests/cli/commands/test_plugins_command.py | 15 +- .../test_rotate_fernet_key_command.py | 5 +- tests/cli/commands/test_scheduler_command.py | 3 +- tests/cli/commands/test_task_command.py | 7 +- tests/cli/commands/test_variable_command.py | 3 +- tests/cli/commands/test_webserver_command.py | 3 +- tests/cli/conftest.py | 6 +- tests/cli/test_cli_parser.py | 3 +- tests/conftest.py | 1425 +--------------- tests/core/test_configuration.py | 5 +- tests/core/test_core.py | 3 +- tests/core/test_example_dags_system.py | 2 +- tests/core/test_impersonation_tests.py | 3 +- tests/core/test_logging_config.py | 3 +- tests/core/test_otel_tracer.py | 3 +- tests/core/test_sentry.py | 3 +- tests/core/test_settings.py | 3 +- tests/core/test_sqlalchemy_config.py | 3 +- tests/core/test_stats.py | 3 +- tests/dag_processing/test_job_runner.py | 7 +- tests/dag_processing/test_processor.py | 11 +- tests/dags/test_miscellaneous.py | 3 +- tests/dags/test_sensor.py | 3 +- tests/decorators/test_bash.py | 3 +- tests/decorators/test_python.py | 3 +- tests/deprecations_ignore.yml | 113 -- tests/executors/test_base_executor.py | 2 +- tests/executors/test_executor_loader.py | 3 +- .../cli/commands/test_celery_command.py | 3 +- .../executors/test_celery_executor.py | 3 +- tests/integration/security/test_kerberos.py | 3 +- tests/jobs/test_backfill_job.py | 9 +- tests/jobs/test_base_job.py | 3 +- tests/jobs/test_local_task_job.py | 11 +- tests/jobs/test_scheduler_job.py | 15 +- tests/jobs/test_triggerer_job.py | 3 +- tests/jobs/test_triggerer_job_logging.py | 3 +- tests/lineage/test_hook.py | 3 +- tests/lineage/test_lineage.py | 3 +- tests/listeners/class_listener.py | 3 +- .../test_dag_import_error_listener.py | 7 +- tests/models/test_backfill.py | 8 +- tests/models/test_base.py | 3 +- tests/models/test_baseoperator.py | 3 +- tests/models/test_cleartasks.py | 5 +- tests/models/test_dag.py | 24 +- tests/models/test_dagbag.py | 7 +- tests/models/test_dagcode.py | 3 +- tests/models/test_dagrun.py | 9 +- tests/models/test_dagwarning.py | 3 +- tests/models/test_mappedoperator.py | 9 +- tests/models/test_param.py | 3 +- tests/models/test_pool.py | 8 +- tests/models/test_renderedtifields.py | 5 +- tests/models/test_serialized_dag.py | 5 +- tests/models/test_skipmixin.py | 3 +- tests/models/test_taskinstance.py | 11 +- tests/models/test_timestamp.py | 3 +- tests/models/test_trigger.py | 3 +- tests/models/test_variable.py | 5 +- tests/models/test_xcom.py | 5 +- tests/models/test_xcom_arg.py | 5 +- tests/operators/test_branch_operator.py | 3 +- tests/operators/test_email.py | 3 +- tests/operators/test_generic_transfer.py | 2 +- tests/operators/test_latest_only_operator.py | 5 +- tests/operators/test_python.py | 7 +- tests/plugins/test_plugin.py | 3 +- tests/plugins/test_plugins_manager.py | 5 +- tests/secrets/test_cache.py | 3 +- tests/security/test_kerberos.py | 3 +- tests/sensors/test_base.py | 5 +- tests/sensors/test_external_task_sensor.py | 7 +- tests/serialization/test_dag_serialization.py | 57 +- tests/serialization/test_pydantic_models.py | 3 +- tests/serialization/test_serde.py | 3 +- .../serialization/test_serialized_objects.py | 3 +- .../example_external_task_child_deferrable.py | 2 +- ...example_external_task_parent_deferrable.py | 4 +- tests/system/example_empty.py | 4 +- .../providers/microsoft/azure/__init__.py | 17 - tests/system/providers/papermill/__init__.py | 17 - .../task_runner/test_standard_task_runner.py | 5 +- .../deps/test_pool_slots_available_dep.py | 3 +- tests/ti_deps/deps/test_prev_dagrun_dep.py | 5 +- .../deps/test_ready_to_reschedule_dep.py | 3 +- tests/utils/log/test_log_reader.py | 5 +- tests/utils/log/test_secrets_masker.py | 3 +- tests/utils/log/test_task_context_logger.py | 5 +- tests/utils/test_db.py | 3 +- tests/utils/test_db_cleanup.py | 8 +- tests/utils/test_db_manager.py | 3 +- tests/utils/test_dot_renderer.py | 5 +- tests/utils/test_email.py | 3 +- tests/utils/test_file.py | 3 +- tests/utils/test_helpers.py | 5 +- tests/utils/test_log_handlers.py | 5 +- tests/utils/test_net.py | 3 +- tests/utils/test_serve_logs.py | 3 +- tests/utils/test_sqlalchemy.py | 3 +- tests/utils/test_state.py | 3 +- tests/utils/test_task_group.py | 5 +- ...test_task_handler_with_custom_formatter.py | 7 +- tests/utils/test_types.py | 3 +- tests/www/test_app.py | 5 +- tests/www/test_utils.py | 8 +- tests/www/views/conftest.py | 13 +- .../www/views/test_anonymous_as_admin_role.py | 3 +- tests/www/views/test_session.py | 5 +- tests/www/views/test_views.py | 15 +- tests/www/views/test_views_acl.py | 15 +- tests/www/views/test_views_base.py | 7 +- .../www/views/test_views_cluster_activity.py | 3 +- tests/www/views/test_views_configuration.py | 5 +- tests/www/views/test_views_connection.py | 7 +- .../www/views/test_views_custom_user_views.py | 11 +- tests/www/views/test_views_dagrun.py | 13 +- tests/www/views/test_views_dataset.py | 5 +- tests/www/views/test_views_decorators.py | 11 +- tests/www/views/test_views_extra_links.py | 7 +- tests/www/views/test_views_grid.py | 7 +- tests/www/views/test_views_home.py | 13 +- tests/www/views/test_views_log.py | 11 +- tests/www/views/test_views_mount.py | 3 +- tests/www/views/test_views_paused.py | 3 +- tests/www/views/test_views_pool.py | 3 +- tests/www/views/test_views_rate_limit.py | 7 +- tests/www/views/test_views_rendered.py | 13 +- tests/www/views/test_views_robots.py | 2 +- tests/www/views/test_views_task_norun.py | 2 +- tests/www/views/test_views_tasks.py | 15 +- tests/www/views/test_views_trigger_dag.py | 7 +- tests/www/views/test_views_variable.py | 5 +- 4407 files changed, 7355 insertions(+), 6193 deletions(-) delete mode 100644 airflow/providers/.gitignore rename scripts/ci/pre_commit/check_providers_init.py => dev/tests_common/__init__.py (62%) mode change 100755 => 100644 rename {airflow/providers/airbyte/sensors => dev/tests_common/_internals}/__init__.py (100%) rename {tests => dev/tests_common}/_internals/capture_warnings.py (100%) rename {tests => dev/tests_common}/_internals/forbidden_warnings.py (90%) create mode 100644 dev/tests_common/pyproject.toml create mode 100644 dev/tests_common/pytest_plugin.py rename {tests => dev/tests_common}/test_utils/README.md (67%) rename {tests/system/providers/google/marketing_platform => dev/tests_common/test_utils}/__init__.py (87%) rename {tests => dev/tests_common}/test_utils/api_connexion_utils.py (93%) rename {tests => dev/tests_common}/test_utils/asserts.py (100%) rename {tests => dev/tests_common}/test_utils/azure_system_helpers.py (97%) rename {tests => dev/tests_common}/test_utils/compat.py (100%) rename {tests => dev/tests_common}/test_utils/config.py (100%) rename {tests => dev/tests_common}/test_utils/db.py (89%) rename {tests => dev/tests_common}/test_utils/decorators.py (100%) rename {tests => dev/tests_common}/test_utils/fake_datetime.py (100%) rename {tests => dev/tests_common}/test_utils/gcp_system_helpers.py (90%) rename {tests => dev/tests_common}/test_utils/get_all_tests.py (100%) rename {tests => dev/tests_common}/test_utils/hdfs_utils.py (100%) rename {tests => dev/tests_common}/test_utils/logging_command_executor.py (100%) rename {tests => dev/tests_common}/test_utils/mapping.py (100%) rename {tests => dev/tests_common}/test_utils/mock_executor.py (100%) rename {tests => dev/tests_common}/test_utils/mock_operators.py (98%) rename {tests => dev/tests_common}/test_utils/mock_plugins.py (100%) rename {tests => dev/tests_common}/test_utils/mock_security_manager.py (92%) rename {airflow/providers/airbyte/triggers => dev/tests_common/test_utils/operators}/__init__.py (100%) rename {tests => dev/tests_common}/test_utils/operators/postgres_local_executor.cfg (100%) rename {airflow/providers/alibaba/cloud/log => dev/tests_common/test_utils/perf}/__init__.py (100%) rename {tests => dev/tests_common}/test_utils/perf/perf_kit/__init__.py (100%) rename {tests => dev/tests_common}/test_utils/perf/perf_kit/memory.py (100%) rename {tests => dev/tests_common}/test_utils/perf/perf_kit/python.py (100%) rename {tests => dev/tests_common}/test_utils/perf/perf_kit/repeat_and_time.py (100%) rename {tests => dev/tests_common}/test_utils/perf/perf_kit/sqlalchemy.py (100%) rename {tests => dev/tests_common}/test_utils/permissions.py (100%) rename {tests => dev/tests_common}/test_utils/providers.py (100%) rename {tests => dev/tests_common}/test_utils/remote_user_api_auth_backend.py (100%) rename {tests => dev/tests_common}/test_utils/reset_warning_registry.py (100%) rename {tests => dev/tests_common}/test_utils/salesforce_system_helpers.py (100%) rename {tests => dev/tests_common}/test_utils/sftp_system_helpers.py (100%) rename tests/system/utils/__init__.py => dev/tests_common/test_utils/system_tests.py (100%) rename {tests => dev/tests_common}/test_utils/system_tests_class.py (97%) rename {tests => dev/tests_common}/test_utils/terraform.py (95%) rename {tests => dev/tests_common}/test_utils/timetables.py (100%) rename {tests/system/utils => dev/tests_common/test_utils}/watcher.py (100%) rename {tests => dev/tests_common}/test_utils/www.py (100%) delete mode 100644 docs/exts/provider_init_hack.py create mode 100644 providers/.gitignore rename {tests/test_utils => providers}/__init__.py (74%) create mode 100644 providers/pyproject.toml create mode 100644 providers/src/airflow/providers/.gitignore rename {airflow => providers/src/airflow}/providers/MANAGING_PROVIDERS_LIFECYCLE.rst (98%) rename {airflow => providers/src/airflow}/providers/airbyte/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/airbyte/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/airbyte/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/airbyte/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/airbyte/hooks/airbyte.py (100%) rename {airflow => providers/src/airflow}/providers/airbyte/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/airbyte/operators/airbyte.py (100%) rename {airflow => providers/src/airflow}/providers/airbyte/provider.yaml (100%) rename {airflow/providers/alibaba/cloud/operators => providers/src/airflow/providers/airbyte/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/airbyte/sensors/airbyte.py (100%) rename {airflow/providers/alibaba/cloud/sensors => providers/src/airflow/providers/airbyte/triggers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/airbyte/triggers/airbyte.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/alibaba/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/alibaba/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/hooks/analyticdb_spark.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/hooks/oss.py (100%) rename {airflow/providers/amazon/aws => providers/src/airflow/providers/alibaba/cloud/log}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/log/oss_task_handler.py (100%) rename {airflow/providers/amazon/aws/assets => providers/src/airflow/providers/alibaba/cloud/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/operators/analyticdb_spark.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/operators/oss.py (100%) rename {airflow/providers/amazon/aws/auth_manager => providers/src/airflow/providers/alibaba/cloud/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/sensors/analyticdb_spark.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/cloud/sensors/oss_key.py (100%) rename {airflow => providers/src/airflow}/providers/alibaba/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/amazon/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/amazon/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/amazon/__init__.py (100%) rename {airflow/providers/amazon/aws/auth_manager/avp => providers/src/airflow/providers/amazon/aws}/__init__.py (100%) rename {airflow/providers/amazon/aws/auth_manager/cli => providers/src/airflow/providers/amazon/aws/assets}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/assets/s3.py (100%) rename {airflow/providers/amazon/aws/auth_manager/security_manager => providers/src/airflow/providers/amazon/aws/auth_manager}/__init__.py (100%) rename {airflow/providers/amazon/aws/auth_manager/views => providers/src/airflow/providers/amazon/aws/auth_manager/avp}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/avp/entities.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/avp/facade.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/avp/schema.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/aws_auth_manager.py (100%) rename {airflow/providers/amazon/aws/executors => providers/src/airflow/providers/amazon/aws/auth_manager/cli}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/cli/avp_commands.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/cli/definition.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/constants.py (100%) rename {airflow/providers/amazon/aws/executors/utils => providers/src/airflow/providers/amazon/aws/auth_manager/security_manager}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/security_manager/aws_security_manager_override.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/user.py (100%) rename {airflow/providers/amazon/aws/fs => providers/src/airflow/providers/amazon/aws/auth_manager/views}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/auth_manager/views/auth.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/exceptions.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/Dockerfile (100%) rename {airflow/providers/amazon/aws/hooks => providers/src/airflow/providers/amazon/aws/executors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/batch/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/batch/batch_executor.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/batch/batch_executor_config.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/batch/boto_schema.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/batch/utils.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/ecs/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/ecs/boto_schema.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/ecs/ecs_executor.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/ecs/ecs_executor_config.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/ecs/utils.py (100%) rename {airflow/providers/amazon/aws/links => providers/src/airflow/providers/amazon/aws/executors/utils}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/utils/base_config_keys.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/executors/utils/exponential_backoff_retry.py (100%) rename {airflow/providers/amazon/aws/log => providers/src/airflow/providers/amazon/aws/fs}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/fs/s3.py (100%) rename {airflow/providers/amazon/aws/notifications => providers/src/airflow/providers/amazon/aws/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/appflow.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/athena.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/athena_sql.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/base_aws.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/batch_client.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/batch_waiters.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/batch_waiters.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/bedrock.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/chime.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/cloud_formation.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/comprehend.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/datasync.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/dms.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/dynamodb.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/ec2.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/ecr.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/ecs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/eks.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/elasticache_replication_group.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/emr.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/eventbridge.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/glacier.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/glue.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/glue_catalog.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/glue_crawler.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/glue_databrew.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/kinesis.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/kinesis_analytics.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/lambda_function.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/logs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/neptune.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/opensearch_serverless.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/quicksight.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/rds.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/redshift_cluster.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/redshift_data.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/redshift_sql.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/sagemaker.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/secrets_manager.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/ses.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/sns.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/sqs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/ssm.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/step_function.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/sts.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/hooks/verified_permissions.py (100%) rename {airflow/providers/amazon/aws/operators => providers/src/airflow/providers/amazon/aws/links}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/links/athena.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/links/base_aws.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/links/batch.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/links/emr.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/links/glue.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/links/logs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/links/step_function.py (100%) rename {airflow/providers/amazon/aws/secrets => providers/src/airflow/providers/amazon/aws/log}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/log/cloudwatch_task_handler.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/log/s3_task_handler.py (100%) rename {airflow/providers/amazon/aws/sensors => providers/src/airflow/providers/amazon/aws/notifications}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/notifications/chime.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/notifications/sns.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/notifications/sqs.py (100%) rename {airflow/providers/amazon/aws/transfers => providers/src/airflow/providers/amazon/aws/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/appflow.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/athena.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/base_aws.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/batch.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/bedrock.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/cloud_formation.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/comprehend.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/datasync.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/dms.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/ec2.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/ecs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/eks.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/emr.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/eventbridge.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/glacier.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/glue.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/glue_crawler.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/glue_databrew.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/kinesis_analytics.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/lambda_function.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/neptune.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/quicksight.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/rds.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/redshift_cluster.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/redshift_data.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/sagemaker.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/sns.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/sqs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/operators/step_function.py (100%) rename {airflow/providers/amazon/aws/waiters => providers/src/airflow/providers/amazon/aws/secrets}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/secrets/secrets_manager.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/secrets/systems_manager.py (100%) rename {airflow/providers/apache => providers/src/airflow/providers/amazon/aws/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/athena.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/base_aws.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/batch.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/bedrock.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/cloud_formation.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/comprehend.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/dms.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/dynamodb.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/ec2.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/ecs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/eks.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/emr.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/glacier.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/glue.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/glue_catalog_partition.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/glue_crawler.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/kinesis_analytics.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/lambda_function.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/opensearch_serverless.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/quicksight.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/rds.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/redshift_cluster.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/sagemaker.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/sqs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/sensors/step_function.py (100%) rename {airflow/providers/apache/beam/triggers => providers/src/airflow/providers/amazon/aws/transfers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/azure_blob_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/base.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/dynamodb_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/exasol_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/ftp_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/gcs_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/glacier_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/google_api_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/hive_to_dynamodb.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/http_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/imap_attachment_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/local_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/mongo_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/redshift_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/s3_to_dynamodb.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/s3_to_ftp.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/s3_to_redshift.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/s3_to_sftp.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/s3_to_sql.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/salesforce_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/sftp_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/transfers/sql_to_s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/README.md (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/athena.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/base.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/batch.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/bedrock.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/comprehend.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/ec2.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/ecs.py (96%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/eks.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/emr.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/glue.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/glue_crawler.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/glue_databrew.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/kinesis_analytics.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/lambda_function.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/neptune.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/opensearch_serverless.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/rds.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/redshift_cluster.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/redshift_data.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/s3.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/sagemaker.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/sqs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/triggers/step_function.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/asset_compat_lineage_collector.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/connection_wrapper.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/eks_get_token.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/emailer.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/identifiers.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/mixins.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/openlineage.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/rds.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/redshift.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/sagemaker.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/sqs.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/suppress.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/tags.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/task_log_fetcher.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/waiter.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/utils/waiter_with_logging.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/README.md (100%) rename {airflow/providers/apache/cassandra/hooks => providers/src/airflow/providers/amazon/aws/waiters}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/appflow.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/athena.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/base_waiter.py (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/batch.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/bedrock-agent.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/bedrock.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/comprehend.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/databrew.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/dynamodb.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/ecs.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/eks.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/emr-containers.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/emr-serverless.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/emr.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/glue.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/kinesisanalyticsv2.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/neptune.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/opensearchserverless.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/rds.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/redshift.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/sagemaker.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/aws/waiters/stepfunctions.json (100%) rename {airflow => providers/src/airflow}/providers/amazon/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/README.md (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/hooks/beam.py (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/operators/beam.py (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/provider.yaml (100%) rename {airflow/providers/apache/cassandra/sensors => providers/src/airflow/providers/apache/beam/triggers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/beam/triggers/beam.py (100%) rename {airflow => providers/src/airflow}/providers/apache/cassandra/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/cassandra/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/cassandra/__init__.py (100%) rename {airflow/providers/apache/druid/transfers => providers/src/airflow/providers/apache/cassandra/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/cassandra/hooks/cassandra.py (100%) rename {airflow => providers/src/airflow}/providers/apache/cassandra/provider.yaml (100%) rename {airflow/providers/apache/hdfs/log => providers/src/airflow/providers/apache/cassandra/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/cassandra/sensors/record.py (100%) rename {airflow => providers/src/airflow}/providers/apache/cassandra/sensors/table.py (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/hooks/drill.py (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/operators/drill.py (100%) rename {airflow => providers/src/airflow}/providers/apache/drill/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/hooks/druid.py (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/operators/druid.py (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/operators/druid_check.py (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/provider.yaml (100%) rename {airflow/providers/apache/hive => providers/src/airflow/providers/apache/druid}/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/druid/transfers/hive_to_druid.py (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/operators/flink_kubernetes.py (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/flink/sensors/flink_kubernetes.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/hooks/hdfs.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/hooks/webhdfs.py (100%) rename {airflow/providers/apache/kafka/hooks => providers/src/airflow/providers/apache/hdfs/log}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/log/hdfs_task_handler.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/sensors/hdfs.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hdfs/sensors/web_hdfs.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/hooks/hive.py (99%) rename {airflow => providers/src/airflow}/providers/apache/hive/macros/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/macros/hive.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/operators/hive.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/operators/hive_stats.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/plugins/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/plugins/hive.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/sensors/hive_partition.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/sensors/metastore_partition.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/sensors/named_hive_partition.py (100%) rename {airflow/providers/apache/kafka/operators => providers/src/airflow/providers/apache/hive/transfers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/transfers/hive_to_mysql.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/transfers/hive_to_samba.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/transfers/mssql_to_hive.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/transfers/mysql_to_hive.py (100%) rename {airflow => providers/src/airflow}/providers/apache/hive/transfers/s3_to_hive.py (97%) rename {airflow => providers/src/airflow}/providers/apache/hive/transfers/vertica_to_hive.py (100%) rename {airflow => providers/src/airflow}/providers/apache/iceberg/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/iceberg/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/iceberg/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/iceberg/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/iceberg/hooks/iceberg.py (100%) rename {airflow => providers/src/airflow}/providers/apache/iceberg/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/impala/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/impala/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/impala/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/impala/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/impala/hooks/impala.py (100%) rename {airflow => providers/src/airflow}/providers/apache/impala/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/__init__.py (100%) rename {airflow/providers/apache/kafka/sensors => providers/src/airflow/providers/apache/kafka/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/hooks/base.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/hooks/client.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/hooks/consume.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/hooks/produce.py (100%) rename {airflow/providers/apache/kafka/triggers => providers/src/airflow/providers/apache/kafka/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/operators/consume.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/operators/produce.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/provider.yaml (100%) rename {airflow/providers/apache/kylin/hooks => providers/src/airflow/providers/apache/kafka/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/sensors/kafka.py (100%) rename {airflow/providers/apache/kylin/operators => providers/src/airflow/providers/apache/kafka/triggers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kafka/triggers/await_message.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kylin/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/kylin/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/kylin/__init__.py (100%) rename {airflow/providers/apache/livy => providers/src/airflow/providers/apache/kylin}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kylin/hooks/kylin.py (100%) rename {airflow/providers/apache/livy => providers/src/airflow/providers/apache/kylin}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kylin/operators/kylin_cube.py (100%) rename {airflow => providers/src/airflow}/providers/apache/kylin/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/__init__.py (100%) rename {airflow/providers/apache/livy/sensors => providers/src/airflow/providers/apache/livy/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/hooks/livy.py (100%) rename {airflow/providers/apache/livy/triggers => providers/src/airflow/providers/apache/livy/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/operators/livy.py (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/provider.yaml (100%) rename {airflow/providers/arangodb/example_dags => providers/src/airflow/providers/apache/livy/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/sensors/livy.py (100%) rename {airflow/providers/arangodb/hooks => providers/src/airflow/providers/apache/livy/triggers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/livy/triggers/livy.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/hooks/pig.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/operators/pig.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pig/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/pinot/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/pinot/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/pinot/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pinot/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pinot/hooks/pinot.py (100%) rename {airflow => providers/src/airflow}/providers/apache/pinot/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/decorators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/decorators/pyspark.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/hooks/spark_connect.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/hooks/spark_jdbc.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/hooks/spark_jdbc_script.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/hooks/spark_sql.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/hooks/spark_submit.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/operators/spark_jdbc.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/operators/spark_sql.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/operators/spark_submit.py (100%) rename {airflow => providers/src/airflow}/providers/apache/spark/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/apprise/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/apprise/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/apprise/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apprise/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apprise/hooks/apprise.py (100%) rename {airflow => providers/src/airflow}/providers/apprise/notifications/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/apprise/notifications/apprise.py (100%) rename {airflow => providers/src/airflow}/providers/apprise/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/arangodb/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/arangodb/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/arangodb/__init__.py (100%) rename {airflow/providers/arangodb/operators => providers/src/airflow/providers/arangodb/example_dags}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/arangodb/example_dags/example_arangodb.py (100%) rename {airflow/providers/arangodb/sensors => providers/src/airflow/providers/arangodb/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/arangodb/hooks/arangodb.py (100%) rename {airflow/providers/atlassian => providers/src/airflow/providers/arangodb/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/arangodb/operators/arangodb.py (100%) rename {airflow => providers/src/airflow}/providers/arangodb/provider.yaml (100%) rename {airflow/providers/atlassian/jira/hooks => providers/src/airflow/providers/arangodb/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/arangodb/sensors/arangodb.py (100%) rename {airflow => providers/src/airflow}/providers/asana/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/asana/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/asana/README.md (100%) rename {airflow => providers/src/airflow}/providers/asana/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/asana/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/asana/hooks/asana.py (100%) rename {airflow => providers/src/airflow}/providers/asana/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/asana/operators/asana_tasks.py (100%) rename {airflow => providers/src/airflow}/providers/asana/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/__init__.py (100%) rename {airflow/providers/atlassian/jira/notifications => providers/src/airflow/providers/atlassian/jira/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/hooks/jira.py (100%) rename {airflow/providers/atlassian/jira/operators => providers/src/airflow/providers/atlassian/jira/notifications}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/notifications/jira.py (100%) rename {airflow/providers/atlassian/jira/sensors => providers/src/airflow/providers/atlassian/jira/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/operators/jira.py (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/provider.yaml (100%) rename {airflow/providers/celery/cli => providers/src/airflow/providers/atlassian/jira/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/atlassian/jira/sensors/jira.py (100%) rename {airflow => providers/src/airflow}/providers/celery/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/celery/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/celery/__init__.py (100%) rename {airflow/providers/cncf/kubernetes/backcompat => providers/src/airflow/providers/celery/cli}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/celery/cli/celery_command.py (100%) rename {airflow => providers/src/airflow}/providers/celery/executors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/celery/executors/celery_executor.py (100%) rename {airflow => providers/src/airflow}/providers/celery/executors/celery_executor_utils.py (100%) rename {airflow => providers/src/airflow}/providers/celery/executors/celery_kubernetes_executor.py (100%) rename {airflow => providers/src/airflow}/providers/celery/executors/default_celery.py (100%) rename {airflow => providers/src/airflow}/providers/celery/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/celery/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/celery/sensors/celery_queue.py (100%) rename {airflow => providers/src/airflow}/providers/cloudant/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/cloudant/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/cloudant/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cloudant/cloudant_fake.py (100%) rename {airflow => providers/src/airflow}/providers/cloudant/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cloudant/hooks/cloudant.py (100%) rename {airflow => providers/src/airflow}/providers/cloudant/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/__init__.py (100%) rename {airflow/providers/cncf/kubernetes/cli => providers/src/airflow/providers/cncf/kubernetes/backcompat}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/callbacks.py (100%) rename {airflow/providers/cncf/kubernetes/hooks => providers/src/airflow/providers/cncf/kubernetes/cli}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/cli/kubernetes_command.py (100%) rename {airflow/providers/cncf => providers/src/airflow/providers/cncf/kubernetes/decorators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/decorators/kubernetes.py (100%) rename {airflow/providers/cncf/kubernetes/decorators => providers/src/airflow/providers/cncf/kubernetes/executors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/executors/kubernetes_executor.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/executors/kubernetes_executor_types.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/executors/local_kubernetes_executor.py (100%) rename {airflow/providers/cncf/kubernetes/kubernetes_executor_templates => providers/src/airflow/providers/cncf/kubernetes/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/hooks/kubernetes.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/k8s_model.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/kube_client.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/kube_config.py (100%) rename {airflow/providers/cncf/kubernetes/pod_template_file_examples => providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/kubernetes_helper_functions.py (100%) rename {airflow/providers/cncf/kubernetes/executors => providers/src/airflow/providers/cncf/kubernetes/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/operators/custom_object_launcher.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/operators/job.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/operators/kubernetes_pod.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/operators/pod.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/operators/resource.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/operators/spark_kubernetes.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/pod_generator.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/pod_generator_deprecated.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/pod_launcher_deprecated.py (100%) rename {airflow/providers/cncf/kubernetes/resource_convert => providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/python_kubernetes_script.jinja2 (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/python_kubernetes_script.py (100%) rename {airflow/providers/cncf/kubernetes/sensors => providers/src/airflow/providers/cncf/kubernetes/resource_convert}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/resource_convert/configmap.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/resource_convert/env_variable.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/resource_convert/secret.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/secret.py (100%) rename {airflow/providers/cncf/kubernetes/triggers => providers/src/airflow/providers/cncf/kubernetes/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/sensors/spark_kubernetes.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/template_rendering.py (100%) rename {airflow/providers/cohere/operators => providers/src/airflow/providers/cncf/kubernetes/triggers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/triggers/job.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/triggers/kubernetes_pod.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/triggers/pod.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/utils/delete_from.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/utils/k8s_resource_iterator.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/utils/pod_manager.py (100%) rename {airflow => providers/src/airflow}/providers/cncf/kubernetes/utils/xcom_sidecar.py (100%) rename {airflow => providers/src/airflow}/providers/cohere/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/cohere/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/cohere/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cohere/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cohere/hooks/cohere.py (100%) rename {airflow/providers/common => providers/src/airflow/providers/cohere/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/cohere/operators/embedding.py (100%) rename {airflow => providers/src/airflow}/providers/cohere/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/common/compat/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/common/compat/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/common/compat/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/assets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/lineage/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/lineage/hook.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/openlineage/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/openlineage/facet.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/openlineage/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/openlineage/utils/utils.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/common/compat/security/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/compat/security/permissions.py (100%) rename {airflow => providers/src/airflow}/providers/common/io/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/common/io/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/common/io/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/io/assets/__init__.py (100%) rename {airflow/providers/common/io/operators => providers/src/airflow/providers/common/io/assets/assets}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/io/assets/file.py (100%) rename {airflow/providers/common/sql/hooks => providers/src/airflow/providers/common/io/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/io/operators/file_transfer.py (100%) rename {airflow => providers/src/airflow}/providers/common/io/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/common/io/xcom/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/io/xcom/backend.py (100%) rename {airflow => providers/src/airflow}/providers/common/sql/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/common/sql/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/common/sql/README_API.md (100%) rename {airflow => providers/src/airflow}/providers/common/sql/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/sql/doc/adr/0001-record-architecture-decisions.md (100%) rename {airflow => providers/src/airflow}/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md (100%) rename {airflow/providers/common/sql/operators => providers/src/airflow/providers/common/sql/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/sql/hooks/sql.py (99%) rename {airflow => providers/src/airflow}/providers/common/sql/hooks/sql.pyi (92%) rename {airflow/providers/common/sql/sensors => providers/src/airflow/providers/common/sql/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/sql/operators/sql.py (100%) rename {airflow => providers/src/airflow}/providers/common/sql/operators/sql.pyi (92%) rename {airflow => providers/src/airflow}/providers/common/sql/provider.yaml (100%) rename {airflow/providers/databricks/plugins => providers/src/airflow/providers/common/sql/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/common/sql/sensors/sql.py (100%) rename {airflow => providers/src/airflow}/providers/common/sql/sensors/sql.pyi (91%) rename {airflow => providers/src/airflow}/providers/databricks/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/databricks/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/databricks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/exceptions.py (100%) rename {airflow/providers/cncf/kubernetes/operators => providers/src/airflow/providers/databricks/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/hooks/databricks.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/hooks/databricks_base.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/hooks/databricks_sql.py (100%) rename {airflow/providers/databricks/hooks => providers/src/airflow/providers/databricks/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/operators/databricks.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/operators/databricks_repos.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/operators/databricks_sql.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/operators/databricks_workflow.py (100%) rename {airflow/providers/databricks/sensors => providers/src/airflow/providers/databricks/plugins}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/plugins/databricks_workflow.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/provider.yaml (100%) rename {airflow/providers/databricks/utils => providers/src/airflow/providers/databricks/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/sensors/databricks_partition.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/sensors/databricks_sql.py (100%) rename {airflow/providers/databricks/operators => providers/src/airflow/providers/databricks/triggers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/triggers/databricks.py (100%) rename {airflow/providers/dbt => providers/src/airflow/providers/databricks/utils}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/databricks/utils/databricks.py (100%) rename {airflow => providers/src/airflow}/providers/datadog/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/datadog/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/datadog/__init__.py (100%) rename {airflow/providers/databricks/triggers => providers/src/airflow/providers/datadog/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/datadog/hooks/datadog.py (100%) rename {airflow => providers/src/airflow}/providers/datadog/provider.yaml (100%) rename {airflow/providers/datadog/hooks => providers/src/airflow/providers/datadog/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/datadog/sensors/datadog.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/hooks/dbt.py (99%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/operators/dbt.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/provider.yaml (100%) rename {airflow/providers/datadog => providers/src/airflow/providers/dbt/cloud}/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/sensors/dbt.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/triggers/dbt.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dbt/cloud/utils/openlineage.py (100%) rename {airflow => providers/src/airflow}/providers/dingding/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/dingding/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/dingding/__init__.py (100%) rename {airflow/providers/dbt/cloud/sensors => providers/src/airflow/providers/dingding/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dingding/hooks/dingding.py (100%) rename {airflow/providers/dingding/hooks => providers/src/airflow/providers/dingding/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/dingding/operators/dingding.py (100%) rename {airflow => providers/src/airflow}/providers/dingding/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/discord/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/discord/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/discord/__init__.py (100%) rename {airflow/providers/dingding/operators => providers/src/airflow/providers/discord/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/discord/hooks/discord_webhook.py (100%) rename {airflow/providers/discord/hooks => providers/src/airflow/providers/discord/notifications}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/discord/notifications/discord.py (100%) rename {airflow/providers/discord/notifications => providers/src/airflow/providers/discord/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/discord/operators/discord_webhook.py (100%) rename {airflow => providers/src/airflow}/providers/discord/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/docker/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/docker/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/docker/__init__.py (100%) rename {airflow/providers/discord/operators => providers/src/airflow/providers/docker/decorators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/docker/decorators/docker.py (100%) rename {airflow => providers/src/airflow}/providers/docker/exceptions.py (100%) rename {airflow/providers/docker/decorators => providers/src/airflow/providers/docker/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/docker/hooks/docker.py (100%) rename {airflow/providers/docker/hooks => providers/src/airflow/providers/docker/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/docker/operators/docker.py (100%) rename {airflow => providers/src/airflow}/providers/docker/operators/docker_swarm.py (100%) rename {airflow => providers/src/airflow}/providers/docker/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/edge/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/edge/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/edge/api_endpoints/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/edge/api_endpoints/health_endpoint.py (100%) rename {airflow => providers/src/airflow}/providers/edge/api_endpoints/rpc_api_endpoint.py (100%) rename {airflow => providers/src/airflow}/providers/edge/cli/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/edge/cli/edge_command.py (100%) rename {airflow => providers/src/airflow}/providers/edge/example_dags/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/edge/example_dags/integration_test.py (100%) rename {airflow => providers/src/airflow}/providers/edge/models/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/edge/models/edge_job.py (100%) rename {airflow => providers/src/airflow}/providers/edge/models/edge_logs.py (100%) rename {airflow => providers/src/airflow}/providers/edge/models/edge_worker.py (100%) rename {airflow => providers/src/airflow}/providers/edge/openapi/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/edge/openapi/edge_worker_api_v1.yaml (100%) rename {airflow => providers/src/airflow}/providers/edge/plugins/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/edge/plugins/edge_executor_plugin.py (100%) rename {airflow => providers/src/airflow}/providers/edge/plugins/templates/edge_worker_hosts.html (100%) rename {airflow => providers/src/airflow}/providers/edge/plugins/templates/edge_worker_jobs.html (100%) rename {airflow => providers/src/airflow}/providers/edge/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/__init__.py (100%) rename {airflow/providers/docker/operators => providers/src/airflow/providers/elasticsearch/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/hooks/elasticsearch.py (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/log/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/log/es_json_formatter.py (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/log/es_response.py (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/log/es_task_handler.py (100%) rename {airflow => providers/src/airflow}/providers/elasticsearch/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/exasol/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/exasol/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/exasol/__init__.py (100%) rename {airflow/providers/elasticsearch => providers/src/airflow/providers/exasol}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/exasol/hooks/exasol.py (100%) rename {airflow/providers/exasol/hooks => providers/src/airflow/providers/exasol/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/exasol/operators/exasol.py (100%) rename {airflow => providers/src/airflow}/providers/exasol/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/fab/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/fab/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/alembic.ini (100%) rename {airflow/providers/exasol/operators => providers/src/airflow/providers/fab/auth_manager}/__init__.py (100%) rename {airflow/providers/fab/auth_manager => providers/src/airflow/providers/fab/auth_manager/api}/__init__.py (100%) rename {airflow/providers/fab/auth_manager/api => providers/src/airflow/providers/fab/auth_manager/api/auth}/__init__.py (100%) rename {airflow/providers/fab/auth_manager/api/auth => providers/src/airflow/providers/fab/auth_manager/api/auth/backend}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/api/auth/backend/basic_auth.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/api/auth/backend/kerberos_auth.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/api_endpoints/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/api_endpoints/user_endpoint.py (100%) rename {airflow/providers/fab/auth_manager/api/auth/backend => providers/src/airflow/providers/fab/auth_manager/cli_commands}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/cli_commands/db_command.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/cli_commands/definition.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/cli_commands/role_command.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/cli_commands/sync_perm_command.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/cli_commands/user_command.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/cli_commands/utils.py (100%) rename {airflow/providers/fab/auth_manager/cli_commands => providers/src/airflow/providers/fab/auth_manager/decorators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/decorators/auth.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/fab_auth_manager.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/models/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/models/anonymous_user.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/models/db.py (94%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/openapi/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/openapi/v1.yaml (100%) rename {airflow/providers/fab/auth_manager/decorators => providers/src/airflow/providers/fab/auth_manager/security_manager}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/security_manager/constants.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/security_manager/override.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/views/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/views/permissions.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/views/roles_list.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/views/user.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/views/user_edit.py (100%) rename {airflow => providers/src/airflow}/providers/fab/auth_manager/views/user_stats.py (100%) rename {airflow => providers/src/airflow}/providers/fab/migrations/README (100%) rename {airflow => providers/src/airflow}/providers/fab/migrations/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/migrations/env.py (100%) rename {airflow => providers/src/airflow}/providers/fab/migrations/script.py.mako (100%) rename {airflow => providers/src/airflow}/providers/fab/migrations/versions/0001_1_4_0_placeholder_migration.py (100%) rename {airflow => providers/src/airflow}/providers/fab/migrations/versions/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/fab/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/facebook/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/facebook/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/facebook/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/facebook/ads/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/facebook/ads/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/facebook/ads/hooks/ads.py (100%) rename {airflow => providers/src/airflow}/providers/facebook/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/ftp/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/ftp/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/ftp/__init__.py (100%) rename {airflow/providers/fab/auth_manager/security_manager => providers/src/airflow/providers/ftp/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ftp/hooks/ftp.py (100%) rename {airflow => providers/src/airflow}/providers/ftp/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ftp/operators/ftp.py (100%) rename {airflow => providers/src/airflow}/providers/ftp/provider.yaml (100%) rename {airflow/providers/ftp/hooks => providers/src/airflow/providers/ftp/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ftp/sensors/ftp.py (100%) rename {airflow => providers/src/airflow}/providers/github/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/github/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/github/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/github/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/github/hooks/github.py (100%) rename {airflow => providers/src/airflow}/providers/github/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/github/operators/github.py (100%) rename {airflow => providers/src/airflow}/providers/github/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/github/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/github/sensors/github.py (100%) rename {airflow => providers/src/airflow}/providers/google/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/google/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/google/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/ads/.gitignore (100%) rename {airflow => providers/src/airflow}/providers/google/ads/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/ads/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/ads/hooks/ads.py (100%) rename {airflow => providers/src/airflow}/providers/google/ads/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/ads/operators/ads.py (100%) rename {airflow => providers/src/airflow}/providers/google/ads/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/ads/transfers/ads_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/_internal_client/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/_internal_client/secret_manager_client.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/example_dags/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/example_dags/example_cloud_task.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/example_dags/example_looker.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/example_dags/example_presto_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/example_dags/example_salesforce_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/fs/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/fs/gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/automl.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/bigquery_dts.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/bigtable.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/cloud_batch.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/cloud_build.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/cloud_composer.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/cloud_memorystore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/cloud_run.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/cloud_sql.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/cloud_storage_transfer_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/compute.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/compute_ssh.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/datacatalog.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/dataflow.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/dataform.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/datafusion.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/datapipeline.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/dataplex.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/dataprep.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/dataproc_metastore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/datastore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/dlp.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/functions.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/gdm.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/kms.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/kubernetes_engine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/life_sciences.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/looker.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/mlengine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/natural_language.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/os_login.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/pubsub.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/secret_manager.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/spanner.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/speech_to_text.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/stackdriver.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/tasks.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/text_to_speech.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/translate.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/auto_ml.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/custom_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/dataset.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/endpoint_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/generative_model.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/model_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/pipeline_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vertex_ai/prediction_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/video_intelligence.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/vision.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/hooks/workflows.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/automl.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/base.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/bigquery_dts.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/bigtable.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/cloud_build.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/cloud_functions.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/cloud_memorystore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/cloud_sql.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/cloud_storage_transfer.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/cloud_tasks.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/compute.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/data_loss_prevention.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/datacatalog.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/dataflow.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/dataform.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/datafusion.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/dataplex.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/dataprep.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/datastore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/kubernetes_engine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/life_sciences.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/mlengine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/pubsub.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/spanner.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/stackdriver.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/translate.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/vertex_ai.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/links/workflows.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/log/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/log/gcs_task_handler.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/log/stackdriver_task_handler.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/openlineage/BigQueryErrorRunFacet.json (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/openlineage/BigQueryJobRunFacet.json (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/openlineage/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/openlineage/mixins.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/openlineage/utils.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/automl.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/bigquery.py (99%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/bigquery_dts.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/bigtable.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_base.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_batch.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_build.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_composer.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_memorystore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_run.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_sql.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/cloud_storage_transfer_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/compute.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/datacatalog.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/dataflow.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/dataform.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/datafusion.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/datapipeline.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/dataplex.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/dataprep.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/dataproc_metastore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/datastore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/dlp.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/functions.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/kubernetes_engine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/life_sciences.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/looker.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/mlengine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/natural_language.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/pubsub.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/spanner.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/speech_to_text.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/stackdriver.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/tasks.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/text_to_speech.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/translate.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/translate_speech.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/auto_ml.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/custom_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/dataset.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/endpoint_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/generative_model.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/model_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vertex_ai/pipeline_job.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/video_intelligence.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/vision.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/operators/workflows.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/secrets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/secrets/secret_manager.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/bigquery_dts.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/bigtable.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/cloud_composer.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/cloud_storage_transfer_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/dataflow.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/dataform.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/datafusion.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/dataplex.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/dataprep.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/dataproc_metastore.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/looker.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/pubsub.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/tasks.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/sensors/workflows.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/adls_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/azure_blob_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/azure_fileshare_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/bigquery_to_bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/bigquery_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/bigquery_to_mssql.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/bigquery_to_mysql.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/bigquery_to_postgres.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/bigquery_to_sql.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/calendar_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/cassandra_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/facebook_ads_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/gcs_to_bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/gcs_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/gcs_to_local.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/gcs_to_sftp.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/gdrive_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/gdrive_to_local.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/local_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/mssql_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/mysql_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/oracle_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/postgres_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/presto_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/s3_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/salesforce_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/sftp_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/sheets_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/sql_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/transfers/trino_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/bigquery_dts.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/cloud_batch.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/cloud_build.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/cloud_composer.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/cloud_run.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/cloud_sql.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/cloud_storage_transfer_service.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/dataflow.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/datafusion.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/dataplex.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/gcs.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/kubernetes_engine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/mlengine.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/pubsub.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/triggers/vertex_ai.py (100%) rename {airflow/providers/ftp/sensors => providers/src/airflow/providers/google/cloud/utils}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/bigquery_get_data.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/credentials_provider.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/dataform.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/datafusion.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/external_token_supplier.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/field_sanitizer.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/field_validator.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/helpers.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/mlengine_operator_utils.py (100%) rename {airflow => providers/src/airflow}/providers/google/cloud/utils/mlengine_prediction_summary.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/auth_backend/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/auth_backend/google_openid.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/consts.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/deprecated.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/hooks/base_google.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/hooks/discovery_api.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/links/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/links/storage.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/common/utils/id_token_credentials.py (100%) rename {airflow => providers/src/airflow}/providers/google/datasets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/datasets/bigquery.py (100%) rename {airflow => providers/src/airflow}/providers/google/firebase/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/firebase/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/firebase/hooks/firestore.py (100%) rename {airflow => providers/src/airflow}/providers/google/firebase/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/firebase/operators/firestore.py (100%) rename {airflow => providers/src/airflow}/providers/google/go_module_utils.py (100%) rename {airflow => providers/src/airflow}/providers/google/leveldb/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/leveldb/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/leveldb/hooks/leveldb.py (100%) rename {airflow => providers/src/airflow}/providers/google/leveldb/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/leveldb/operators/leveldb.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/example_dags/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/example_dags/example_display_video.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/hooks/analytics.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/hooks/analytics_admin.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/hooks/campaign_manager.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/hooks/display_video.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/hooks/search_ads.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/links/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/links/analytics_admin.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/operators/analytics.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/operators/analytics_admin.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/operators/campaign_manager.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/operators/display_video.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/operators/search_ads.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/sensors/campaign_manager.py (100%) rename {airflow => providers/src/airflow}/providers/google/marketing_platform/sensors/display_video.py (100%) rename {airflow => providers/src/airflow}/providers/google/provider.yaml (100%) rename {airflow/providers/google/cloud/utils => providers/src/airflow/providers/google/suite}/__init__.py (100%) rename {airflow/providers/google/suite => providers/src/airflow/providers/google/suite/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/hooks/calendar.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/hooks/drive.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/hooks/sheets.py (100%) rename {airflow/providers/google/suite/hooks => providers/src/airflow/providers/google/suite/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/operators/sheets.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/sensors/drive.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/transfers/gcs_to_gdrive.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/transfers/gcs_to_sheets.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/transfers/local_to_drive.py (100%) rename {airflow => providers/src/airflow}/providers/google/suite/transfers/sql_to_sheets.py (100%) rename {airflow => providers/src/airflow}/providers/grpc/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/grpc/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/grpc/__init__.py (100%) rename {airflow/providers/google/suite/operators => providers/src/airflow/providers/grpc/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/grpc/hooks/grpc.py (100%) rename {airflow/providers/grpc/hooks => providers/src/airflow/providers/grpc/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/grpc/operators/grpc.py (100%) rename {airflow => providers/src/airflow}/providers/grpc/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/_internal_client/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/_internal_client/vault_client.py (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/hooks/vault.py (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/secrets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/hashicorp/secrets/vault.py (100%) rename {airflow => providers/src/airflow}/providers/http/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/http/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/http/__init__.py (100%) rename {airflow/providers/grpc/operators => providers/src/airflow/providers/http/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/http/hooks/http.py (100%) rename {airflow/providers/http/hooks => providers/src/airflow/providers/http/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/http/operators/http.py (100%) rename {airflow => providers/src/airflow}/providers/http/provider.yaml (100%) rename {airflow/providers/http/operators => providers/src/airflow/providers/http/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/http/sensors/http.py (100%) rename {airflow/providers/http/sensors => providers/src/airflow/providers/http/triggers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/http/triggers/http.py (100%) rename {airflow => providers/src/airflow}/providers/imap/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/imap/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/imap/__init__.py (100%) rename {airflow/providers/http/triggers => providers/src/airflow/providers/imap/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/imap/hooks/imap.py (100%) rename {airflow => providers/src/airflow}/providers/imap/provider.yaml (100%) rename {airflow/providers/imap/hooks => providers/src/airflow/providers/imap/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/imap/sensors/imap_attachment.py (100%) rename {airflow => providers/src/airflow}/providers/influxdb/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/influxdb/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/influxdb/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/influxdb/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/influxdb/hooks/influxdb.py (100%) rename {airflow => providers/src/airflow}/providers/influxdb/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/influxdb/operators/influxdb.py (100%) rename {airflow => providers/src/airflow}/providers/influxdb/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/jdbc/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/jdbc/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/jdbc/__init__.py (100%) rename {airflow/providers/imap/sensors => providers/src/airflow/providers/jdbc/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/jdbc/hooks/jdbc.py (100%) rename {airflow/providers/jdbc/hooks => providers/src/airflow/providers/jdbc/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/jdbc/operators/jdbc.py (100%) rename {airflow => providers/src/airflow}/providers/jdbc/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/jenkins/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/jenkins/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/jenkins/__init__.py (100%) rename {airflow/providers/jdbc/operators => providers/src/airflow/providers/jenkins/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/jenkins/hooks/jenkins.py (100%) rename {airflow/providers/jenkins/hooks => providers/src/airflow/providers/jenkins/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/jenkins/operators/jenkins_job_trigger.py (100%) rename {airflow => providers/src/airflow}/providers/jenkins/provider.yaml (100%) rename {airflow/providers/jenkins/operators => providers/src/airflow/providers/jenkins/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/jenkins/sensors/jenkins.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/fs/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/fs/adls.py (100%) rename {airflow/providers/jenkins/sensors => providers/src/airflow/providers/microsoft/azure/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/adx.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/asb.py (96%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/base_azure.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/batch.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/container_instance.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/container_registry.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/container_volume.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/cosmos.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/data_factory.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/data_lake.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/fileshare.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/msgraph.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/powerbi.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/synapse.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/hooks/wasb.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/log/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/log/wasb_task_handler.py (100%) rename {airflow/providers/microsoft => providers/src/airflow/providers/microsoft/azure/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/adls.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/adx.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/asb.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/batch.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/container_instances.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/cosmos.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/data_factory.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/msgraph.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/powerbi.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/synapse.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/operators/wasb_delete_blob.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/secrets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/secrets/key_vault.py (100%) rename {airflow/providers/microsoft/azure/hooks => providers/src/airflow/providers/microsoft/azure/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/sensors/cosmos.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/sensors/data_factory.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/sensors/msgraph.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/sensors/wasb.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/transfers/azure_blob_to_gcs.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/transfers/local_to_adls.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/transfers/local_to_wasb.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/transfers/s3_to_wasb.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/transfers/sftp_to_wasb.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/triggers/data_factory.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/triggers/msgraph.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/triggers/powerbi.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/triggers/wasb.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/azure/utils.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/mssql/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/microsoft/mssql/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/microsoft/mssql/__init__.py (100%) rename {airflow/providers/microsoft/azure/operators => providers/src/airflow/providers/microsoft/mssql/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/mssql/hooks/mssql.py (100%) rename {airflow/providers/microsoft/azure/sensors => providers/src/airflow/providers/microsoft/mssql/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/mssql/operators/mssql.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/mssql/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/microsoft/psrp/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/microsoft/psrp/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/microsoft/psrp/__init__.py (100%) rename {airflow/providers/microsoft/mssql => providers/src/airflow/providers/microsoft/psrp}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/psrp/hooks/psrp.py (100%) rename {airflow/providers/microsoft/mssql => providers/src/airflow/providers/microsoft/psrp}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/psrp/operators/psrp.py (93%) rename {airflow => providers/src/airflow}/providers/microsoft/psrp/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/microsoft/winrm/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/microsoft/winrm/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/microsoft/winrm/__init__.py (100%) rename {airflow/providers/microsoft/psrp => providers/src/airflow/providers/microsoft/winrm}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/winrm/hooks/winrm.py (100%) rename {airflow/providers/microsoft/psrp => providers/src/airflow/providers/microsoft/winrm}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/winrm/operators/winrm.py (100%) rename {airflow => providers/src/airflow}/providers/microsoft/winrm/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/mongo/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/mongo/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/mongo/__init__.py (100%) rename {airflow/providers/microsoft/winrm => providers/src/airflow/providers/mongo}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/mongo/hooks/mongo.py (100%) rename {airflow => providers/src/airflow}/providers/mongo/provider.yaml (100%) rename {airflow/providers/microsoft/winrm/operators => providers/src/airflow/providers/mongo/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/mongo/sensors/mongo.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/mysql/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/mysql/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/assets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/assets/mysql.py (100%) rename {airflow/providers/mongo => providers/src/airflow/providers/mysql}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/hooks/mysql.py (100%) rename {airflow/providers/mongo/sensors => providers/src/airflow/providers/mysql/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/operators/mysql.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/mysql/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/transfers/presto_to_mysql.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/transfers/s3_to_mysql.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/transfers/trino_to_mysql.py (100%) rename {airflow => providers/src/airflow}/providers/mysql/transfers/vertica_to_mysql.py (100%) rename {airflow => providers/src/airflow}/providers/neo4j/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/neo4j/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/neo4j/README.md (100%) rename {airflow => providers/src/airflow}/providers/neo4j/__init__.py (100%) rename {airflow/providers/mysql => providers/src/airflow/providers/neo4j}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/neo4j/hooks/neo4j.py (100%) rename {airflow/providers/mysql => providers/src/airflow/providers/neo4j}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/neo4j/operators/neo4j.py (100%) rename {airflow => providers/src/airflow}/providers/neo4j/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/odbc/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/odbc/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/odbc/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/odbc/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/odbc/hooks/odbc.py (100%) rename {airflow => providers/src/airflow}/providers/odbc/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/openai/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/openai/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/openai/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openai/exceptions.py (100%) rename {airflow => providers/src/airflow}/providers/openai/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openai/hooks/openai.py (100%) rename {airflow => providers/src/airflow}/providers/openai/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openai/operators/openai.py (100%) rename {airflow => providers/src/airflow}/providers/openai/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/openai/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openai/triggers/openai.py (100%) rename {airflow => providers/src/airflow}/providers/openfaas/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/openfaas/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/openfaas/__init__.py (100%) rename {airflow/providers/neo4j => providers/src/airflow/providers/openfaas}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openfaas/hooks/openfaas.py (100%) rename {airflow => providers/src/airflow}/providers/openfaas/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/openlineage/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/openlineage/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/openlineage/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/conf.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/extractors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/extractors/base.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/extractors/bash.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/extractors/manager.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/extractors/python.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/facets/AirflowDagRunFacet.json (100%) rename {airflow => providers/src/airflow}/providers/openlineage/facets/AirflowDebugRunFacet.json (100%) rename {airflow => providers/src/airflow}/providers/openlineage/facets/AirflowJobFacet.json (100%) rename {airflow => providers/src/airflow}/providers/openlineage/facets/AirflowRunFacet.json (100%) rename {airflow => providers/src/airflow}/providers/openlineage/facets/AirflowStateRunFacet.json (100%) rename {airflow => providers/src/airflow}/providers/openlineage/facets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/plugins/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/plugins/adapter.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/plugins/facets.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/plugins/listener.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/plugins/macros.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/plugins/openlineage.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/openlineage/sqlparser.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/utils/asset_compat_lineage_collector.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/utils/selective_enable.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/utils/sql.py (100%) rename {airflow => providers/src/airflow}/providers/openlineage/utils/utils.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/opensearch/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/opensearch/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/hooks/opensearch.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/log/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/log/os_json_formatter.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/log/os_response.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/log/os_task_handler.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/operators/opensearch.py (100%) rename {airflow => providers/src/airflow}/providers/opensearch/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/__init__.py (100%) rename {airflow/providers/neo4j/operators => providers/src/airflow/providers/opsgenie/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/hooks/opsgenie.py (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/notifications/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/notifications/opsgenie.py (100%) rename {airflow/providers/openfaas/hooks => providers/src/airflow/providers/opsgenie/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/operators/opsgenie.py (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/typing/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/opsgenie/typing/opsgenie.py (100%) rename {airflow => providers/src/airflow}/providers/oracle/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/oracle/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/oracle/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/oracle/example_dags/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/oracle/example_dags/example_oracle.py (100%) rename {airflow/providers/opsgenie => providers/src/airflow/providers/oracle}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/oracle/hooks/oracle.py (100%) rename {airflow/providers/opsgenie => providers/src/airflow/providers/oracle}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/oracle/operators/oracle.py (100%) rename {airflow => providers/src/airflow}/providers/oracle/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/oracle/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/oracle/transfers/oracle_to_oracle.py (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/__init__.py (100%) rename {airflow/providers/oracle => providers/src/airflow/providers/pagerduty}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/hooks/pagerduty.py (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/hooks/pagerduty_events.py (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/notifications/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/notifications/pagerduty.py (100%) rename {airflow => providers/src/airflow}/providers/pagerduty/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/papermill/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/papermill/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/papermill/__init__.py (100%) rename {airflow/providers/oracle/operators => providers/src/airflow/providers/papermill/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/papermill/hooks/kernel.py (100%) rename {airflow/providers/pagerduty/hooks => providers/src/airflow/providers/papermill/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/papermill/operators/papermill.py (100%) rename {airflow => providers/src/airflow}/providers/papermill/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/pgvector/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/pgvector/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/pgvector/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pgvector/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pgvector/hooks/pgvector.py (100%) rename {airflow => providers/src/airflow}/providers/pgvector/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pgvector/operators/pgvector.py (100%) rename {airflow => providers/src/airflow}/providers/pgvector/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/pinecone/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/pinecone/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/pinecone/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pinecone/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pinecone/hooks/pinecone.py (100%) rename {airflow => providers/src/airflow}/providers/pinecone/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/pinecone/operators/pinecone.py (100%) rename {airflow => providers/src/airflow}/providers/pinecone/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/postgres/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/postgres/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/postgres/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/postgres/assets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/postgres/assets/postgres.py (100%) rename {airflow/providers/papermill => providers/src/airflow/providers/postgres}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/postgres/hooks/postgres.py (100%) rename {airflow/providers/papermill => providers/src/airflow/providers/postgres}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/postgres/operators/postgres.py (100%) rename {airflow => providers/src/airflow}/providers/postgres/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/presto/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/presto/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/presto/__init__.py (100%) rename {airflow/providers/postgres => providers/src/airflow/providers/presto}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/presto/hooks/presto.py (100%) rename {airflow => providers/src/airflow}/providers/presto/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/presto/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/presto/transfers/gcs_to_presto.py (100%) rename {airflow => providers/src/airflow}/providers/qdrant/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/qdrant/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/qdrant/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/qdrant/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/qdrant/hooks/qdrant.py (100%) rename {airflow => providers/src/airflow}/providers/qdrant/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/qdrant/operators/qdrant.py (100%) rename {airflow => providers/src/airflow}/providers/qdrant/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/redis/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/redis/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/redis/__init__.py (100%) rename {airflow/providers/postgres/operators => providers/src/airflow/providers/redis/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/redis/hooks/redis.py (100%) rename {airflow/providers/presto/hooks => providers/src/airflow/providers/redis/log}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/redis/log/redis_task_handler.py (100%) rename {airflow/providers/redis/hooks => providers/src/airflow/providers/redis/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/redis/operators/redis_publish.py (100%) rename {airflow => providers/src/airflow}/providers/redis/provider.yaml (100%) rename {airflow/providers/redis/log => providers/src/airflow/providers/redis/sensors}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/redis/sensors/redis_key.py (100%) rename {airflow => providers/src/airflow}/providers/redis/sensors/redis_pub_sub.py (100%) rename {airflow => providers/src/airflow}/providers/salesforce/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/salesforce/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/salesforce/__init__.py (100%) rename {airflow/providers/redis/operators => providers/src/airflow/providers/salesforce/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/salesforce/hooks/salesforce.py (100%) rename {airflow => providers/src/airflow}/providers/salesforce/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/salesforce/operators/bulk.py (100%) rename {airflow => providers/src/airflow}/providers/salesforce/operators/salesforce_apex_rest.py (100%) rename {airflow => providers/src/airflow}/providers/salesforce/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/samba/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/samba/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/samba/__init__.py (100%) rename {airflow/providers/redis/sensors => providers/src/airflow/providers/samba/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/samba/hooks/samba.py (100%) rename {airflow => providers/src/airflow}/providers/samba/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/samba/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/samba/transfers/gcs_to_samba.py (100%) rename {airflow => providers/src/airflow}/providers/segment/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/segment/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/segment/__init__.py (100%) rename {airflow/providers/salesforce => providers/src/airflow/providers/segment}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/segment/hooks/segment.py (100%) rename {airflow/providers/samba/hooks => providers/src/airflow/providers/segment/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/segment/operators/segment_track_event.py (100%) rename {airflow => providers/src/airflow}/providers/segment/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/sendgrid/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/sendgrid/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/sendgrid/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sendgrid/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/sendgrid/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sendgrid/utils/emailer.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/sftp/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/sftp/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/decorators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/decorators/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/decorators/sensors/sftp.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/hooks/sftp.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/operators/sftp.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/sftp/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/sensors/sftp.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sftp/triggers/sftp.py (100%) rename {airflow => providers/src/airflow}/providers/singularity/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/singularity/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/singularity/__init__.py (100%) rename {airflow/providers/segment/hooks => providers/src/airflow/providers/singularity/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/singularity/operators/singularity.py (100%) rename {airflow => providers/src/airflow}/providers/singularity/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/slack/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/slack/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/slack/__init__.py (100%) rename {airflow/providers/segment/operators => providers/src/airflow/providers/slack/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/slack/hooks/slack.py (100%) rename {airflow => providers/src/airflow}/providers/slack/hooks/slack_webhook.py (100%) rename {airflow => providers/src/airflow}/providers/slack/notifications/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/slack/notifications/slack.py (100%) rename {airflow => providers/src/airflow}/providers/slack/notifications/slack_notifier.py (100%) rename {airflow => providers/src/airflow}/providers/slack/notifications/slack_webhook.py (100%) rename {airflow/providers/singularity => providers/src/airflow/providers/slack}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/slack/operators/slack.py (100%) rename {airflow => providers/src/airflow}/providers/slack/operators/slack_webhook.py (100%) rename {airflow => providers/src/airflow}/providers/slack/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/slack/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/slack/transfers/base_sql_to_slack.py (100%) rename {airflow => providers/src/airflow}/providers/slack/transfers/sql_to_slack.py (100%) rename {airflow => providers/src/airflow}/providers/slack/transfers/sql_to_slack_webhook.py (100%) rename {airflow => providers/src/airflow}/providers/slack/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/smtp/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/smtp/__init__.py (100%) rename {airflow/providers/slack => providers/src/airflow/providers/smtp}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/hooks/smtp.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/notifications/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/notifications/smtp.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/notifications/templates/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/notifications/templates/email.html (100%) rename {airflow => providers/src/airflow}/providers/smtp/notifications/templates/email_subject.jinja2 (100%) rename {airflow/providers/slack => providers/src/airflow/providers/smtp}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/operators/smtp.py (100%) rename {airflow => providers/src/airflow}/providers/smtp/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/snowflake/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/snowflake/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/snowflake/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/decorators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/decorators/snowpark.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/hooks/snowflake.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/hooks/snowflake_sql_api.py (99%) rename {airflow => providers/src/airflow}/providers/snowflake/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/operators/snowflake.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/operators/snowpark.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/snowflake/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/transfers/copy_into_snowflake.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/triggers/snowflake_trigger.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/utils/common.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/utils/openlineage.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/utils/snowpark.py (100%) rename {airflow => providers/src/airflow}/providers/snowflake/utils/sql_api_generate_jwt.py (100%) rename {airflow => providers/src/airflow}/providers/sqlite/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/sqlite/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/sqlite/__init__.py (100%) rename {airflow/providers/smtp => providers/src/airflow/providers/sqlite}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sqlite/hooks/sqlite.py (100%) rename {airflow/providers/smtp => providers/src/airflow/providers/sqlite}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/sqlite/operators/sqlite.py (100%) rename {airflow => providers/src/airflow}/providers/sqlite/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/ssh/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/ssh/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/ssh/__init__.py (100%) rename {airflow/providers/sqlite => providers/src/airflow/providers/ssh}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ssh/hooks/ssh.py (100%) rename {airflow/providers/sqlite => providers/src/airflow/providers/ssh}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ssh/operators/ssh.py (100%) rename {airflow => providers/src/airflow}/providers/ssh/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/standard/CHANGELOG.rst (100%) rename {airflow/providers/ssh/hooks => providers/src/airflow/providers/standard}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/standard/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/standard/operators/bash.py (100%) rename {airflow => providers/src/airflow}/providers/standard/operators/datetime.py (100%) rename {airflow => providers/src/airflow}/providers/standard/operators/weekday.py (100%) rename {airflow => providers/src/airflow}/providers/standard/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/standard/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/standard/sensors/bash.py (100%) rename {airflow => providers/src/airflow}/providers/standard/sensors/date_time.py (100%) rename {airflow => providers/src/airflow}/providers/standard/sensors/time.py (100%) rename {airflow => providers/src/airflow}/providers/standard/sensors/time_delta.py (100%) rename {airflow => providers/src/airflow}/providers/standard/sensors/weekday.py (100%) rename {airflow => providers/src/airflow}/providers/tableau/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/tableau/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/tableau/__init__.py (100%) rename {airflow/providers/ssh/operators => providers/src/airflow/providers/tableau/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/tableau/hooks/tableau.py (100%) rename {airflow => providers/src/airflow}/providers/tableau/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/tableau/operators/tableau.py (100%) rename {airflow => providers/src/airflow}/providers/tableau/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/tableau/sensors/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/tableau/sensors/tableau.py (100%) rename {airflow => providers/src/airflow}/providers/telegram/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/telegram/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/telegram/__init__.py (100%) rename {airflow/providers/standard => providers/src/airflow/providers/telegram/hooks}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/telegram/hooks/telegram.py (100%) rename {airflow/providers/tableau/hooks => providers/src/airflow/providers/telegram/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/telegram/operators/telegram.py (100%) rename {airflow => providers/src/airflow}/providers/telegram/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/teradata/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/teradata/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/teradata/__init__.py (100%) rename {airflow/providers/telegram => providers/src/airflow/providers/teradata}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/hooks/teradata.py (100%) rename {airflow/providers/telegram => providers/src/airflow/providers/teradata}/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/operators/teradata.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/operators/teradata_compute_cluster.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/teradata/transfers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/transfers/azure_blob_to_teradata.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/transfers/s3_to_teradata.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/transfers/teradata_to_teradata.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/triggers/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/triggers/teradata_compute_cluster.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/teradata/utils/constants.py (100%) rename {airflow => providers/src/airflow}/providers/trino/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/trino/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/trino/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/trino/assets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/trino/assets/trino.py (100%) rename {airflow/providers/teradata => providers/src/airflow/providers/trino}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/trino/hooks/trino.py (100%) rename {airflow => providers/src/airflow}/providers/trino/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/trino/operators/trino.py (100%) rename {airflow => providers/src/airflow}/providers/trino/provider.yaml (100%) rename {airflow/providers/teradata/operators => providers/src/airflow/providers/trino/transfers}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/trino/transfers/gcs_to_trino.py (100%) rename {airflow => providers/src/airflow}/providers/vertica/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/vertica/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/vertica/__init__.py (100%) rename {airflow/providers/trino => providers/src/airflow/providers/vertica}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/vertica/hooks/vertica.py (100%) rename {airflow/providers/trino/transfers => providers/src/airflow/providers/vertica/operators}/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/vertica/operators/vertica.py (100%) rename {airflow => providers/src/airflow}/providers/vertica/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/weaviate/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/weaviate/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/weaviate/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/weaviate/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/weaviate/hooks/weaviate.py (100%) rename {airflow => providers/src/airflow}/providers/weaviate/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/weaviate/operators/weaviate.py (100%) rename {airflow => providers/src/airflow}/providers/weaviate/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/yandex/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/yandex/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/yandex/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/hooks/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/hooks/yandex.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/hooks/yandexcloud_dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/hooks/yq.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/links/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/links/yq.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/operators/dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/operators/yandexcloud_dataproc.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/operators/yq.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/yandex/secrets/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/secrets/lockbox.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/utils/credentials.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/utils/defaults.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/utils/fields.py (100%) rename {airflow => providers/src/airflow}/providers/yandex/utils/user_agent.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/ydb/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/_vendor/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/_vendor/dbapi/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/_vendor/dbapi/connection.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/_vendor/dbapi/constants.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/_vendor/dbapi/cursor.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/_vendor/dbapi/errors.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/_vendor/readme.md (100%) rename {airflow => providers/src/airflow}/providers/ydb/hooks/ydb.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/operators/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/operators/ydb.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/provider.yaml (100%) rename {airflow => providers/src/airflow}/providers/ydb/utils/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/utils/credentials.py (100%) rename {airflow => providers/src/airflow}/providers/ydb/utils/defaults.py (100%) rename {airflow => providers/src/airflow}/providers/zendesk/.latest-doc-only-change.txt (100%) rename {airflow => providers/src/airflow}/providers/zendesk/CHANGELOG.rst (100%) rename {airflow => providers/src/airflow}/providers/zendesk/__init__.py (100%) rename {airflow/providers/vertica => providers/src/airflow/providers/zendesk}/hooks/__init__.py (100%) rename {airflow => providers/src/airflow}/providers/zendesk/hooks/zendesk.py (100%) rename {airflow => providers/src/airflow}/providers/zendesk/provider.yaml (100%) rename {tests/_internals => providers/tests}/__init__.py (100%) rename {tests/integration/providers => providers/tests/airbyte}/__init__.py (100%) rename {tests/integration/providers/apache => providers/tests/airbyte/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/airbyte/hooks/test_airbyte.py (100%) rename {tests/integration/providers/apache/cassandra => providers/tests/airbyte/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/airbyte/operators/test_airbyte.py (100%) rename {tests/integration/providers/apache/cassandra/hooks => providers/tests/airbyte/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/airbyte/sensors/test_airbyte.py (100%) rename {tests/integration/providers/apache/drill => providers/tests/airbyte/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/airbyte/triggers/test_airbyte.py (100%) rename {tests/integration/providers/apache/drill/hooks => providers/tests/alibaba}/__init__.py (100%) rename {tests/integration/providers/apache/drill/operators => providers/tests/alibaba/cloud}/__init__.py (100%) rename {tests/integration/providers/apache/hive => providers/tests/alibaba/cloud/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/hooks/test_analyticdb_spark.py (99%) rename {tests/providers => providers/tests}/alibaba/cloud/hooks/test_oss.py (99%) rename {tests/integration/providers/apache/hive/transfers => providers/tests/alibaba/cloud/log}/__init__.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/log/test_oss_task_handler.py (98%) rename {tests/integration/providers/apache/kafka => providers/tests/alibaba/cloud/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/operators/test_analyticdb_spark.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/operators/test_oss.py (100%) rename {tests/integration/providers/apache/kafka/hooks => providers/tests/alibaba/cloud/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/sensors/test_analyticdb_spark.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/sensors/test_oss_key.py (100%) rename {tests/integration/providers/apache/kafka/operators => providers/tests/alibaba/cloud/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/utils/analyticdb_spark_mock.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/utils/oss_mock.py (100%) rename {tests/providers => providers/tests}/alibaba/cloud/utils/test_utils.py (100%) rename {tests/integration/providers/apache/kafka/sensors => providers/tests/amazon}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/.gitignore (100%) rename {tests/integration/providers/apache/kafka/triggers => providers/tests/amazon/aws}/__init__.py (100%) rename {tests/integration/providers/apache/pinot => providers/tests/amazon/aws/assets}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/assets/test_s3.py (100%) rename {tests/integration/providers/apache/pinot/hooks => providers/tests/amazon/aws/auth_manager}/__init__.py (100%) rename {tests/integration/providers/google => providers/tests/amazon/aws/auth_manager/avp}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/avp/test_entities.py (100%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/avp/test_facade.py (98%) rename {tests/integration/providers/google/cloud => providers/tests/amazon/aws/auth_manager/cli}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/cli/test_avp_commands.py (97%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/cli/test_definition.py (100%) rename {tests/integration/providers/google/cloud/transfers => providers/tests/amazon/aws/auth_manager/security_manager}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py (96%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/test_aws_auth_manager.py (97%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/test_constants.py (100%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/test_user.py (100%) rename {tests/integration/providers/microsoft => providers/tests/amazon/aws/auth_manager/views}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/auth_manager/views/test_auth.py (80%) rename {tests/integration/providers/microsoft/mssql => providers/tests/amazon/aws/config_templates}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/config_templates/args.json (100%) rename {tests/providers => providers/tests}/amazon/aws/config_templates/job.j2.json (100%) rename {tests/providers => providers/tests}/amazon/aws/config_templates/steps.j2.json (100%) rename {tests/providers => providers/tests}/amazon/aws/config_templates/steps.json (100%) rename {tests/integration/providers/microsoft/mssql/hooks => providers/tests/amazon/aws/deferrable}/__init__.py (100%) rename {tests/integration/providers/mongo => providers/tests/amazon/aws/deferrable/hooks}/__init__.py (100%) rename {tests/integration/providers/mongo/sensors => providers/tests/amazon/aws/executors}/__init__.py (100%) rename {tests/integration/providers/openlineage => providers/tests/amazon/aws/executors/batch}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/executors/batch/test_batch_executor.py (99%) rename {tests/integration/providers/openlineage/operators => providers/tests/amazon/aws/executors/ecs}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/executors/ecs/test_ecs_executor.py (99%) rename {tests/integration/providers/qdrant => providers/tests/amazon/aws/executors/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/executors/utils/test_exponential_backoff_retry.py (100%) rename {tests/integration/providers/qdrant/hooks => providers/tests/amazon/aws/fs}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/fs/test_s3.py (100%) rename {tests/integration/providers/qdrant/operators => providers/tests/amazon/aws/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_appflow.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_athena.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_athena_sql.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_base_aws.py (98%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_batch_client.py (98%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_batch_waiters.py (97%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_bedrock.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_chime.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_cloud_formation.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_comprehend.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_datasync.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_dms.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_dynamodb.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_ec2.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_ecr.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_ecs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_eks.py (99%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_elasticache_replication_group.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_emr.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_emr_containers.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_emr_serverless.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_eventbridge.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_glacier.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_glue.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_glue_catalog.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_glue_crawler.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_glue_databrew.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_hooks_signature.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_kinesis.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_kinesis_analytics.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_lambda_function.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_logs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_neptune.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_opensearch_serverless.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_quicksight.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_rds.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_redshift_cluster.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_redshift_data.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_redshift_sql.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_s3.py (99%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_sagemaker.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_secrets_manager.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_ses.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_sns.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_sqs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_ssm.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_step_function.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_sts.py (100%) rename {tests/providers => providers/tests}/amazon/aws/hooks/test_verified_permissions.py (100%) rename {tests/integration/providers/redis => providers/tests/amazon/aws/infrastructure}/__init__.py (100%) rename {tests/integration/providers/redis/hooks => providers/tests/amazon/aws/infrastructure/example_s3_to_redshift}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/infrastructure/example_s3_to_redshift/outputs.tf (100%) rename {tests/providers => providers/tests}/amazon/aws/infrastructure/example_s3_to_redshift/resources.tf (100%) rename {tests/providers => providers/tests}/amazon/aws/infrastructure/example_s3_to_redshift/variables.tf (100%) rename {tests/integration/providers/redis/operators => providers/tests/amazon/aws/links}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/links/test_athena.py (95%) rename {tests/providers => providers/tests}/amazon/aws/links/test_base_aws.py (98%) rename {tests/providers => providers/tests}/amazon/aws/links/test_batch.py (97%) rename {tests/providers => providers/tests}/amazon/aws/links/test_emr.py (99%) rename {tests/providers => providers/tests}/amazon/aws/links/test_glue.py (95%) rename {tests/providers => providers/tests}/amazon/aws/links/test_logs.py (96%) rename {tests/providers => providers/tests}/amazon/aws/links/test_step_function.py (97%) rename {tests/integration/providers/redis/sensors => providers/tests/amazon/aws/log}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/log/test_cloudwatch_task_handler.py (99%) rename {tests/providers => providers/tests}/amazon/aws/log/test_s3_task_handler.py (99%) rename {tests/integration/providers/trino => providers/tests/amazon/aws/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/notifications/test_chime.py (100%) rename {tests/providers => providers/tests}/amazon/aws/notifications/test_sns.py (100%) rename {tests/providers => providers/tests}/amazon/aws/notifications/test_sqs.py (100%) rename {tests/integration/providers/trino/hooks => providers/tests/amazon/aws/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/operators/athena_metadata.json (100%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_appflow.py (100%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_athena.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_base_aws.py (100%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_batch.py (100%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_bedrock.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_cloud_formation.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_comprehend.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_datasync.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_dms.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_ec2.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_ecs.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_eks.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_emr_add_steps.py (97%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_emr_containers.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_emr_create_job_flow.py (95%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_emr_modify_cluster.py (97%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_emr_notebook_execution.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_emr_serverless.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_emr_terminate_job_flow.py (97%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_eventbridge.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_glacier.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_glue.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_glue_crawler.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_glue_databrew.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_kinesis_analytics.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_lambda_function.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_neptune.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_quicksight.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_rds.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_redshift_cluster.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_redshift_data.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_redshift_sql.py (100%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_s3.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_base.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_endpoint.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_endpoint_config.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_model.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_notebook.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_pipeline.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_processing.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_training.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_transform.py (99%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sagemaker_tuning.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sns.py (97%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_sqs.py (98%) rename {tests/providers => providers/tests}/amazon/aws/operators/test_step_function.py (95%) rename {tests/integration/providers/ydb => providers/tests/amazon/aws/secrets}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/secrets/test_secrets_manager.py (100%) rename {tests/providers => providers/tests}/amazon/aws/secrets/test_systems_manager.py (99%) rename {tests/integration/providers/ydb/hooks => providers/tests/amazon/aws/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_athena.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_base_aws.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_batch.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_bedrock.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_cloud_formation.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_comprehend.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_dms.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_dynamodb.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_ec2.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_ecs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_eks.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_emr_base.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_emr_containers.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_emr_job_flow.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_emr_notebook_execution.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_emr_serverless_application.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_emr_serverless_job.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_emr_step.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_glacier.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_glue.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_glue_catalog_partition.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_glue_crawler.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_glue_data_quality.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_kinesis_analytics.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_lambda_function.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_opensearch_serverless.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_quicksight.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_rds.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_redshift_cluster.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sagemaker_automl.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sagemaker_base.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sagemaker_endpoint.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sagemaker_pipeline.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sagemaker_training.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sagemaker_transform.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sagemaker_tuning.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_sqs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/sensors/test_step_function.py (100%) rename {tests/integration/providers/ydb/operators => providers/tests/amazon/aws/system}/__init__.py (100%) rename {tests/providers => providers/tests/amazon/aws/system/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/system/utils/test_helpers.py (96%) rename {tests/providers/airbyte => providers/tests/amazon/aws/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_azure_blob_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_base.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_dynamodb_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_exasol_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_ftp_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_gcs_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_glacier_to_gcs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_google_api_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_hive_to_dynamodb.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_http_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_imap_attachment_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_local_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_mongo_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_redshift_to_s3.py (99%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_s3_to_dynamodb.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_s3_to_ftp.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_s3_to_redshift.py (99%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_s3_to_sftp.py (98%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_s3_to_sql.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_salesforce_to_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_sftp_to_s3.py (98%) rename {tests/providers => providers/tests}/amazon/aws/transfers/test_sql_to_s3.py (100%) rename {tests/providers/airbyte/hooks => providers/tests/amazon/aws/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_athena.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_base.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_batch.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_bedrock.py (99%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_comprehend.py (98%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_ec2.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_ecs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_eks.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_emr.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_glue.py (99%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_glue_crawler.py (97%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_glue_databrew.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_kinesis_analytics.py (98%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_lambda_function.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_neptune.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_opensearch_serverless.py (98%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_rds.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_redshift_cluster.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_redshift_data.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_s3.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_sagemaker.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_serialization.py (100%) rename {tests/providers => providers/tests}/amazon/aws/triggers/test_sqs.py (100%) rename {tests/providers/airbyte/operators => providers/tests/amazon/aws/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/eks_test_constants.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/eks_test_utils.py (99%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_connection_wrapper.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_eks_get_token.py (97%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_emailer.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_identifiers.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_mixins.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_openlineage.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_redshift.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_sqs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_suppress.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_tags.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_task_log_fetcher.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_template_fields.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_utils.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_waiter.py (100%) rename {tests/providers => providers/tests}/amazon/aws/utils/test_waiter_with_logging.py (100%) rename {tests/providers/airbyte/sensors => providers/tests/amazon/aws/waiters}/__init__.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test.json (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_batch.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_bedrock.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_bedrock_agent.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_comprehend.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_custom_waiters.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_dynamo.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_ecs.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_eks.py (97%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_emr.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_glue.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_glue_databrew.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_kinesis_analytics.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_neptune.py (100%) rename {tests/providers => providers/tests}/amazon/aws/waiters/test_opensearch_serverless.py (100%) rename {tests/providers => providers/tests}/amazon/conftest.py (100%) rename {tests/providers/airbyte/triggers => providers/tests/apache}/__init__.py (100%) rename {tests/providers/alibaba => providers/tests/apache/beam}/__init__.py (100%) rename {tests/providers/alibaba/cloud => providers/tests/apache/beam/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/beam/hooks/test_beam.py (100%) rename {tests/providers/alibaba/cloud/hooks => providers/tests/apache/beam/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/beam/operators/test_beam.py (100%) rename {tests/providers/alibaba/cloud/log => providers/tests/apache/beam/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/beam/triggers/test_beam.py (100%) rename {tests/providers/alibaba/cloud/operators => providers/tests/apache/cassandra}/__init__.py (100%) rename {tests/providers/alibaba/cloud => providers/tests/apache/cassandra}/sensors/__init__.py (100%) rename {tests/providers => providers/tests}/apache/cassandra/sensors/test_record.py (100%) rename {tests/providers => providers/tests}/apache/cassandra/sensors/test_table.py (100%) rename {airflow/providers/vertica/operators => providers/tests/apache/drill}/__init__.py (100%) rename {airflow/providers/zendesk => providers/tests/apache/drill}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/apache/drill/hooks/test_drill.py (94%) rename {tests/providers/apache/drill => providers/tests/apache/druid}/__init__.py (100%) rename {tests/providers/apache/drill => providers/tests/apache/druid}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/apache/druid/hooks/test_druid.py (100%) rename {tests/providers/apache/druid => providers/tests/apache/druid/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/druid/operators/test_druid.py (100%) rename {tests/providers/alibaba/cloud/utils => providers/tests/apache/druid/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/druid/transfers/test_hive_to_druid.py (100%) rename {tests/providers/apache/druid/hooks => providers/tests/apache/flink}/__init__.py (100%) rename {tests/providers/apache/druid => providers/tests/apache/flink}/operators/__init__.py (100%) rename {tests/providers => providers/tests}/apache/flink/operators/test_flink_kubernetes.py (100%) rename {tests/providers/apache/flink => providers/tests/apache/flink/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/flink/sensors/test_flink_kubernetes.py (100%) rename {tests/providers/apache/flink/operators => providers/tests/apache/hdfs}/__init__.py (100%) rename {tests/providers/apache/flink/sensors => providers/tests/apache/hdfs/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/hdfs/hooks/test_webhdfs.py (100%) rename {tests/providers/apache/hdfs => providers/tests/apache/hdfs/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/hdfs/sensors/test_web_hdfs.py (100%) rename {tests/providers => providers/tests}/apache/hive/__init__.py (100%) rename {tests/providers/apache/hdfs => providers/tests/apache/hive}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/apache/hive/hooks/query_results.csv (100%) rename {tests/providers => providers/tests}/apache/hive/hooks/test_hive.py (98%) rename {tests/providers/apache/hdfs/sensors => providers/tests/apache/hive/macros}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/hive/macros/test_hive.py (100%) rename {tests/providers/apache/hive/hooks => providers/tests/apache/hive/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/hive/operators/test_hive.py (99%) rename {tests/providers => providers/tests}/apache/hive/operators/test_hive_stats.py (99%) rename {tests/providers/apache/hive/macros => providers/tests/apache/hive/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/hive/sensors/test_hive_partition.py (96%) rename {tests/providers => providers/tests}/apache/hive/sensors/test_metastore_partition.py (96%) rename {tests/providers => providers/tests}/apache/hive/sensors/test_named_hive_partition.py (99%) rename {tests/providers/amazon => providers/tests/apache/hive/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/hive/transfers/test_hive_to_mysql.py (99%) rename {tests/providers => providers/tests}/apache/hive/transfers/test_hive_to_samba.py (99%) rename {tests/providers => providers/tests}/apache/hive/transfers/test_mssql_to_hive.py (100%) rename {tests/providers => providers/tests}/apache/hive/transfers/test_mysql_to_hive.py (100%) rename {tests/providers => providers/tests}/apache/hive/transfers/test_s3_to_hive.py (100%) rename {tests/providers => providers/tests}/apache/hive/transfers/test_vertica_to_hive.py (100%) rename {tests/providers/amazon/aws => providers/tests/apache/iceberg}/__init__.py (100%) rename {tests/providers/amazon/aws/assets => providers/tests/apache/iceberg/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/iceberg/hooks/test_iceberg.py (100%) rename {tests/providers/amazon/aws/auth_manager => providers/tests/apache/impala}/__init__.py (100%) rename {tests/providers/amazon/aws/auth_manager/avp => providers/tests/apache/impala/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/impala/hooks/test_impala.py (100%) rename {tests/providers/amazon/aws/auth_manager/cli => providers/tests/apache/kafka}/__init__.py (100%) rename {tests/providers/amazon/aws/auth_manager/security_manager => providers/tests/apache/kafka/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/kafka/hooks/test_base.py (100%) rename {tests/providers => providers/tests}/apache/kafka/hooks/test_client.py (100%) rename {tests/providers => providers/tests}/apache/kafka/hooks/test_consume.py (100%) rename {tests/providers => providers/tests}/apache/kafka/hooks/test_produce.py (100%) rename {tests/providers/amazon/aws/auth_manager/views => providers/tests/apache/kafka/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/kafka/operators/test_consume.py (98%) rename {tests/providers => providers/tests}/apache/kafka/operators/test_produce.py (97%) rename {tests/providers/amazon/aws/config_templates => providers/tests/apache/kafka/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/kafka/sensors/test_kafka.py (100%) rename {tests/providers/amazon/aws/executors => providers/tests/apache/kafka/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/kafka/triggers/test_await_message.py (96%) rename {tests/providers/apache/hive/operators => providers/tests/apache/kylin}/__init__.py (100%) rename {tests/providers/apache/hive/sensors => providers/tests/apache/kylin/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/kylin/hooks/test_kylin.py (100%) rename {tests/providers/apache/kylin => providers/tests/apache/kylin/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/kylin/operators/test_kylin_cube.py (100%) rename {tests/providers/amazon/aws/executors/batch => providers/tests/apache/livy}/__init__.py (100%) rename {tests/providers/amazon/aws/executors/ecs => providers/tests/apache/livy/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/livy/hooks/test_livy.py (99%) rename {tests/providers/amazon/aws/executors/utils => providers/tests/apache/livy/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/livy/operators/test_livy.py (100%) rename {tests/providers/amazon/aws/fs => providers/tests/apache/livy/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/livy/sensors/test_livy.py (100%) rename {tests/providers/amazon/aws/hooks => providers/tests/apache/livy/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/livy/triggers/test_livy.py (100%) rename {tests/providers/apache/kylin/hooks => providers/tests/apache/pig}/__init__.py (100%) rename {tests/providers/apache/kylin/operators => providers/tests/apache/pig/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/pig/hooks/test_pig.py (100%) rename {tests/providers/amazon/aws/infrastructure => providers/tests/apache/pig/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/pig/operators/test_pig.py (100%) rename {tests/providers/apache/pig => providers/tests/apache/pinot}/__init__.py (100%) rename {tests/providers/apache/pig => providers/tests/apache/pinot}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/apache/pinot/hooks/test_pinot.py (100%) rename {tests/providers/apache/pinot => providers/tests/apache/spark}/__init__.py (100%) rename {tests/providers/amazon/aws/infrastructure/example_s3_to_redshift => providers/tests/apache/spark/decorators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/spark/decorators/test_pyspark.py (100%) rename {tests/providers/apache/pinot => providers/tests/apache/spark}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/apache/spark/hooks/test_spark_connect.py (100%) rename {tests/providers => providers/tests}/apache/spark/hooks/test_spark_jdbc.py (100%) rename {tests/providers => providers/tests}/apache/spark/hooks/test_spark_jdbc_script.py (100%) rename {tests/providers => providers/tests}/apache/spark/hooks/test_spark_sql.py (99%) rename {tests/providers => providers/tests}/apache/spark/hooks/test_spark_submit.py (100%) rename {tests/providers/apache/spark => providers/tests/apache/spark/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/apache/spark/operators/test_spark_jdbc.py (100%) rename {tests/providers => providers/tests}/apache/spark/operators/test_spark_sql.py (100%) rename {tests/providers => providers/tests}/apache/spark/operators/test_spark_submit.py (100%) rename {tests/providers/amazon/aws/links => providers/tests/apprise}/__init__.py (100%) rename {tests/providers/apache/spark => providers/tests/apprise}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/apprise/hooks/test_apprise.py (100%) rename {tests/providers/apache/spark/operators => providers/tests/apprise/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/apprise/notifications/test_apprise.py (100%) rename {tests/providers/amazon/aws/log => providers/tests/arangodb}/__init__.py (100%) rename {tests/providers/amazon/aws/notifications => providers/tests/arangodb/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/arangodb/hooks/test_arangodb.py (100%) rename {tests/providers/amazon/aws => providers/tests/arangodb}/operators/__init__.py (100%) rename {tests/providers => providers/tests}/arangodb/operators/test_arangodb.py (100%) rename {tests/providers/amazon/aws/secrets => providers/tests/arangodb/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/arangodb/sensors/test_arangodb.py (100%) rename {tests/providers/apprise/hooks => providers/tests/asana}/__init__.py (100%) rename {tests/providers/apprise/notifications => providers/tests/asana/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/asana/hooks/test_asana.py (100%) rename {tests/providers/asana => providers/tests/asana/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/asana/operators/test_asana_tasks.py (100%) rename {tests/providers/amazon/aws/sensors => providers/tests/atlassian}/__init__.py (100%) rename {tests/providers/amazon/aws/system => providers/tests/atlassian/jira}/__init__.py (100%) rename {tests/providers/amazon/aws/system/utils => providers/tests/atlassian/jira/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/atlassian/jira/hooks/test_jira.py (98%) rename {tests/providers/amazon/aws/transfers => providers/tests/atlassian/jira/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/atlassian/jira/notifications/test_jira.py (100%) rename {tests/providers/amazon/aws/triggers => providers/tests/atlassian/jira/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/atlassian/jira/operators/test_jira.py (98%) rename {tests/providers/amazon/aws/utils => providers/tests/atlassian/jira/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/atlassian/jira/sensors/test_jira.py (97%) rename {tests/providers/asana/hooks => providers/tests/celery}/__init__.py (100%) rename {tests/providers/amazon/aws/waiters => providers/tests/celery/cli}/__init__.py (100%) rename {tests/providers => providers/tests}/celery/cli/test_celery_command.py (99%) rename {tests/providers/asana/operators => providers/tests/celery/executors}/__init__.py (100%) rename {tests/providers => providers/tests}/celery/executors/test_celery_executor.py (96%) rename {tests/providers => providers/tests}/celery/executors/test_celery_kubernetes_executor.py (100%) rename {tests/providers/apache => providers/tests/celery/log_handlers}/__init__.py (100%) rename {tests/providers => providers/tests}/celery/log_handlers/test_log_handlers.py (96%) rename {tests/providers/celery => providers/tests/celery/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/celery/sensors/test_celery_queue.py (100%) rename {tests/providers/celery/executors => providers/tests/cloudant}/__init__.py (100%) rename {tests/providers/celery/sensors => providers/tests/cloudant/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/cloudant/hooks/test_cloudant.py (100%) rename {tests/providers/apache/beam => providers/tests/cncf}/__init__.py (100%) rename {tests/providers/cloudant => providers/tests/cncf/kubernetes}/__init__.py (100%) rename {tests/providers/apache/beam/hooks => providers/tests/cncf/kubernetes/cli}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/cli/test_kubernetes_command.py (99%) rename {tests/providers => providers/tests}/cncf/kubernetes/conftest.py (100%) rename {tests/providers/apache/beam/operators => providers/tests/cncf/kubernetes/data_files}/__init__.py (100%) rename {tests/providers/apache/beam/triggers => providers/tests/cncf/kubernetes/data_files/executor}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/executor/basic_template.yaml (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/kube_config (100%) rename {tests/providers/apache/cassandra => providers/tests/cncf/kubernetes/data_files/pods}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/pods/generator_base.yaml (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/pods/template.yaml (100%) rename {tests/providers/apache/cassandra/sensors => providers/tests/cncf/kubernetes/data_files/spark}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/spark/application_template.yaml (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/spark/application_test.json (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/spark/application_test.yaml (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml (100%) rename {tests/providers/apache/druid/transfers => providers/tests/cncf/kubernetes/decorators}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/decorators/test_kubernetes.py (100%) rename {tests/providers/apache/hive/transfers => providers/tests/cncf/kubernetes/executors}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/executors/test_kubernetes_executor.py (99%) rename {tests/providers => providers/tests}/cncf/kubernetes/executors/test_local_kubernetes_executor.py (100%) rename {tests/providers/apache/iceberg => providers/tests/cncf/kubernetes/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/hooks/test_kubernetes.py (99%) rename {tests/providers/apache/iceberg/hooks => providers/tests/cncf/kubernetes/log_handlers}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/log_handlers/test_log_handlers.py (98%) rename {tests/providers/cloudant/hooks => providers/tests/cncf/kubernetes/models}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/models/test_secret.py (100%) rename {tests/providers/apache/impala => providers/tests/cncf/kubernetes/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/operators/test_custom_object_launcher.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/operators/test_job.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/operators/test_pod.py (99%) rename {tests/providers => providers/tests}/cncf/kubernetes/operators/test_resource.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/operators/test_spark_kubernetes.py (99%) rename {tests/providers/apache/impala/hooks => providers/tests/cncf/kubernetes/resource_convert}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/resource_convert/test_configmap.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/resource_convert/test_env_variable.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/resource_convert/test_secret.py (100%) rename {tests/providers/apache/kafka => providers/tests/cncf/kubernetes/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/sensors/test_spark_kubernetes.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/test_callbacks.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/test_client.py (98%) rename {tests/providers => providers/tests}/cncf/kubernetes/test_kubernetes_helper_functions.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/test_pod_generator.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/test_template_rendering.py (96%) rename {tests/providers/apache/kafka/hooks => providers/tests/cncf/kubernetes/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/triggers/test_job.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/triggers/test_pod.py (100%) rename {tests/providers/apache/kafka/operators => providers/tests/cncf/kubernetes/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/utils/test_k8s_resource_iterator.py (100%) rename {tests/providers => providers/tests}/cncf/kubernetes/utils/test_pod_manager.py (99%) rename {tests/providers/apache/kafka/sensors => providers/tests/cohere}/__init__.py (100%) rename {tests/providers/apache/kafka/triggers => providers/tests/cohere/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/cohere/hooks/test_cohere.py (84%) rename {tests/providers/apache/livy => providers/tests/cohere/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/cohere/operators/test_embedding.py (100%) rename {tests/providers/apache/livy/hooks => providers/tests/common}/__init__.py (100%) rename {tests/providers/apache/livy/operators => providers/tests/common/compat}/__init__.py (100%) rename {tests/providers/apache/livy/sensors => providers/tests/common/compat/lineage}/__init__.py (100%) rename {tests/providers => providers/tests}/common/compat/lineage/test_hook.py (100%) rename {tests/providers/apache/livy/triggers => providers/tests/common/compat/openlineage}/__init__.py (100%) rename {tests/providers => providers/tests}/common/compat/openlineage/test_facet.py (100%) rename {tests/providers/apache/pig/operators => providers/tests/common/compat/openlineage/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/common/compat/openlineage/utils/test_utils.py (100%) rename {tests/providers/apache/spark/decorators => providers/tests/common/compat/security}/__init__.py (100%) rename {tests/providers => providers/tests}/common/compat/security/test_permissions.py (100%) rename {tests/providers/apprise => providers/tests/common/io}/__init__.py (100%) rename {tests/providers/arangodb => providers/tests/common/io/assets}/__init__.py (100%) rename {tests/providers => providers/tests}/common/io/assets/test_file.py (100%) rename {tests/providers/arangodb/hooks => providers/tests/common/io/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/common/io/operators/test_file_transfer.py (97%) rename {tests/providers/arangodb/operators => providers/tests/common/io/xcom}/__init__.py (100%) rename {tests/providers => providers/tests}/common/io/xcom/test_backend.py (96%) rename {tests/providers/arangodb/sensors => providers/tests/common/sql}/__init__.py (100%) rename {tests/providers/atlassian => providers/tests/common/sql/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/common/sql/hooks/test_dbapi.py (99%) rename {tests/providers => providers/tests}/common/sql/hooks/test_sql.py (98%) rename {tests/providers => providers/tests}/common/sql/hooks/test_sqlparse.py (96%) rename {tests/providers/atlassian/jira => providers/tests/common/sql/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/common/sql/operators/test_sql.py (99%) rename {tests/providers => providers/tests}/common/sql/operators/test_sql_execute.py (99%) rename {tests/providers/atlassian/jira/hooks => providers/tests/common/sql/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/common/sql/sensors/test_sql.py (99%) rename {tests/providers => providers/tests}/common/sql/test_utils.py (96%) rename {tests/providers => providers/tests}/conftest.py (81%) rename {tests/providers/cncf/kubernetes => providers/tests/databricks}/__init__.py (100%) rename {tests/providers/cncf/kubernetes/models => providers/tests/databricks/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/databricks/hooks/test_databricks.py (100%) rename {tests/providers => providers/tests}/databricks/hooks/test_databricks_base.py (100%) rename {tests/providers => providers/tests}/databricks/hooks/test_databricks_sql.py (90%) rename {tests/providers/databricks => providers/tests/databricks/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/databricks/operators/test_databricks.py (100%) rename {tests/providers => providers/tests}/databricks/operators/test_databricks_copy.py (100%) rename {tests/providers => providers/tests}/databricks/operators/test_databricks_repos.py (100%) rename {tests/providers => providers/tests}/databricks/operators/test_databricks_sql.py (100%) rename {tests/providers => providers/tests}/databricks/operators/test_databricks_workflow.py (100%) rename {tests/providers/atlassian/jira/notifications => providers/tests/databricks/plugins}/__init__.py (100%) rename {tests/providers => providers/tests}/databricks/plugins/test_databricks_workflow.py (97%) rename {tests/providers/atlassian/jira/operators => providers/tests/databricks/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/databricks/sensors/test_databricks_partition.py (100%) rename {tests/providers => providers/tests}/databricks/sensors/test_databricks_sql.py (100%) rename {tests/providers => providers/tests}/databricks/test_exceptions.py (100%) rename {tests/providers/databricks/hooks => providers/tests/databricks/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/databricks/triggers/test_databricks.py (100%) rename {tests/providers/atlassian/jira/sensors => providers/tests/databricks/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/databricks/utils/test_databricks.py (100%) rename {tests/providers/databricks/operators => providers/tests/datadog}/__init__.py (100%) rename {tests/providers/databricks/triggers => providers/tests/datadog/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/datadog/hooks/test_datadog.py (100%) rename {tests/providers/datadog => providers/tests/datadog/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/datadog/sensors/test_datadog.py (100%) rename {tests/providers/celery/cli => providers/tests/dbt}/__init__.py (100%) rename {tests/providers/celery/log_handlers => providers/tests/dbt/cloud}/__init__.py (100%) rename {tests/providers/cncf => providers/tests/dbt/cloud/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/dbt/cloud/hooks/test_dbt.py (99%) rename {tests/providers/cncf/kubernetes/cli => providers/tests/dbt/cloud/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/dbt/cloud/operators/test_dbt.py (99%) rename {tests/providers/cncf/kubernetes/data_files => providers/tests/dbt/cloud/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/dbt/cloud/sensors/test_dbt.py (100%) rename {tests/providers/cncf/kubernetes/data_files/executor => providers/tests/dbt/cloud/test_data}/__init__.py (100%) rename {tests/providers => providers/tests}/dbt/cloud/test_data/catalog.json (100%) rename {tests/providers => providers/tests}/dbt/cloud/test_data/job_run.json (100%) rename {tests/providers => providers/tests}/dbt/cloud/test_data/manifest.json (100%) rename {tests/providers => providers/tests}/dbt/cloud/test_data/run_results.json (100%) rename {tests/providers/cncf/kubernetes/data_files/pods => providers/tests/dbt/cloud/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/dbt/cloud/triggers/test_dbt.py (100%) rename {tests/providers/cncf/kubernetes/data_files/spark => providers/tests/dbt/cloud/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/dbt/cloud/utils/test_openlineage.py (100%) create mode 100644 providers/tests/deprecations_ignore.yml rename {tests/providers/datadog/hooks => providers/tests/dingding}/__init__.py (100%) rename {tests/providers/datadog/sensors => providers/tests/dingding/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/dingding/hooks/test_dingding.py (100%) rename {tests/providers/dingding => providers/tests/dingding/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/dingding/operators/test_dingding.py (100%) rename {tests/providers/dingding/hooks => providers/tests/discord}/__init__.py (100%) rename {tests/providers/dingding/operators => providers/tests/discord/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/discord/hooks/test_discord_webhook.py (100%) rename {tests/providers/cncf/kubernetes/decorators => providers/tests/discord/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/discord/notifications/test_discord.py (100%) rename {tests/providers/discord => providers/tests/discord/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/discord/operators/test_discord_webhook.py (100%) rename {tests/providers/discord/hooks => providers/tests/docker}/__init__.py (100%) rename {tests/providers => providers/tests}/docker/conftest.py (100%) rename {tests/providers/cncf/kubernetes/executors => providers/tests/docker/decorators}/__init__.py (100%) rename {tests/providers => providers/tests}/docker/decorators/test_docker.py (100%) rename {tests/providers/discord/operators => providers/tests/docker/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/docker/hooks/test_docker.py (100%) rename {tests/providers/docker => providers/tests/docker/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/docker/operators/test_docker.py (99%) rename {tests/providers => providers/tests}/docker/operators/test_docker_swarm.py (100%) rename {tests/providers => providers/tests}/docker/test_exceptions.py (100%) rename {tests/providers/docker/hooks => providers/tests/edge}/__init__.py (100%) rename {tests/providers/docker/operators => providers/tests/edge/api_endpoints}/__init__.py (100%) rename {tests/providers => providers/tests}/edge/api_endpoints/test_health_endpoint.py (100%) rename {tests/providers => providers/tests}/edge/api_endpoints/test_rpc_api_endpoint.py (98%) rename {tests/providers/edge => providers/tests/edge/cli}/__init__.py (100%) rename {tests/providers => providers/tests}/edge/cli/test_edge_command.py (99%) rename {tests/providers/edge/api_endpoints => providers/tests/edge/models}/__init__.py (100%) rename {tests/providers => providers/tests}/edge/models/test_edge_job.py (100%) rename {tests/providers => providers/tests}/edge/models/test_edge_logs.py (100%) rename {tests/providers => providers/tests}/edge/models/test_edge_worker.py (100%) rename {tests/providers/edge/cli => providers/tests/edge/plugins}/__init__.py (100%) rename {tests/providers => providers/tests}/edge/plugins/test_edge_executor_plugin.py (97%) rename {tests/providers/edge/models => providers/tests/elasticsearch}/__init__.py (100%) rename {tests/providers/edge/plugins => providers/tests/elasticsearch/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/elasticsearch/hooks/test_elasticsearch.py (100%) rename {tests/providers/cncf/kubernetes/hooks => providers/tests/elasticsearch/log}/__init__.py (100%) rename {tests/providers => providers/tests}/elasticsearch/log/elasticmock/__init__.py (98%) rename {tests/providers => providers/tests}/elasticsearch/log/elasticmock/fake_elasticsearch.py (99%) rename {tests/providers => providers/tests}/elasticsearch/log/elasticmock/utilities/__init__.py (100%) rename {tests/providers => providers/tests}/elasticsearch/log/test_es_json_formatter.py (100%) rename {tests/providers => providers/tests}/elasticsearch/log/test_es_response.py (100%) rename {tests/providers => providers/tests}/elasticsearch/log/test_es_task_handler.py (99%) rename {tests/providers/elasticsearch => providers/tests/email}/__init__.py (100%) rename {tests/providers/elasticsearch/hooks => providers/tests/email/operators}/__init__.py (100%) rename {tests/providers/email => providers/tests/exasol}/__init__.py (100%) rename {tests/providers/email/operators => providers/tests/exasol/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/exasol/hooks/test_exasol.py (100%) rename {tests/providers => providers/tests}/exasol/hooks/test_sql.py (100%) rename {tests/providers/exasol => providers/tests/exasol/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/exasol/operators/test_exasol.py (100%) rename {tests/providers => providers/tests}/exasol/operators/test_exasol_sql.py (100%) rename {tests/providers/exasol/hooks => providers/tests/fab}/__init__.py (100%) rename {tests/providers/exasol/operators => providers/tests/fab/auth_manager}/__init__.py (100%) rename {tests/providers/fab => providers/tests/fab/auth_manager/api}/__init__.py (100%) rename {tests/providers/cncf/kubernetes/log_handlers => providers/tests/fab/auth_manager/api/auth}/__init__.py (100%) rename {tests/providers/cncf/kubernetes/operators => providers/tests/fab/auth_manager/api/auth/backend}/__init__.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/api/auth/backend/test_basic_auth.py (98%) rename {tests/providers => providers/tests}/fab/auth_manager/api/auth/backend/test_kerberos_auth.py (92%) rename {tests/providers/cncf/kubernetes/resource_convert => providers/tests/fab/auth_manager/api_endpoints}/__init__.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/api_connexion_utils.py (97%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/remote_user_api_auth_backend.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_asset_endpoint.py (97%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_auth.py (95%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_backfill_endpoint.py (97%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_cors.py (96%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_dag_endpoint.py (96%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py (97%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py (95%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py (92%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_event_log_endpoint.py (96%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_import_error_endpoint.py (96%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py (99%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py (97%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py (98%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_user_endpoint.py (99%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_user_schema.py (97%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_variable_endpoint.py (93%) rename {tests/providers => providers/tests}/fab/auth_manager/api_endpoints/test_xcom_endpoint.py (97%) rename {tests/providers/cncf/kubernetes/sensors => providers/tests/fab/auth_manager/cli_commands}/__init__.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/cli_commands/test_db_command.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/cli_commands/test_definition.py (94%) rename {tests/providers => providers/tests}/fab/auth_manager/cli_commands/test_role_command.py (98%) rename {tests/providers => providers/tests}/fab/auth_manager/cli_commands/test_sync_perm_command.py (96%) rename {tests/providers => providers/tests}/fab/auth_manager/cli_commands/test_user_command.py (99%) rename {tests/providers => providers/tests}/fab/auth_manager/cli_commands/test_utils.py (93%) rename {tests/providers => providers/tests}/fab/auth_manager/conftest.py (91%) rename {tests/providers/cncf/kubernetes/triggers => providers/tests/fab/auth_manager/decorators}/__init__.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/decorators/test_auth.py (98%) rename {tests/providers/fab/auth_manager => providers/tests/fab/auth_manager/models}/__init__.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/models/test_anonymous_user.py (93%) rename {tests/providers => providers/tests}/fab/auth_manager/models/test_db.py (93%) rename {tests/providers/cncf/kubernetes/utils => providers/tests/fab/auth_manager/security_manager}/__init__.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/security_manager/test_constants.py (93%) rename {tests/providers => providers/tests}/fab/auth_manager/security_manager/test_override.py (95%) rename {tests/providers => providers/tests}/fab/auth_manager/test_fab_auth_manager.py (99%) rename {tests/providers => providers/tests}/fab/auth_manager/test_models.py (96%) rename {tests/providers => providers/tests}/fab/auth_manager/test_security.py (98%) rename {tests/providers/fab/auth_manager/api => providers/tests/fab/auth_manager/views}/__init__.py (100%) rename {tests/providers => providers/tests}/fab/auth_manager/views/test_permissions.py (93%) rename {tests/providers => providers/tests}/fab/auth_manager/views/test_roles_list.py (91%) rename {tests/providers => providers/tests}/fab/auth_manager/views/test_user.py (91%) rename {tests/providers => providers/tests}/fab/auth_manager/views/test_user_edit.py (91%) rename {tests/providers => providers/tests}/fab/auth_manager/views/test_user_stats.py (91%) rename {tests/providers/cohere => providers/tests/facebook}/__init__.py (100%) rename {tests/providers/cohere/hooks => providers/tests/facebook/ads}/__init__.py (100%) rename {tests/providers/cohere/operators => providers/tests/facebook/ads/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/facebook/ads/hooks/test_ads.py (100%) rename {tests/providers/fab/auth_manager/models => providers/tests/ftp}/__init__.py (100%) rename {tests/providers/fab/auth_manager/views => providers/tests/ftp/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/ftp/hooks/test_ftp.py (100%) rename {tests/providers/common => providers/tests/ftp/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/ftp/operators/test_ftp.py (100%) rename {tests/providers/ftp => providers/tests/ftp/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/ftp/sensors/test_ftp.py (100%) rename {tests/providers/common/compat => providers/tests/github}/__init__.py (100%) rename {tests/providers/common/compat/lineage => providers/tests/github/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/github/hooks/test_github.py (100%) rename {tests/providers/common/compat/openlineage => providers/tests/github/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/github/operators/test_github.py (100%) rename {tests/providers/common/compat/openlineage/utils => providers/tests/github/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/github/sensors/test_github.py (100%) rename {tests/providers/common/compat/security => providers/tests/google}/__init__.py (100%) rename {tests/providers/common/io => providers/tests/google/ads}/__init__.py (100%) rename {tests/providers/common/io/assets => providers/tests/google/ads/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/google/ads/hooks/test_ads.py (100%) rename {tests/providers/common/io => providers/tests/google/ads}/operators/__init__.py (100%) rename {tests/providers => providers/tests}/google/ads/operators/test_ads.py (100%) rename {tests/providers/common/io/xcom => providers/tests/google/ads/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/google/ads/transfers/test_ads_to_gcs.py (97%) rename {tests/providers/common/sql => providers/tests/google/assets}/__init__.py (100%) rename {tests/providers/google/datasets => providers/tests/google/assets}/test_bigquery.py (100%) rename {tests/providers/common/sql/hooks => providers/tests/google/cloud}/__init__.py (100%) rename {tests/providers/common/sql/operators => providers/tests/google/cloud/_internal_client}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/_internal_client/test_secret_manager_client.py (100%) rename {tests/providers/common/sql/sensors => providers/tests/google/cloud/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_automl.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_bigquery.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_bigquery_dts.py (98%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_bigquery_system.py (94%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_bigtable.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_batch.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_build.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_composer.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_memorystore.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_run.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_sql.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_storage_transfer_service.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_cloud_storage_transfer_service_async.py (98%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_compute.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_compute_ssh.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_datacatalog.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_dataflow.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_dataform.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_datafusion.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_datapipeline.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_dataplex.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_dataprep.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_dataproc.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_dataproc_metastore.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_datastore.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_dlp.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_functions.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_gcs.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_gdm.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_kms.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_kms_system.py (96%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_kubernetes_engine.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_life_sciences.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_looker.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_mlengine.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_natural_language.py (98%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_os_login.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_pubsub.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_secret_manager.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_secret_manager_system.py (95%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_spanner.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_speech_to_text.py (97%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_stackdriver.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_tasks.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_text_to_speech.py (97%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_translate.py (98%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_video_intelligence.py (98%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_vision.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/test_workflows.py (100%) rename {tests/providers/databricks/plugins => providers/tests/google/cloud/hooks/vertex_ai}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_auto_ml.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_custom_job.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_dataset.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_endpoint_service.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_generative_model.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_model_service.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_pipeline_job.py (99%) rename {tests/providers => providers/tests}/google/cloud/hooks/vertex_ai/test_prediction_service.py (98%) rename {tests/providers/databricks/sensors => providers/tests/google/cloud/links}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/links/test_translate.py (100%) rename {tests/providers/databricks/utils => providers/tests/google/cloud/log}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/log/test_gcs_task_handler.py (98%) rename {tests/providers => providers/tests}/google/cloud/log/test_gcs_task_handler_system.py (84%) rename {tests/providers => providers/tests}/google/cloud/log/test_stackdriver_task_handler.py (99%) rename {tests/providers => providers/tests}/google/cloud/log/test_stackdriver_task_handler_system.py (88%) rename {tests/providers/dbt => providers/tests/google/cloud/openlineage}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/openlineage/test_mixins.py (95%) rename {tests/providers => providers/tests}/google/cloud/openlineage/test_utils.py (97%) rename {tests/providers/dbt/cloud => providers/tests/google/cloud/operators}/__init__.py (100%) rename {tests/providers/dbt/cloud/hooks => providers/tests/google/cloud/operators/source}/__init__.py (100%) rename {tests/providers/dbt/cloud/operators => providers/tests/google/cloud/operators/source/source_prefix}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_automl.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_bigquery.py (99%) rename {tests/providers => providers/tests}/google/cloud/operators/test_bigquery_dts.py (98%) rename {tests/providers => providers/tests}/google/cloud/operators/test_bigtable.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_base.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_batch.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_build.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_composer.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_memorystore.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_run.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_sql.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_cloud_storage_transfer_service.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_compute.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_datacatalog.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dataflow.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dataform.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_datafusion.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_datapipeline.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dataplex.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dataprep.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dataprep_system.py (91%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dataproc.py (99%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dataproc_metastore.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_datastore.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_datastore_system.py (89%) rename {tests/providers => providers/tests}/google/cloud/operators/test_dlp.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_functions.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_kubernetes_engine.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_life_sciences.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_looker.py (98%) rename {tests/providers => providers/tests}/google/cloud/operators/test_mlengine.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_natural_language.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_pubsub.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_spanner.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_speech_to_text.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_stackdriver.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_tasks.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_text_to_speech.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_translate.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_translate_speech.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_vertex_ai.py (99%) rename {tests/providers => providers/tests}/google/cloud/operators/test_video_intelligence.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_vision.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/test_workflows.py (100%) rename {tests/providers/dbt/cloud/sensors => providers/tests/google/cloud/operators/vertex_ai}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/operators/vertex_ai/test_generative_model.py (100%) rename {tests/providers/dbt/cloud/test_data => providers/tests/google/cloud/secrets}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/secrets/test_secret_manager.py (100%) rename {tests/providers => providers/tests}/google/cloud/secrets/test_secret_manager_system.py (95%) rename {tests/providers/dbt/cloud/triggers => providers/tests/google/cloud/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_bigquery.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_bigquery_dts.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_bigtable.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_cloud_composer.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_cloud_storage_transfer_service.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_dataflow.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_datafusion.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_dataplex.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_dataprep.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_dataproc.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_dataproc_metastore.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_looker.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_pubsub.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_tasks.py (100%) rename {tests/providers => providers/tests}/google/cloud/sensors/test_workflows.py (100%) rename {tests/providers/dbt/cloud/utils => providers/tests/google/cloud/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_adls_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_azure_blob_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_azure_fileshare_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_bigquery_to_bigquery.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_bigquery_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_bigquery_to_mssql.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_bigquery_to_mysql.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_bigquery_to_postgres.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_calendar_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_cassandra_to_gcs.py (91%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_facebook_ads_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_facebook_ads_to_gcs_system.py (93%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_gcs_to_bigquery.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_gcs_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_gcs_to_local.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_gcs_to_sftp.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_gdrive_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_gdrive_to_local.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_local_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_mssql_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_mysql_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_oracle_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_postgres_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_s3_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_salesforce_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_salesforce_to_gcs_system.py (84%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_sftp_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_sheets_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_sql_to_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/transfers/test_trino_to_gcs.py (100%) rename {tests/providers/discord/notifications => providers/tests/google/cloud/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_bigquery.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_bigquery_dts.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_cloud_batch.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_cloud_build.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_cloud_composer.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_cloud_run.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_cloud_sql.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_cloud_storage_transfer_service.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_dataflow.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_datafusion.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_dataplex.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_dataproc.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_gcs.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_kubernetes_engine.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_mlengine.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_pubsub.py (100%) rename {tests/providers => providers/tests}/google/cloud/triggers/test_vertex_ai.py (99%) rename {tests/providers/ftp/hooks => providers/tests/google/cloud/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/airflow_util.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/base_gcp_mock.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/gcp_authenticator.py (98%) rename {tests/providers => providers/tests}/google/cloud/utils/job_details.json (100%) rename {tests/providers => providers/tests}/google/cloud/utils/out_table_details.json (100%) rename {tests/providers => providers/tests}/google/cloud/utils/script_job_details.json (100%) rename {tests/providers => providers/tests}/google/cloud/utils/table_details.json (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_credentials_provider.py (98%) rename {tests/providers => providers/tests}/google/cloud/utils/test_datafusion.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_dataproc.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_external_token_supplier.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_field_sanitizer.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_field_validator.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_helpers.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_mlengine_operator_utils.py (100%) rename {tests/providers => providers/tests}/google/cloud/utils/test_mlengine_prediction_summary.py (91%) rename {tests/providers/docker/decorators => providers/tests/google/common}/__init__.py (100%) rename {tests/providers/elasticsearch/log => providers/tests/google/common/auth_backend}/__init__.py (100%) rename {tests/providers => providers/tests}/google/common/auth_backend/test_google_openid.py (95%) rename {tests/providers/fab/auth_manager/api/auth => providers/tests/google/common/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/google/common/hooks/test_base_google.py (99%) rename {tests/providers => providers/tests}/google/common/hooks/test_discovery_api.py (100%) rename {tests/providers => providers/tests}/google/common/test_deprecated.py (100%) rename {tests/providers/fab/auth_manager/api/auth/backend => providers/tests/google/common/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/google/common/utils/test_id_token_credentials.py (100%) rename {tests/providers/fab/auth_manager/api_endpoints => providers/tests/google/firebase}/__init__.py (100%) rename {tests/providers/fab/auth_manager/cli_commands => providers/tests/google/firebase/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/google/firebase/hooks/test_firestore.py (99%) rename {tests/providers/fab/auth_manager/decorators => providers/tests/google/firebase/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/google/firebase/operators/test_firestore.py (100%) rename {tests/providers/ftp/sensors => providers/tests/google/leveldb}/__init__.py (100%) rename {tests/providers/google/cloud/utils => providers/tests/google/leveldb/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/google/leveldb/hooks/test_leveldb.py (100%) rename {tests/providers/google/leveldb => providers/tests/google/leveldb/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/google/leveldb/operators/test_leveldb.py (100%) rename {tests/providers/fab/auth_manager/security_manager => providers/tests/google/marketing_platform}/__init__.py (100%) rename {tests/providers/facebook => providers/tests/google/marketing_platform/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/hooks/test_analytics.py (96%) rename {tests/providers => providers/tests}/google/marketing_platform/hooks/test_analytics_admin.py (99%) rename {tests/providers => providers/tests}/google/marketing_platform/hooks/test_campaign_manager.py (99%) rename {tests/providers => providers/tests}/google/marketing_platform/hooks/test_display_video.py (99%) rename {tests/providers => providers/tests}/google/marketing_platform/hooks/test_search_ads.py (99%) rename {tests/providers/google/leveldb/hooks => providers/tests/google/marketing_platform/links}/__init__.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/links/test_analytics_admin.py (100%) rename {tests/providers/facebook/ads => providers/tests/google/marketing_platform/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/operators/test_analytics.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/operators/test_analytics_admin.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/operators/test_campaign_manager.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/operators/test_display_video.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/operators/test_display_video_system.py (91%) rename {tests/providers => providers/tests}/google/marketing_platform/operators/test_search_ads.py (100%) rename {tests/providers/facebook/ads/hooks => providers/tests/google/marketing_platform/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/sensors/test_campaign_manager.py (100%) rename {tests/providers => providers/tests}/google/marketing_platform/sensors/test_display_video.py (100%) rename {tests/providers/google/leveldb/operators => providers/tests/google/suite}/__init__.py (100%) rename {tests/providers/google/marketing_platform/links => providers/tests/google/suite/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/google/suite/hooks/test_calendar.py (98%) rename {tests/providers => providers/tests}/google/suite/hooks/test_drive.py (99%) rename {tests/providers => providers/tests}/google/suite/hooks/test_sheets.py (99%) rename {tests/providers/google/suite => providers/tests/google/suite/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/google/suite/operators/test_sheets.py (100%) rename {tests/providers/google/suite/hooks => providers/tests/google/suite/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/google/suite/sensors/test_drive.py (100%) rename {tests/providers/ftp/operators => providers/tests/google/suite/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/google/suite/transfers/test_gcs_to_gdrive.py (100%) rename {tests/providers => providers/tests}/google/suite/transfers/test_gcs_to_sheets.py (100%) rename {tests/providers => providers/tests}/google/suite/transfers/test_local_to_drive.py (100%) rename {tests/providers => providers/tests}/google/suite/transfers/test_sql_to_sheets.py (100%) rename {tests/providers => providers/tests}/google/test_go_module.py (100%) rename {tests/providers/google/suite/operators => providers/tests/grpc}/__init__.py (100%) rename {tests/providers/google/suite/sensors => providers/tests/grpc/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/grpc/hooks/test_grpc.py (100%) rename {tests/providers/grpc => providers/tests/grpc/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/grpc/operators/test_grpc.py (100%) rename {tests/providers/github => providers/tests/hashicorp}/__init__.py (100%) rename {tests/providers/github/hooks => providers/tests/hashicorp/_internal_client}/__init__.py (100%) rename {tests/providers => providers/tests}/hashicorp/_internal_client/test_vault_client.py (100%) rename {tests/providers/github/operators => providers/tests/hashicorp/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/hashicorp/hooks/test_vault.py (99%) rename {tests/providers/github/sensors => providers/tests/hashicorp/secrets}/__init__.py (100%) rename {tests/providers => providers/tests}/hashicorp/secrets/test_vault.py (100%) rename {tests/providers/grpc/hooks => providers/tests/http}/__init__.py (100%) rename {tests/providers/grpc/operators => providers/tests/http/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/http/hooks/test_http.py (95%) rename {tests/providers/http => providers/tests/http/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/http/operators/test_http.py (100%) rename {tests/providers/http/hooks => providers/tests/http/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/http/sensors/test_http.py (100%) rename {tests/providers/http/operators => providers/tests/http/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/http/triggers/test_http.py (100%) rename {tests/providers/http/sensors => providers/tests/imap}/__init__.py (100%) rename {tests/providers/http/triggers => providers/tests/imap/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/imap/hooks/test_imap.py (99%) rename {tests/providers/imap => providers/tests/imap/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/imap/sensors/test_imap_attachment.py (100%) rename {tests/providers/google => providers/tests/influxdb}/__init__.py (100%) rename {tests/providers/google/ads => providers/tests/influxdb/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/influxdb/hooks/test_influxdb.py (100%) rename {tests/providers/google/ads/hooks => providers/tests/influxdb/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/influxdb/operators/test_influxdb.py (100%) rename {tests/providers/google/ads/operators => providers/tests/integration}/__init__.py (100%) rename {tests/providers/google/ads/transfers => providers/tests/integration/apache}/__init__.py (100%) rename {tests/providers/google/cloud => providers/tests/integration/apache/cassandra}/__init__.py (100%) rename {tests/providers/google/cloud/_internal_client => providers/tests/integration/apache/cassandra/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/cassandra/hooks/test_cassandra.py (100%) rename {tests/providers/google/cloud/hooks => providers/tests/integration/apache/drill}/__init__.py (100%) rename {tests/providers/google/cloud/hooks/vertex_ai => providers/tests/integration/apache/drill/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/drill/hooks/test_drill.py (100%) rename {tests/providers/google/cloud/links => providers/tests/integration/apache/drill/operators}/__init__.py (100%) rename {tests/providers/google/cloud/log => providers/tests/integration/apache/hive}/__init__.py (100%) rename {tests/providers/google/cloud/openlineage => providers/tests/integration/apache/hive/transfers}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/hive/transfers/test_mssql_to_hive.py (100%) rename {tests/providers/google/cloud/operators => providers/tests/integration/apache/kafka}/__init__.py (100%) rename {tests/providers/google/cloud/operators/source => providers/tests/integration/apache/kafka/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/kafka/hooks/test_admin_client.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/kafka/hooks/test_consumer.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/kafka/hooks/test_producer.py (100%) rename {tests/providers/google/cloud/operators/source/source_prefix => providers/tests/integration/apache/kafka/operators}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/kafka/operators/test_consume.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/kafka/operators/test_produce.py (100%) rename {tests/providers/google/cloud/operators/vertex_ai => providers/tests/integration/apache/kafka/sensors}/__init__.py (100%) rename {tests/providers/google/cloud/secrets => providers/tests/integration/apache/kafka/triggers}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/kafka/triggers/test_await_message.py (100%) rename {tests/providers/google/cloud/sensors => providers/tests/integration/apache/pinot}/__init__.py (100%) rename {tests/providers/google/cloud/transfers => providers/tests/integration/apache/pinot/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/apache/pinot/hooks/test_pinot.py (100%) rename {tests/providers/google/cloud/triggers => providers/tests/integration/google}/__init__.py (100%) rename {tests/providers/google/common => providers/tests/integration/google/cloud}/__init__.py (100%) rename {tests/providers/google/common/auth_backend => providers/tests/integration/google/cloud/transfers}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/google/cloud/transfers/test_bigquery_to_mssql.py (100%) rename {tests/integration/providers => providers/tests/integration}/google/cloud/transfers/test_mssql_to_gcs.py (100%) rename {tests/integration/providers => providers/tests/integration}/google/cloud/transfers/test_trino_to_gcs.py (100%) rename {tests/providers/google/common/hooks => providers/tests/integration/microsoft}/__init__.py (100%) rename {tests/providers/google/common/utils => providers/tests/integration/microsoft/mssql}/__init__.py (100%) rename {tests/providers/google/datasets => providers/tests/integration/microsoft/mssql/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/microsoft/mssql/hooks/test_mssql.py (100%) rename {tests/providers/google/firebase => providers/tests/integration/mongo}/__init__.py (100%) rename {tests/providers/google/firebase/hooks => providers/tests/integration/mongo/sensors}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/mongo/sensors/test_mongo.py (100%) rename {tests/providers/google/firebase/operators => providers/tests/integration/openlineage}/__init__.py (100%) rename {tests/providers/google/marketing_platform => providers/tests/integration/openlineage/operators}/__init__.py (100%) rename {tests/providers/google/marketing_platform/hooks => providers/tests/integration/qdrant}/__init__.py (100%) rename {tests/providers/google/marketing_platform/operators => providers/tests/integration/qdrant/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/qdrant/hooks/test_qdrant.py (100%) rename {tests/providers/google/marketing_platform/sensors => providers/tests/integration/qdrant/operators}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/qdrant/operators/test_qdrant_ingest.py (100%) rename {tests/providers/google/suite/transfers => providers/tests/integration/redis}/__init__.py (100%) rename {tests/providers/hashicorp => providers/tests/integration/redis/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/redis/hooks/test_redis.py (100%) rename {tests/providers/hashicorp/_internal_client => providers/tests/integration/redis/operators}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/redis/operators/test_redis_publish.py (100%) rename {tests/providers/hashicorp/hooks => providers/tests/integration/redis/sensors}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/redis/sensors/test_redis_key.py (100%) rename {tests/integration/providers => providers/tests/integration}/redis/sensors/test_redis_pub_sub.py (100%) rename {tests/providers/hashicorp/secrets => providers/tests/integration/trino}/__init__.py (100%) rename {tests/providers/influxdb => providers/tests/integration/trino/hooks}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/trino/hooks/test_trino.py (100%) rename {tests/providers/influxdb/hooks => providers/tests/integration/ydb}/__init__.py (100%) rename {tests/providers/influxdb/operators => providers/tests/integration/ydb/hooks}/__init__.py (100%) rename {tests/providers/jenkins/hooks => providers/tests/integration/ydb/operators}/__init__.py (100%) rename {tests/integration/providers => providers/tests/integration}/ydb/operators/test_ydb.py (100%) rename {tests/providers/imap/hooks => providers/tests/jdbc}/__init__.py (100%) rename {tests/providers/imap/sensors => providers/tests/jdbc/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/jdbc/hooks/test_jdbc.py (97%) rename {tests/providers/jdbc => providers/tests/jdbc/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/jdbc/operators/test_jdbc.py (100%) rename {tests/providers/jdbc/hooks => providers/tests/jenkins}/__init__.py (100%) rename {tests/providers/microsoft/azure/log => providers/tests/jenkins/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/jenkins/hooks/test_jenkins.py (100%) rename {tests/providers/jdbc => providers/tests/jenkins}/operators/__init__.py (100%) rename {tests/providers => providers/tests}/jenkins/operators/test_jenkins_job_trigger.py (85%) rename {tests/providers/jenkins => providers/tests/jenkins/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/jenkins/sensors/test_jenkins.py (100%) rename {tests/providers/jenkins/operators => providers/tests/microsoft}/__init__.py (100%) rename {tests/providers/jenkins/sensors => providers/tests/microsoft/azure}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/base.py (91%) rename {tests/providers/microsoft => providers/tests/microsoft/azure/fs}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/fs/test_adls.py (100%) rename {tests/providers/microsoft/azure => providers/tests/microsoft/azure/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_adx.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_asb.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_base_azure.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_batch.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_container_instance.py (95%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_container_registry.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_container_volume.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_cosmos.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_data_factory.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_data_lake.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_fileshare.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_msgraph.py (99%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_powerbi.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_synapse.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_synapse_pipeline.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/hooks/test_wasb.py (100%) rename {tests/providers/microsoft/azure/resources => providers/tests/microsoft/azure/log}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/log/test_wasb_task_handler.py (98%) rename {tests/providers/microsoft/azure/fs => providers/tests/microsoft/azure/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_adls_create.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_adls_delete.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_adls_list.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_adx.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_asb.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_batch.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_container_instances.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_cosmos.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_data_factory.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_msgraph.py (98%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_powerbi.py (97%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_synapse.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/operators/test_wasb_delete_blob.py (100%) rename {tests/providers/microsoft/azure/secrets => providers/tests/microsoft/azure/resources}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/resources/dummy.pdf (100%) rename {tests/providers => providers/tests}/microsoft/azure/resources/next_users.json (100%) rename {tests/providers => providers/tests}/microsoft/azure/resources/status.json (100%) rename {tests/providers => providers/tests}/microsoft/azure/resources/users.json (100%) rename {tests/providers/microsoft/azure/transfers => providers/tests/microsoft/azure/secrets}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/secrets/test_key_vault.py (100%) rename {tests/providers/microsoft/azure/hooks => providers/tests/microsoft/azure/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/sensors/test_cosmos.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/sensors/test_data_factory.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/sensors/test_msgraph.py (95%) rename {tests/providers => providers/tests}/microsoft/azure/sensors/test_wasb.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/test_utils.py (100%) rename {tests/providers/microsoft/azure/triggers => providers/tests/microsoft/azure/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/transfers/test_local_to_adls.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/transfers/test_local_to_wasb.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/transfers/test_oracle_to_azure_data_lake.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/transfers/test_s3_to_wasb.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/transfers/test_sftp_to_wasb.py (100%) rename {tests/providers/microsoft/mssql/operators => providers/tests/microsoft/azure/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/triggers/test_data_factory.py (100%) rename {tests/providers => providers/tests}/microsoft/azure/triggers/test_msgraph.py (98%) rename {tests/providers => providers/tests}/microsoft/azure/triggers/test_powerbi.py (99%) rename {tests/providers => providers/tests}/microsoft/azure/triggers/test_wasb.py (100%) rename {tests/providers => providers/tests}/microsoft/conftest.py (100%) rename {tests/providers/microsoft/azure/operators => providers/tests/microsoft/mssql}/__init__.py (100%) rename {tests/providers/microsoft/azure/sensors => providers/tests/microsoft/mssql/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/mssql/hooks/test_mssql.py (99%) rename {tests/providers/mysql/assets => providers/tests/microsoft/mssql/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/mssql/operators/test_mssql.py (100%) rename {tests/providers/microsoft/mssql => providers/tests/microsoft/mssql/resources}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/mssql/resources/replace.sql (100%) rename {tests/providers/microsoft/mssql/hooks => providers/tests/microsoft/psrp}/__init__.py (100%) rename {tests/providers/microsoft/mssql/resources => providers/tests/microsoft/psrp/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/psrp/hooks/test_psrp.py (96%) rename {tests/providers/microsoft/psrp => providers/tests/microsoft/psrp/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/psrp/operators/test_psrp.py (100%) rename {tests/providers/microsoft/psrp/hooks => providers/tests/microsoft/winrm}/__init__.py (100%) rename {tests/providers/microsoft/psrp/operators => providers/tests/microsoft/winrm/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/winrm/hooks/test_winrm.py (100%) rename {tests/providers/microsoft/winrm => providers/tests/microsoft/winrm/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/microsoft/winrm/operators/test_winrm.py (100%) rename {tests/providers/microsoft/winrm/hooks => providers/tests/mongo}/__init__.py (100%) rename {tests/providers/microsoft/winrm/operators => providers/tests/mongo/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/mongo/hooks/test_mongo.py (99%) rename {tests/providers/mongo => providers/tests/mongo/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/mongo/sensors/test_mongo.py (100%) rename {tests/providers/mongo/hooks => providers/tests/mysql}/__init__.py (100%) rename {tests/providers/mysql/transfers => providers/tests/mysql/assets}/__init__.py (100%) rename {tests/providers => providers/tests}/mysql/assets/test_mysql.py (100%) rename {tests/providers/mongo/sensors => providers/tests/mysql/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/mysql/hooks/test_mysql.py (99%) rename {tests/providers => providers/tests}/mysql/hooks/test_mysql_connector_python.py (100%) rename {tests/providers/mysql => providers/tests/mysql/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/mysql/operators/test_mysql.py (99%) rename {tests/providers/odbc => providers/tests/mysql/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/mysql/transfers/test_presto_to_mysql.py (100%) rename {tests/providers => providers/tests}/mysql/transfers/test_s3_to_mysql.py (100%) rename {tests/providers => providers/tests}/mysql/transfers/test_trino_to_mysql.py (100%) rename {tests/providers => providers/tests}/mysql/transfers/test_vertica_to_mysql.py (100%) rename {tests/providers/mysql/hooks => providers/tests/neo4j}/__init__.py (100%) rename {tests/providers/mysql/operators => providers/tests/neo4j/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/neo4j/hooks/test_neo4j.py (100%) rename {tests/providers/neo4j => providers/tests/neo4j/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/neo4j/operators/test_neo4j.py (100%) rename {tests/providers/odbc/hooks => providers/tests/odbc}/__init__.py (100%) rename {tests/providers/openai => providers/tests/odbc/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/odbc/hooks/test_odbc.py (99%) rename {tests/providers/openai/hooks => providers/tests/openai}/__init__.py (100%) rename {tests/providers/openai/operators => providers/tests/openai/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/openai/hooks/test_openai.py (100%) rename {tests/providers/openai/triggers => providers/tests/openai/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/openai/operators/test_openai.py (100%) rename {tests/providers => providers/tests}/openai/test_exceptions.py (100%) rename {tests/providers/openlineage => providers/tests/openai/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/openai/triggers/test_openai.py (100%) rename {tests/providers/neo4j/hooks => providers/tests/openfaas}/__init__.py (100%) rename {tests/providers/neo4j/operators => providers/tests/openfaas/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/openfaas/hooks/test_openfaas.py (100%) rename {tests/providers/openlineage/extractors => providers/tests/openlineage}/__init__.py (100%) rename {tests/providers/openlineage/plugins => providers/tests/openlineage/extractors}/__init__.py (100%) rename {tests/providers => providers/tests}/openlineage/extractors/test_base.py (99%) rename {tests/providers => providers/tests}/openlineage/extractors/test_bash.py (98%) rename {tests/providers => providers/tests}/openlineage/extractors/test_manager.py (99%) rename {tests/providers => providers/tests}/openlineage/extractors/test_python.py (98%) rename {tests/providers => providers/tests}/openlineage/log_config.py (100%) rename {tests/providers/openlineage/plugins/openlineage_configs => providers/tests/openlineage/plugins}/__init__.py (100%) rename {tests/providers/openlineage/utils => providers/tests/openlineage/plugins/openlineage_configs}/__init__.py (100%) rename {tests/providers => providers/tests}/openlineage/plugins/openlineage_configs/http.yaml (100%) rename {tests/providers => providers/tests}/openlineage/plugins/test_adapter.py (99%) rename {tests/providers => providers/tests}/openlineage/plugins/test_execution.py (98%) rename {tests/providers => providers/tests}/openlineage/plugins/test_facets.py (100%) rename {tests/providers => providers/tests}/openlineage/plugins/test_listener.py (99%) rename {tests/providers => providers/tests}/openlineage/plugins/test_macros.py (100%) rename {tests/providers => providers/tests}/openlineage/plugins/test_openlineage.py (97%) rename {tests/providers => providers/tests}/openlineage/plugins/test_utils.py (99%) rename {tests/providers => providers/tests}/openlineage/test_conf.py (99%) rename {tests/providers => providers/tests}/openlineage/test_sqlparser.py (100%) rename {tests/providers/opensearch => providers/tests/openlineage/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/openlineage/utils/custom_facet_fixture.py (100%) rename {tests/providers => providers/tests}/openlineage/utils/test_selective_enable.py (100%) rename {tests/providers => providers/tests}/openlineage/utils/test_sql.py (100%) rename {tests/providers => providers/tests}/openlineage/utils/test_utils.py (96%) rename {tests/providers/opensearch/hooks => providers/tests/opensearch}/__init__.py (100%) rename {tests/providers => providers/tests}/opensearch/conftest.py (100%) rename {tests/providers/opensearch/log => providers/tests/opensearch/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/opensearch/hooks/test_opensearch.py (100%) rename {tests/providers/opensearch/operators => providers/tests/opensearch/log}/__init__.py (100%) rename {tests/providers => providers/tests}/opensearch/log/test_os_json_formatter.py (100%) rename {tests/providers => providers/tests}/opensearch/log/test_os_response.py (100%) rename {tests/providers => providers/tests}/opensearch/log/test_os_task_handler.py (99%) rename {tests/providers/opsgenie/notifications => providers/tests/opensearch/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/opensearch/operators/test_opensearch.py (100%) rename {tests/providers/openfaas => providers/tests/opsgenie}/__init__.py (100%) rename {tests/providers/openfaas => providers/tests/opsgenie}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/opsgenie/hooks/test_opsgenie.py (100%) rename {tests/providers/opsgenie/typing => providers/tests/opsgenie/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/opsgenie/notifications/test_opsgenie.py (100%) rename {tests/providers/opsgenie => providers/tests/opsgenie/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/opsgenie/operators/test_opsgenie.py (100%) rename {tests/providers/oracle/operators => providers/tests/opsgenie/typing}/__init__.py (100%) rename {tests/providers => providers/tests}/opsgenie/typing/test_opsgenie.py (100%) rename {tests/providers/opsgenie/hooks => providers/tests/oracle}/__init__.py (100%) rename {tests/providers/opsgenie/operators => providers/tests/oracle/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/oracle/hooks/test_oracle.py (100%) rename {tests/providers/pagerduty/notifications => providers/tests/oracle/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/oracle/operators/test_oracle.py (100%) rename {tests/providers/oracle => providers/tests/oracle/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/oracle/transfers/test_oracle_to_oracle.py (100%) rename {tests/providers/oracle/hooks => providers/tests/pagerduty}/__init__.py (100%) rename {tests/providers/oracle/transfers => providers/tests/pagerduty/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/pagerduty/hooks/test_pagerduty.py (100%) rename {tests/providers => providers/tests}/pagerduty/hooks/test_pagerduty_events.py (100%) rename {tests/providers/papermill/hooks => providers/tests/pagerduty/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/pagerduty/notifications/test_pagerduty.py (100%) rename {tests/providers/pagerduty => providers/tests/papermill}/__init__.py (100%) rename {tests/providers/pgvector => providers/tests/papermill/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/papermill/hooks/test_kernel.py (100%) rename {tests/providers/pagerduty/hooks => providers/tests/papermill/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/papermill/operators/test_papermill.py (100%) rename {tests/providers/pgvector/hooks => providers/tests/pgvector}/__init__.py (100%) rename {tests/providers/pgvector/operators => providers/tests/pgvector/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/pgvector/hooks/test_pgvector.py (100%) rename {tests/providers/pinecone => providers/tests/pgvector/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/pgvector/operators/test_pgvector.py (100%) rename {tests/providers/pinecone/hooks => providers/tests/pinecone}/__init__.py (100%) rename {tests/providers/pinecone/operators => providers/tests/pinecone/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/pinecone/hooks/test_pinecone.py (100%) rename {tests/providers/postgres/assets => providers/tests/pinecone/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/pinecone/operators/test_pinecone.py (100%) rename {tests/providers/papermill => providers/tests/postgres}/__init__.py (100%) rename {tests/providers/qdrant => providers/tests/postgres/assets}/__init__.py (100%) rename {tests/providers => providers/tests}/postgres/assets/test_postgres.py (100%) rename {tests/providers/papermill/operators => providers/tests/postgres/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/postgres/hooks/test_postgres.py (100%) rename {tests/providers/postgres => providers/tests/postgres/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/postgres/operators/test_postgres.py (100%) rename {tests/providers/postgres/hooks => providers/tests/presto}/__init__.py (100%) rename {tests/providers/postgres/operators => providers/tests/presto/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/presto/hooks/test_presto.py (100%) rename {tests/providers/presto => providers/tests/presto/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/presto/transfers/test_gcs_to_presto.py (100%) rename {tests/providers/qdrant/hooks => providers/tests/qdrant}/__init__.py (100%) rename {tests/providers/qdrant/operators => providers/tests/qdrant/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/qdrant/hooks/test_qdrant.py (100%) rename {tests/providers/salesforce => providers/tests/qdrant}/operators/__init__.py (100%) rename {tests/providers => providers/tests}/qdrant/operators/test_qdrant.py (100%) rename {tests/providers/presto/hooks => providers/tests/redis}/__init__.py (100%) rename {tests/providers/presto/transfers => providers/tests/redis/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/redis/hooks/test_redis.py (100%) rename {tests/providers/redis => providers/tests/redis/log}/__init__.py (100%) rename {tests/providers => providers/tests}/redis/log/test_redis_task_handler.py (98%) rename {tests/providers/redis/hooks => providers/tests/redis/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/redis/operators/test_redis_publish.py (100%) rename {tests/providers/redis/log => providers/tests/redis/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/redis/sensors/test_redis_key.py (100%) rename {tests/providers => providers/tests}/redis/sensors/test_redis_pub_sub.py (100%) rename {tests/providers/redis/operators => providers/tests/salesforce}/__init__.py (100%) rename {tests/providers/redis/sensors => providers/tests/salesforce/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/salesforce/hooks/test_salesforce.py (100%) rename {tests/providers/sendgrid => providers/tests/salesforce/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/salesforce/operators/test_bulk.py (100%) rename {tests/providers => providers/tests}/salesforce/operators/test_salesforce_apex_rest.py (100%) rename {tests/providers/salesforce => providers/tests/samba}/__init__.py (100%) rename {tests/providers/salesforce => providers/tests/samba}/hooks/__init__.py (100%) rename {tests/providers => providers/tests}/samba/hooks/test_samba.py (100%) rename {tests/providers/samba => providers/tests/samba/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/samba/transfers/test_gcs_to_samba.py (100%) rename {tests/providers/samba/hooks => providers/tests/segment}/__init__.py (100%) rename {tests/providers/samba/transfers => providers/tests/segment/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/segment/hooks/test_segment.py (100%) rename {tests/providers/segment => providers/tests/segment/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/segment/operators/test_segment_track_event.py (100%) rename {tests/providers/sendgrid/utils => providers/tests/sendgrid}/__init__.py (100%) rename {tests/providers/sftp => providers/tests/sendgrid/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/sendgrid/utils/test_emailer.py (100%) rename {tests/providers/sftp/decorators => providers/tests/sftp}/__init__.py (100%) rename {tests/providers/sftp/decorators/sensors => providers/tests/sftp/decorators}/__init__.py (100%) rename {tests/providers/sftp/hooks => providers/tests/sftp/decorators/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/sftp/decorators/sensors/test_sftp.py (100%) rename {tests/providers/sftp/operators => providers/tests/sftp/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/sftp/hooks/test_sftp.py (99%) rename {tests/providers/sftp/sensors => providers/tests/sftp/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/sftp/operators/test_sftp.py (97%) rename {tests/providers/sftp/triggers => providers/tests/sftp/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/sftp/sensors/test_sftp.py (100%) rename {tests/providers/slack/notifications => providers/tests/sftp/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/sftp/triggers/test_sftp.py (100%) rename {tests/providers/segment/hooks => providers/tests/singularity}/__init__.py (100%) rename {tests/providers/segment => providers/tests/singularity}/operators/__init__.py (100%) rename {tests/providers => providers/tests}/singularity/operators/test_singularity.py (100%) rename {tests/providers/singularity => providers/tests/slack}/__init__.py (100%) rename {tests/providers/singularity/operators => providers/tests/slack/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/slack/hooks/test_slack.py (100%) rename {tests/providers => providers/tests}/slack/hooks/test_slack_webhook.py (100%) rename {tests/providers/slack/utils => providers/tests/slack/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/slack/notifications/test_slack.py (100%) rename {tests/providers => providers/tests}/slack/notifications/test_slack_webhook.py (100%) rename {tests/providers/slack => providers/tests/slack/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/slack/operators/test_slack.py (98%) rename {tests/providers => providers/tests}/slack/operators/test_slack_webhook.py (100%) rename {tests/providers/slack/hooks => providers/tests/slack/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/slack/transfers/conftest.py (100%) rename {tests/providers => providers/tests}/slack/transfers/test_base_sql_to_slack.py (100%) rename {tests/providers => providers/tests}/slack/transfers/test_sql_to_slack.py (100%) rename {tests/providers => providers/tests}/slack/transfers/test_sql_to_slack_webhook.py (96%) rename {tests/providers/smtp/notifications => providers/tests/slack/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/slack/utils/test_utils.py (100%) rename {tests/providers/slack/operators => providers/tests/smtp}/__init__.py (100%) rename {tests/providers/slack/transfers => providers/tests/smtp/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/smtp/hooks/test_smtp.py (99%) rename {tests/providers/snowflake => providers/tests/smtp/notifications}/__init__.py (100%) rename {tests/providers => providers/tests}/smtp/notifications/test_smtp.py (96%) rename {tests/providers/smtp => providers/tests/smtp/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/smtp/operators/test_smtp.py (100%) rename {tests/providers/snowflake/decorators => providers/tests/snowflake}/__init__.py (100%) rename {tests/providers/snowflake/hooks => providers/tests/snowflake/decorators}/__init__.py (100%) rename {tests/providers => providers/tests}/snowflake/decorators/test_snowpark.py (100%) rename {tests/providers/snowflake/operators => providers/tests/snowflake/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/snowflake/hooks/test_snowflake.py (93%) rename {tests/providers => providers/tests}/snowflake/hooks/test_snowflake_sql_api.py (97%) rename {tests/providers => providers/tests}/snowflake/hooks/test_sql.py (100%) rename {tests/providers/snowflake/transfers => providers/tests/snowflake/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/snowflake/operators/test_snowflake.py (100%) rename {tests/providers => providers/tests}/snowflake/operators/test_snowflake_sql.py (100%) rename {tests/providers => providers/tests}/snowflake/operators/test_snowpark.py (100%) rename {tests/providers/snowflake/triggers => providers/tests/snowflake/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/snowflake/transfers/test_copy_into_snowflake.py (100%) rename {tests/providers/snowflake/utils => providers/tests/snowflake/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/snowflake/triggers/test_snowflake.py (100%) rename {tests/providers/standard => providers/tests/snowflake/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/snowflake/utils/test_common.py (100%) rename {tests/providers => providers/tests}/snowflake/utils/test_openlineage.py (100%) rename {tests/providers => providers/tests}/snowflake/utils/test_snowpark.py (100%) rename {tests/providers => providers/tests}/snowflake/utils/test_sql_api_generate_jwt.py (100%) rename {tests/providers/smtp/hooks => providers/tests/sqlite}/__init__.py (100%) rename {tests/providers/smtp/operators => providers/tests/sqlite/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/sqlite/hooks/test_sqlite.py (100%) rename {tests/providers/sqlite => providers/tests/sqlite/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/sqlite/operators/test_sqlite.py (100%) rename {tests/providers/sqlite/hooks => providers/tests/ssh}/__init__.py (100%) rename {tests/providers/sqlite/operators => providers/tests/ssh/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/ssh/hooks/test_ssh.py (99%) rename {tests/providers/ssh => providers/tests/ssh/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/ssh/operators/test_ssh.py (99%) rename {tests/providers/standard/operators => providers/tests/standard}/__init__.py (100%) rename {tests/providers/standard/sensors => providers/tests/standard/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/standard/operators/test_bash.py (99%) rename {tests/providers => providers/tests}/standard/operators/test_datetime.py (99%) rename {tests/providers => providers/tests}/standard/operators/test_weekday.py (99%) rename {tests/providers/tableau => providers/tests/standard/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/standard/sensors/test_bash.py (100%) rename {tests/providers => providers/tests}/standard/sensors/test_date_time.py (100%) rename {tests/providers => providers/tests}/standard/sensors/test_time.py (100%) rename {tests/providers => providers/tests}/standard/sensors/test_time_delta.py (100%) rename {tests/providers => providers/tests}/standard/sensors/test_weekday.py (99%) rename {tests/providers/tableau/operators => providers/tests/system}/__init__.py (100%) rename {tests/providers/tableau/sensors => providers/tests/system/airbyte}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/airbyte/example_airbyte_trigger_job.py (96%) rename {tests/providers/teradata => providers/tests/system/alibaba}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/alibaba/example_adb_spark_batch.py (93%) rename {tests/system/providers => providers/tests/system}/alibaba/example_adb_spark_sql.py (93%) rename {tests/system/providers => providers/tests/system}/alibaba/example_oss_bucket.py (93%) rename {tests/system/providers => providers/tests/system}/alibaba/example_oss_object.py (94%) rename {tests/system/providers => providers/tests/system}/amazon/CONTRIBUTING.md (99%) rename {tests/system/providers => providers/tests/system}/amazon/README.md (100%) rename {tests/providers/ssh/hooks => providers/tests/system/amazon}/__init__.py (100%) rename {tests/providers/ssh/operators => providers/tests/system/amazon/aws}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_appflow.py (94%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_appflow_run.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_athena.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_azure_blob_to_s3.py (93%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_batch.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_bedrock.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_bedrock_retrieve_and_generate.py (99%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_cloudformation.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_comprehend.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_comprehend_document_classifier.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_datasync.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_dms.py (98%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_dynamodb.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_dynamodb_to_s3.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_ec2.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_ecs.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_ecs_fargate.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_eks_templated.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_eks_with_fargate_in_one_step.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_eks_with_fargate_profile.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_eks_with_nodegroup_in_one_step.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_eks_with_nodegroups.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_emr.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_emr_eks.py (98%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_emr_notebook_execution.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_emr_serverless.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_eventbridge.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_ftp_to_s3.py (92%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_gcs_to_s3.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_glacier_to_gcs.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_glue.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_glue_data_quality.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_glue_data_quality_with_recommendation.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_glue_databrew.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_google_api_sheets_to_s3.py (94%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_google_api_youtube_to_s3.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_hive_to_dynamodb.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_http_to_s3.py (94%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_imap_attachment_to_s3.py (94%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_kinesis_analytics.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_lambda.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_local_to_s3.py (93%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_mongo_to_s3.py (93%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_neptune.py (93%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_quicksight.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_rds_event.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_rds_export.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_rds_instance.py (95%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_rds_snapshot.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_redshift.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_redshift_s3_transfers.py (98%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_s3.py (98%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_s3_to_dynamodb.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_s3_to_ftp.py (92%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_s3_to_sftp.py (92%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_s3_to_sql.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sagemaker.py (99%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sagemaker_endpoint.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sagemaker_notebook.py (94%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sagemaker_pipeline.py (97%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_salesforce_to_s3.py (93%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sftp_to_s3.py (92%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sns.py (92%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sql_to_s3.py (96%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_sqs.py (94%) rename {tests/system/providers => providers/tests/system}/amazon/aws/example_step_functions.py (95%) rename {tests/providers/trino/assets => providers/tests/system/amazon/aws/tests}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/amazon/aws/tests/test_aws_auth_manager.py (91%) rename {tests/system/providers => providers/tests/system}/amazon/aws/utils/__init__.py (99%) rename {tests/system/providers => providers/tests/system}/amazon/aws/utils/ec2.py (100%) rename {tests/system/providers => providers/tests/system}/amazon/aws/utils/k8s.py (100%) rename {tests/providers/trino/operators => providers/tests/system/apache}/__init__.py (100%) rename {tests/providers/tableau/hooks => providers/tests/system/apache/beam}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_beam.py (95%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_beam_java_flink.py (94%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_beam_java_spark.py (94%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_go.py (96%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_go_dataflow.py (95%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_java_dataflow.py (95%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_python.py (97%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_python_async.py (97%) rename {tests/system/providers => providers/tests/system}/apache/beam/example_python_dataflow.py (96%) rename {tests/system/providers => providers/tests/system}/apache/beam/utils.py (100%) rename {tests/providers/weaviate => providers/tests/system/apache/cassandra}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/cassandra/example_cassandra_dag.py (96%) rename {tests/providers/weaviate/hooks => providers/tests/system/apache/drill}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/drill/example_drill_dag.py (95%) rename {tests/providers/weaviate/operators => providers/tests/system/apache/druid}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/druid/example_druid_dag.py (96%) rename {tests/providers/telegram => providers/tests/system/apache/hive}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/hive/example_twitter_README.md (100%) rename {tests/system/providers => providers/tests/system}/apache/hive/example_twitter_dag.py (97%) rename {tests/providers/telegram/hooks => providers/tests/system/apache/iceberg}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/iceberg/example_iceberg.py (95%) rename {tests/providers/yandex => providers/tests/system/apache/kafka}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/kafka/example_dag_event_listener.py (97%) rename {tests/system/providers => providers/tests/system}/apache/kafka/example_dag_hello_kafka.py (98%) rename {tests/providers/yandex/hooks => providers/tests/system/apache/kylin}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/kylin/example_kylin_dag.py (96%) rename {tests/providers/yandex/links => providers/tests/system/apache/livy}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/livy/example_livy.py (95%) rename {tests/providers/yandex/operators => providers/tests/system/apache/pig}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/pig/example_pig.py (95%) rename {tests/providers/yandex/secrets => providers/tests/system/apache/pinot}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/pinot/example_pinot_dag.py (96%) rename {tests/providers/yandex/utils => providers/tests/system/apache/spark}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/apache/spark/example_pyspark.py (96%) rename {tests/system/providers => providers/tests/system}/apache/spark/example_spark_dag.py (97%) rename {tests/providers/ydb => providers/tests/system/asana}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/asana/example_asana.py (97%) rename {tests/providers/ydb/hooks => providers/tests/system/cncf}/__init__.py (100%) rename {tests/providers/ydb/operators => providers/tests/system/cncf/kubernetes}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/example_kubernetes.py (97%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/example_kubernetes_async.py (98%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/example_kubernetes_decorator.py (97%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/example_kubernetes_job.py (95%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/example_kubernetes_resource.py (94%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/example_spark_kubernetes.py (95%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml (100%) rename {tests/system/providers => providers/tests/system}/cncf/kubernetes/spark_job_template.yaml (100%) rename {tests/providers/ydb/utils => providers/tests/system/cohere}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/cohere/example_cohere_embedding_operator.py (96%) rename {tests/providers/zendesk => providers/tests/system/common}/__init__.py (100%) rename {tests/providers/zendesk/hooks => providers/tests/system/common/io}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/common/io/example_file_transfer_local_to_s3.py (94%) rename {tests/system/providers/airbyte => providers/tests/system/common/sql}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/common/sql/example_sql_column_table_check.py (97%) rename {tests/system/providers => providers/tests/system}/common/sql/example_sql_execute_query.py (96%) rename {tests/system/providers/alibaba => providers/tests/system/databricks}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/databricks/example_databricks.py (98%) rename {tests/system/providers => providers/tests/system}/databricks/example_databricks_repos.py (95%) rename {tests/system/providers => providers/tests/system}/databricks/example_databricks_sensors.py (96%) rename {tests/system/providers => providers/tests/system}/databricks/example_databricks_sql.py (97%) rename {tests/system/providers => providers/tests/system}/databricks/example_databricks_workflow.py (97%) rename {tests/system/providers/amazon/aws/tests => providers/tests/system/dbt}/__init__.py (100%) rename {tests/system/providers/apache/cassandra => providers/tests/system/dbt/cloud}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/dbt/cloud/example_dbt_cloud.py (94%) rename {tests/system/providers/apache/drill => providers/tests/system/dingding}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/dingding/example_dingding.py (98%) rename {tests/system/providers/apache/druid => providers/tests/system/docker}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/docker/example_docker.py (96%) rename {tests/system/providers => providers/tests/system}/docker/example_docker_copy_data.py (97%) rename {tests/system/providers => providers/tests/system}/docker/example_docker_swarm.py (95%) rename {tests/system/providers => providers/tests/system}/docker/example_taskflow_api_docker_virtualenv.py (97%) rename {tests/system/providers/apache/kafka => providers/tests/system/elasticsearch}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/elasticsearch/example_elasticsearch_query.py (97%) rename {tests/system/providers/apache/kylin => providers/tests/system/ftp}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/ftp/example_ftp.py (95%) rename {tests/system/providers/apache/livy => providers/tests/system/github}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/github/example_github.py (97%) rename {tests/system/providers => providers/tests/system}/google/README.md (100%) rename {tests/system/providers => providers/tests/system}/google/__init__.py (100%) rename {tests/system/providers/apache/pig => providers/tests/system/google/ads}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/ads/example_ads.py (95%) rename {tests/system/providers/apache/pinot => providers/tests/system/google/cloud}/__init__.py (100%) rename {tests/system/providers/apache/spark => providers/tests/system/google/cloud/automl}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/automl/example_automl_dataset.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/automl/example_automl_translation.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/automl/example_automl_video_classification.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/automl/example_automl_video_tracking.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/automl/example_automl_vision_classification.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/automl/example_automl_vision_object_detection.py (97%) rename {tests/system/providers/asana => providers/tests/system/google/cloud/automl/resources}/__init__.py (100%) rename {tests/system/providers/cncf => providers/tests/system/google/cloud/azure}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/azure/example_azure_blob_to_gcs.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/azure/example_azure_fileshare_to_gcs.py (93%) rename {tests/system/providers/cncf/kubernetes => providers/tests/system/google/cloud/bigquery}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_dataset.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_dts.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_operations.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_operations_location.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_queries.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_queries_async.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_sensors.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_tables.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_to_bigquery.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_to_gcs.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_to_gcs_async.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_to_mssql.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_to_mysql.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_to_postgres.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_transfer.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/example_bigquery_value_check.py (95%) rename {tests/system/providers/cohere => providers/tests/system/google/cloud/bigquery/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/resources/example_bigquery_query.sql (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/resources/update_table_schema.json (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigquery/resources/us-states.csv (100%) rename {tests/system/providers/common => providers/tests/system/google/cloud/bigtable}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/bigtable/example_bigtable.py (97%) rename {tests/system/providers/common/io => providers/tests/system/google/cloud/cloud_batch}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_batch/example_cloud_batch.py (96%) rename {tests/system/providers/common/sql => providers/tests/system/google/cloud/cloud_build}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_build/example_cloud_build.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_build/example_cloud_build_trigger.py (96%) rename {tests/system/providers/databricks => providers/tests/system/google/cloud/cloud_build/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_build/resources/example_cloud_build.yaml (100%) rename {tests/system/providers/dbt/cloud => providers/tests/system/google/cloud/cloud_functions}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_functions/example_functions.py (95%) rename {tests/system/providers/dingding => providers/tests/system/google/cloud/cloud_memorystore}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py (97%) rename {tests/system/providers/docker => providers/tests/system/google/cloud/cloud_run}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_run/example_cloud_run.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_run/example_cloud_run_service.py (95%) rename {tests/system/providers/elasticsearch => providers/tests/system/google/cloud/cloud_sql}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_sql/example_cloud_sql.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_sql/example_cloud_sql_query.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py (98%) rename {tests/system/providers/ftp => providers/tests/system/google/cloud/composer}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/composer/example_cloud_composer.py (98%) rename {tests/system/providers/github => providers/tests/system/google/cloud/compute}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/compute/example_compute.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/compute/example_compute_igm.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/compute/example_compute_ssh.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/compute/example_compute_ssh_os_login.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/compute/example_compute_ssh_parallel.py (96%) rename {tests/system/providers/google/ads => providers/tests/system/google/cloud/data_loss_prevention}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/data_loss_prevention/example_dlp_info_types.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/data_loss_prevention/example_dlp_inspect_template.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/data_loss_prevention/example_dlp_job.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/data_loss_prevention/example_dlp_job_trigger.py (94%) rename {tests/system/providers/google/cloud => providers/tests/system/google/cloud/data_loss_prevention/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/data_loss_prevention/resources/dictionary.txt (100%) rename {tests/system/providers/google/cloud/automl => providers/tests/system/google/cloud/dataflow}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_go.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_native_java.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_native_python.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_native_python_async.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_pipeline.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_sensors_deferrable.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_sql.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_streaming_python.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_template.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/example_dataflow_yaml.py (96%) rename {tests/system/providers/google/cloud/automl => providers/tests/system/google/cloud/dataflow}/resources/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/resources/input.csv (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/resources/schema.json (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/resources/text.txt (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataflow/resources/wordcount.go (100%) rename {tests/system/providers/google/cloud/azure => providers/tests/system/google/cloud/dataform}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataform/example_dataform.py (98%) rename {tests/system/providers/google/cloud/bigquery => providers/tests/system/google/cloud/datafusion}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/datafusion/example_datafusion.py (98%) rename {tests/system/providers/google/cloud/bigquery/resources => providers/tests/system/google/cloud/datapipelines}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/datapipelines/example_datapipeline.py (96%) rename {tests/system/providers/google/cloud/bigtable => providers/tests/system/google/cloud/dataplex}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataplex/example_dataplex.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataplex/example_dataplex_dp.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataplex/example_dataplex_dq.py (98%) rename {tests/system/providers/google/cloud/cloud_batch => providers/tests/system/google/cloud/dataprep}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataprep/example_dataprep.py (98%) rename {tests/system/providers/google/cloud/cloud_build => providers/tests/system/google/cloud/dataproc}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_batch.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_batch_deferrable.py (93%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_batch_persistent.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_cluster_deferrable.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_cluster_diagnose.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_cluster_generator.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_cluster_start_stop.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_cluster_update.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_flink.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_gke.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_hadoop.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_hive.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_pig.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_presto.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_pyspark.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_spark.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_spark_async.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_spark_deferrable.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_spark_sql.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_sparkr.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_trino.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_workflow.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc/example_dataproc_workflow_deferrable.py (95%) rename {tests/system/providers/google/cloud/cloud_build/resources => providers/tests/system/google/cloud/dataproc_metastore}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc_metastore/example_dataproc_metastore.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py (97%) rename {tests/system/providers/google/cloud/cloud_functions => providers/tests/system/google/cloud/datastore}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/datastore/example_datastore_commit.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/datastore/example_datastore_query.py (93%) rename {tests/system/providers => providers/tests/system}/google/cloud/datastore/example_datastore_rollback.py (91%) rename {tests/system/providers/google/cloud/cloud_memorystore => providers/tests/system/google/cloud/gcs}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_calendar_to_gcs.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_firestore.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_acl.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_copy_delete.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_sensor.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_to_bigquery.py (93%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_to_bigquery_async.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_to_gcs.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_to_gdrive.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_to_sheets.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_transform.py (93%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_transform_timespan.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gcs_upload_download.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_gdrive_to_gcs.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_mssql_to_gcs.py (93%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_mysql_to_gcs.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_oracle_to_gcs.py (92%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_s3_to_gcs.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_sftp_to_gcs.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_sheets.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_sheets_to_gcs.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/example_trino_to_gcs.py (98%) rename {tests/system/providers/google/cloud/cloud_run => providers/tests/system/google/cloud/gcs/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/resources/example_upload.txt (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/resources/tmp.tar.gz (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/resources/transform_script.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/resources/transform_timespan.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/gcs/resources/us-states.csv (100%) rename {tests/system/providers/google/cloud/cloud_sql => providers/tests/system/google/cloud/kubernetes_engine}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/kubernetes_engine/example_kubernetes_engine.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py (95%) rename {tests/system/providers => providers/tests/system}/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py (96%) rename {tests/system/providers/google/cloud/composer => providers/tests/system/google/cloud/life_sciences}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/life_sciences/example_life_sciences.py (95%) rename {tests/system/providers/google/cloud/compute => providers/tests/system/google/cloud/life_sciences/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/life_sciences/resources/file (100%) rename {tests/system/providers/google/cloud/data_loss_prevention => providers/tests/system/google/cloud/ml_engine}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/ml_engine/example_mlengine.py (98%) rename {tests/system/providers/google/cloud/data_loss_prevention/resources => providers/tests/system/google/cloud/natural_language}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/natural_language/example_natural_language.py (97%) rename {tests/system/providers/google/cloud/dataflow => providers/tests/system/google/cloud/pubsub}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/pubsub/example_pubsub.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/pubsub/example_pubsub_deferrable.py (96%) rename {tests/system/providers/google/cloud/dataflow/resources => providers/tests/system/google/cloud/spanner}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/spanner/example_spanner.py (96%) rename {tests/system/providers/google/cloud/dataform => providers/tests/system/google/cloud/speech_to_text}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/speech_to_text/example_speech_to_text.py (94%) rename {tests/system/providers/google/cloud/datafusion => providers/tests/system/google/cloud/sql_to_sheets}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/sql_to_sheets/example_sql_to_sheets.py (98%) rename {tests/system/providers/google/cloud/datapipelines => providers/tests/system/google/cloud/stackdriver}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/stackdriver/example_stackdriver.py (97%) rename {tests/system/providers/google/cloud/dataplex => providers/tests/system/google/cloud/storage_transfer}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py (96%) rename {tests/system/providers/google/cloud/dataprep => providers/tests/system/google/cloud/storage_transfer/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/storage_transfer/resources/transfer_service_gcp_file (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/storage_transfer/resources/transfer_service_gcs_to_gcs_file (100%) rename {tests/system/providers/google/cloud/dataproc => providers/tests/system/google/cloud/tasks}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/tasks/example_queue.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/tasks/example_tasks.py (97%) rename {tests/system/providers/google/cloud/dataproc_metastore => providers/tests/system/google/cloud/text_to_speech}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/text_to_speech/example_text_to_speech.py (93%) rename {tests/system/providers/google/cloud/datastore => providers/tests/system/google/cloud/transfers}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/transfers/example_gcs_to_sftp.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/transfers/example_gdrive_to_local.py (96%) rename {tests/system/providers => providers/tests/system}/google/cloud/transfers/example_postgres_to_gcs.py (98%) rename {tests/system/providers/google/cloud/gcs => providers/tests/system/google/cloud/transfers/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/transfers/resources/empty.txt (100%) rename {tests/system/providers/google/cloud/gcs/resources => providers/tests/system/google/cloud/translate}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/translate/example_translate.py (94%) rename {tests/system/providers/google/cloud/kubernetes_engine => providers/tests/system/google/cloud/translate_speech}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/translate_speech/example_translate_speech.py (96%) rename {tests/system/providers/google/cloud/life_sciences => providers/tests/system/google/cloud/vertex_ai}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py (93%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_custom_container.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_custom_job.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_dataset.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_endpoint.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_generative_model.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py (94%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py (93%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_model_service.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py (97%) rename {tests/system/providers/google/cloud/life_sciences/resources => providers/tests/system/google/cloud/video_intelligence}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/video_intelligence/example_video_intelligence.py (97%) rename {tests/system/providers/google/cloud/ml_engine => providers/tests/system/google/cloud/vision}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/vision/example_vision_annotate_image.py (97%) rename {tests/system/providers => providers/tests/system}/google/cloud/vision/example_vision_autogenerated.py (98%) rename {tests/system/providers => providers/tests/system}/google/cloud/vision/example_vision_explicit.py (98%) rename {tests/system/providers/google/cloud/natural_language => providers/tests/system/google/cloud/workflows}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/cloud/workflows/example_workflows.py (98%) rename {tests/system/providers => providers/tests/system}/google/conftest.py (100%) rename {tests/system/providers/google/cloud/pubsub => providers/tests/system/google/datacatalog}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/datacatalog/example_datacatalog_entries.py (97%) rename {tests/system/providers => providers/tests/system}/google/datacatalog/example_datacatalog_search_catalog.py (97%) rename {tests/system/providers => providers/tests/system}/google/datacatalog/example_datacatalog_tag_templates.py (97%) rename {tests/system/providers => providers/tests/system}/google/datacatalog/example_datacatalog_tags.py (97%) rename {tests/system/providers/google/cloud/spanner => providers/tests/system/google/firebase}/__init__.py (100%) rename {tests/providers/telegram/operators => providers/tests/system/google/leveldb}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/leveldb/example_leveldb.py (94%) rename {tests/providers/teradata/hooks => providers/tests/system/google/marketing_platform}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/marketing_platform/example_analytics_admin.py (98%) rename {tests/system/providers => providers/tests/system}/google/marketing_platform/example_campaign_manager.py (98%) rename {tests/system/providers => providers/tests/system}/google/marketing_platform/example_search_ads.py (97%) rename {tests/system/providers/google/cloud/speech_to_text => providers/tests/system/google/suite}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/suite/example_local_to_drive.py (97%) rename {tests/system/providers/google/cloud/sql_to_sheets => providers/tests/system/google/suite/resources}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/google/suite/resources/test1 (100%) rename {tests/system/providers => providers/tests/system}/google/suite/resources/test2 (100%) rename {tests/system/providers/google/cloud/stackdriver => providers/tests/system/google/workplace}/__init__.py (100%) rename {tests/system/providers/google/cloud/storage_transfer => providers/tests/system/http}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/http/example_http.py (98%) rename {tests/system/providers/google/cloud/storage_transfer/resources => providers/tests/system/influxdb}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/influxdb/example_influxdb.py (94%) rename {tests/system/providers => providers/tests/system}/influxdb/example_influxdb_query.py (95%) rename {tests/system/providers/google/cloud/tasks => providers/tests/system/jdbc}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/jdbc/example_jdbc_queries.py (94%) rename {tests/system/providers/google/cloud/text_to_speech => providers/tests/system/jenkins}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/jenkins/example_jenkins_job_trigger.py (97%) rename {tests/system/providers/google/cloud/transfers => providers/tests/system/microsoft}/__init__.py (100%) rename {tests/providers/teradata/operators => providers/tests/system/microsoft/azure}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_adf_run_pipeline.py (96%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_adls_create.py (93%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_adls_delete.py (93%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_adls_list.py (92%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_azure_batch_operator.py (96%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_azure_container_instances.py (97%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_azure_cosmosdb.py (94%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_azure_service_bus.py (98%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_azure_synapse.py (97%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_fileshare.py (93%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_local_to_adls.py (93%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_local_to_wasb.py (94%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_msfabric.py (94%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_msgraph.py (93%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_powerbi.py (96%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_powerbi_dataset_refresh.py (95%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_s3_to_wasb.py (94%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_sftp_to_wasb.py (95%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_synapse_run_pipeline.py (93%) rename {tests/system/providers => providers/tests/system}/microsoft/azure/example_wasb_sensors.py (96%) rename {tests/system/providers/google/cloud/transfers/resources => providers/tests/system/microsoft/mssql}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/microsoft/mssql/create_table.sql (100%) rename {tests/system/providers => providers/tests/system}/microsoft/mssql/example_mssql.py (97%) rename {tests/system/providers/google/cloud/translate => providers/tests/system/microsoft/winrm}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/microsoft/winrm/example_winrm.py (95%) rename {tests/system/providers/google/cloud/translate_speech => providers/tests/system/mysql}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/mysql/example_mysql.py (95%) rename {tests/system/providers/google/cloud/vertex_ai => providers/tests/system/neo4j}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/neo4j/example_neo4j.py (95%) rename {tests/system/providers/google/cloud/video_intelligence => providers/tests/system/openai}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/openai/example_openai.py (97%) rename {tests/system/providers => providers/tests/system}/openai/example_trigger_batch_operator.py (97%) rename {tests/system/providers/google/cloud/vision => providers/tests/system/opensearch}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/opensearch/example_opensearch.py (96%) rename {tests/system/providers/google/cloud/workflows => providers/tests/system/opsgenie}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/opsgenie/example_opsgenie_alert.py (96%) rename {tests/system/providers => providers/tests/system}/opsgenie/example_opsgenie_notifier.py (96%) rename {tests/providers/teradata/transfers => providers/tests/system/papermill}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/papermill/conftest.py (100%) rename {tests/system/providers => providers/tests/system}/papermill/example_papermill.py (96%) rename {tests/system/providers => providers/tests/system}/papermill/example_papermill_remote_verify.py (97%) rename {tests/system/providers => providers/tests/system}/papermill/example_papermill_verify.py (97%) rename {tests/system/providers => providers/tests/system}/papermill/input_notebook.ipynb (100%) rename {tests/system/providers/google/datacatalog => providers/tests/system/pgvector}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/pgvector/example_pgvector.py (97%) rename {tests/system/providers => providers/tests/system}/pgvector/example_pgvector_openai.py (97%) rename {tests/system/providers/google/firebase => providers/tests/system/pinecone}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/pinecone/example_create_pod_index.py (96%) rename {tests/system/providers => providers/tests/system}/pinecone/example_create_serverless_index.py (96%) rename {tests/system/providers => providers/tests/system}/pinecone/example_dag_pinecone.py (95%) rename {tests/system/providers => providers/tests/system}/pinecone/example_pinecone_cohere.py (97%) rename {tests/system/providers => providers/tests/system}/pinecone/example_pinecone_openai.py (98%) rename {tests/system/providers/google/suite => providers/tests/system/postgres}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/postgres/example_postgres.py (96%) rename {tests/system/providers/google/suite/resources => providers/tests/system/presto}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/presto/example_gcs_to_presto.py (95%) rename {tests/system/providers/google/workplace => providers/tests/system/qdrant}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/qdrant/example_dag_qdrant.py (96%) rename {tests/system/providers/http => providers/tests/system/redis}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/redis/example_redis_publish.py (95%) rename {tests/system/providers/influxdb => providers/tests/system/salesforce}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/salesforce/example_bulk.py (97%) rename {tests/system/providers => providers/tests/system}/salesforce/example_salesforce_apex_rest.py (95%) rename {tests/system/providers/jdbc => providers/tests/system/samba}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/samba/example_gcs_to_samba.py (96%) rename {tests/system/providers/jenkins => providers/tests/system/sftp}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/sftp/example_sftp_sensor.py (96%) rename {tests/system/providers/microsoft/mssql => providers/tests/system/singularity}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/singularity/example_singularity.py (95%) rename {tests/system/providers/microsoft/winrm => providers/tests/system/slack}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/slack/example_slack.py (97%) rename {tests/system/providers => providers/tests/system}/slack/example_slack_webhook.py (97%) rename {tests/system/providers => providers/tests/system}/slack/example_sql_to_slack.py (96%) rename {tests/system/providers => providers/tests/system}/slack/example_sql_to_slack_webhook.py (95%) rename {tests/system/providers/mysql => providers/tests/system/snowflake}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/snowflake/example_copy_into_snowflake.py (96%) rename {tests/system/providers => providers/tests/system}/snowflake/example_snowflake.py (97%) rename {tests/system/providers => providers/tests/system}/snowflake/example_snowflake_snowflake_op_template_file.sql (100%) rename {tests/system/providers => providers/tests/system}/snowflake/example_snowpark_decorator.py (97%) rename {tests/system/providers => providers/tests/system}/snowflake/example_snowpark_operator.py (97%) rename {tests/system/providers/neo4j => providers/tests/system/sqlite}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/sqlite/create_table.sql (100%) rename {tests/system/providers => providers/tests/system}/sqlite/example_sqlite.py (96%) rename {tests/system/providers/openai => providers/tests/system/tableau}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/tableau/example_tableau.py (97%) rename {tests/system/providers/opensearch => providers/tests/system/telegram}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/telegram/example_telegram.py (95%) rename {tests/system/providers/opsgenie => providers/tests/system/teradata}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/teradata/create_ssl_table.sql (100%) rename {tests/system/providers => providers/tests/system}/teradata/create_table.sql (100%) rename {tests/system/providers => providers/tests/system}/teradata/example_azure_blob_to_teradata_transfer.py (98%) rename {tests/system/providers => providers/tests/system}/teradata/example_s3_to_teradata_transfer.py (98%) rename {tests/system/providers => providers/tests/system}/teradata/example_ssl_teradata.py (97%) rename {tests/system/providers => providers/tests/system}/teradata/example_teradata.py (97%) rename {tests/system/providers => providers/tests/system}/teradata/example_teradata_call_sp.py (97%) rename {tests/system/providers => providers/tests/system}/teradata/example_teradata_compute_cluster.py (97%) rename {tests/system/providers => providers/tests/system}/teradata/example_teradata_to_teradata_transfer.py (97%) rename {tests/system/providers/pgvector => providers/tests/system/trino}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/trino/example_gcs_to_trino.py (95%) rename {tests/system/providers => providers/tests/system}/trino/example_trino.py (97%) rename {tests/system/providers/pinecone => providers/tests/system/weaviate}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/weaviate/example_weaviate_cohere.py (98%) rename {tests/system/providers => providers/tests/system}/weaviate/example_weaviate_dynamic_mapping_dag.py (97%) rename {tests/system/providers => providers/tests/system}/weaviate/example_weaviate_openai.py (98%) rename {tests/system/providers => providers/tests/system}/weaviate/example_weaviate_operator.py (99%) rename {tests/system/providers => providers/tests/system}/weaviate/example_weaviate_using_hook.py (98%) rename {tests/system/providers => providers/tests/system}/weaviate/example_weaviate_vectorizer_dag.py (97%) rename {tests/system/providers => providers/tests/system}/weaviate/example_weaviate_without_vectorizer_dag.py (98%) rename {tests/system/providers => providers/tests/system}/weaviate/jeopardy_data_with_vectors.json (100%) rename {tests/system/providers => providers/tests/system}/weaviate/jeopardy_data_without_vectors.json (100%) rename {tests/system/providers => providers/tests/system}/weaviate/jeopardy_doc_data_without_vectors.json (100%) rename {tests/system/providers/postgres => providers/tests/system/yandex}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/yandex/example_yandexcloud.py (97%) rename {tests/system/providers => providers/tests/system}/yandex/example_yandexcloud_dataproc.py (96%) rename {tests/system/providers => providers/tests/system}/yandex/example_yandexcloud_dataproc_lightweight.py (92%) rename {tests/system/providers => providers/tests/system}/yandex/example_yandexcloud_yq.py (88%) rename {tests/system/providers/presto => providers/tests/system/ydb}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/ydb/example_ydb.py (97%) rename {tests/system/providers/qdrant => providers/tests/system/zendesk}/__init__.py (100%) rename {tests/system/providers => providers/tests/system}/zendesk/example_zendesk_custom_get.py (95%) rename {tests/system/providers/redis => providers/tests/tableau}/__init__.py (100%) rename {tests/providers/teradata/triggers => providers/tests/tableau/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/tableau/hooks/test_tableau.py (95%) rename {tests/system/providers/salesforce => providers/tests/tableau/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/tableau/operators/test_tableau.py (100%) rename {tests/system/providers/samba => providers/tests/tableau/sensors}/__init__.py (100%) rename {tests/providers => providers/tests}/tableau/sensors/test_tableau.py (100%) rename {tests/providers/teradata/utils => providers/tests/telegram}/__init__.py (100%) rename {tests/providers/trino => providers/tests/telegram/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/telegram/hooks/test_telegram.py (100%) rename {tests/providers/trino/hooks => providers/tests/telegram/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/telegram/operators/test_telegram.py (100%) rename {tests/system/providers/sftp => providers/tests/teradata}/__init__.py (100%) rename {tests/providers/trino/transfers => providers/tests/teradata/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/teradata/hooks/test_teradata.py (98%) rename {tests/providers/vertica => providers/tests/teradata/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/teradata/operators/test_teradata.py (100%) rename {tests/providers => providers/tests}/teradata/operators/test_teradata_compute_cluster.py (100%) rename {tests/providers/vertica/hooks => providers/tests/teradata/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/teradata/transfers/test_azure_blob_to_teradata.py (100%) rename {tests/providers => providers/tests}/teradata/transfers/test_s3_to_teradata.py (100%) rename {tests/providers => providers/tests}/teradata/transfers/test_teradata_to_teradata.py (100%) rename {tests/providers/vertica/operators => providers/tests/teradata/triggers}/__init__.py (100%) rename {tests/providers => providers/tests}/teradata/triggers/test_teradata_compute_cluster.py (100%) rename {tests/system/providers => providers/tests/teradata/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/teradata/utils/test_constants.py (100%) rename {tests/system/providers/amazon => providers/tests/trino}/__init__.py (100%) rename {tests/system/providers/singularity => providers/tests/trino/assets}/__init__.py (100%) rename {tests/providers => providers/tests}/trino/assets/test_trino.py (100%) rename {tests/system/providers/amazon/aws => providers/tests/trino/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/trino/hooks/test_trino.py (100%) rename {tests/system/providers/slack => providers/tests/trino/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/trino/operators/test_trino.py (100%) rename {tests/system/providers/apache => providers/tests/trino/transfers}/__init__.py (100%) rename {tests/providers => providers/tests}/trino/transfers/test_gcs_to_trino.py (100%) rename {tests/system/providers/apache/beam => providers/tests/vertica}/__init__.py (100%) rename {tests/system/providers/apache/hive => providers/tests/vertica/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/vertica/hooks/test_vertica.py (100%) rename {tests/system/providers/apache/iceberg => providers/tests/vertica/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/vertica/operators/test_vertica.py (100%) rename {tests/system/providers/snowflake => providers/tests/weaviate}/__init__.py (100%) rename {tests/system/providers/sqlite => providers/tests/weaviate/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/weaviate/hooks/test_weaviate.py (100%) rename {tests/system/providers/tableau => providers/tests/weaviate/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/weaviate/operators/test_weaviate.py (100%) rename {tests/system/providers/telegram => providers/tests/yandex}/__init__.py (100%) rename {tests/system/providers/teradata => providers/tests/yandex/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/yandex/hooks/test_dataproc.py (100%) rename {tests/providers => providers/tests}/yandex/hooks/test_yandex.py (91%) rename {tests/providers => providers/tests}/yandex/hooks/test_yq.py (100%) rename {tests/system/providers/trino => providers/tests/yandex/links}/__init__.py (100%) rename {tests/providers => providers/tests}/yandex/links/test_yq.py (94%) rename {tests/system/providers/weaviate => providers/tests/yandex/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/yandex/operators/test_dataproc.py (100%) rename {tests/providers => providers/tests}/yandex/operators/test_yq.py (98%) rename {tests/system/providers/yandex => providers/tests/yandex/secrets}/__init__.py (100%) rename {tests/providers => providers/tests}/yandex/secrets/test_lockbox.py (100%) rename {tests/system/providers/ydb => providers/tests/yandex/utils}/__init__.py (100%) rename {tests/providers => providers/tests}/yandex/utils/test_credentials.py (100%) rename {tests/providers => providers/tests}/yandex/utils/test_defaults.py (100%) rename {tests/providers => providers/tests}/yandex/utils/test_fields.py (100%) rename {tests/providers => providers/tests}/yandex/utils/test_user_agent.py (100%) rename {tests/system/providers/zendesk => providers/tests/ydb}/__init__.py (100%) rename {tests/test_utils/operators => providers/tests/ydb/hooks}/__init__.py (100%) rename {tests/providers => providers/tests}/ydb/hooks/test_ydb.py (100%) rename {tests/test_utils/perf => providers/tests/ydb/operators}/__init__.py (100%) rename {tests/providers => providers/tests}/ydb/operators/test_ydb.py (100%) rename tests/providers/ydb/utils/test_defaults.py => providers/tests/ydb/utils/__init__.py (100%) rename {tests/providers => providers/tests}/ydb/utils/test_credentials.py (100%) rename tests/system/providers/google/leveldb/__init__.py => providers/tests/ydb/utils/test_defaults.py (99%) rename {tests/system/providers/microsoft => providers/tests/zendesk}/__init__.py (99%) rename {tests/system/providers/dbt => providers/tests/zendesk/hooks}/__init__.py (99%) rename {tests/providers => providers/tests}/zendesk/hooks/test_zendesk.py (100%) delete mode 100644 tests/system/providers/microsoft/azure/__init__.py delete mode 100644 tests/system/providers/papermill/__init__.py diff --git a/.dockerignore b/.dockerignore index 31ef8bb9ac260..bdbf5fb0883af 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,6 +34,7 @@ !chart !docs !licenses +!providers/ # Add those folders to the context so that they are available in the CI container !scripts diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index be62d541f0dea..019ea900f8e12 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -19,532 +19,532 @@ labelPRBasedOnFilePath: provider:airbyte: - - airflow/providers/airbyte/**/* + - providers/src/airflow/providers/airbyte/**/* - docs/apache-airflow-providers-airbyte/**/* - - tests/providers/airbyte/**/* - - tests/system/providers/airbyte/**/* + - providers/tests/airbyte/**/* + - providers/tests/system/airbyte/**/* provider:alibaba: - - airflow/providers/alibaba/**/* + - providers/src/airflow/providers/alibaba/**/* - docs/apache-airflow-providers-alibaba/**/* - - tests/providers/alibaba/**/* - - tests/system/providers/alibaba/**/* + - providers/tests/alibaba/**/* + - providers/tests/system/alibaba/**/* provider:amazon-aws: - - airflow/providers/amazon/aws/**/* - - tests/providers/amazon/aws/**/* + - providers/src/airflow/providers/amazon/aws/**/* + - providers/tests/amazon/aws/**/* - docs/apache-airflow-providers-amazon/**/* - - tests/system/providers/amazon/aws/**/* + - providers/tests/system/amazon/aws/**/* provider:apache-beam: - - airflow/providers/apache/beam/**/* + - providers/src/airflow/providers/apache/beam/**/* - docs/apache-airflow-providers-apache-beam/**/* - - tests/providers/apache/beam/**/* - - tests/system/providers/apache/beam/**/* + - providers/tests/apache/beam/**/* + - providers/tests/system/apache/beam/**/* provider:apache-cassandra: - - airflow/providers/apache/cassandra/**/* + - providers/src/airflow/providers/apache/cassandra/**/* - docs/apache-airflow-providers-apache-cassandra/**/* - - tests/providers/apache/cassandra/**/* - - tests/system/providers/apache/cassandra/**/* + - providers/tests/apache/cassandra/**/* + - providers/tests/system/apache/cassandra/**/* provider:apache-drill: - - airflow/providers/apache/drill/**/* + - providers/src/airflow/providers/apache/drill/**/* - docs/apache-airflow-providers-apache-drill/**/* - - tests/providers/apache/drill/**/* - - tests/system/providers/apache/drill/**/* + - providers/tests/apache/drill/**/* + - providers/tests/system/apache/drill/**/* provider:apache-druid: - - airflow/providers/apache/druid/**/* + - providers/src/airflow/providers/apache/druid/**/* - docs/apache-airflow-providers-apache-druid/**/* - - tests/providers/apache/druid/**/* - - tests/system/providers/apache/druid/**/* + - providers/tests/apache/druid/**/* + - providers/tests/system/apache/druid/**/* provider:apache-flink: - - airflow/providers/apache/flink/**/* + - providers/src/airflow/providers/apache/flink/**/* - docs/apache-airflow-providers-apache-flink/**/* - - tests/providers/apache/flink/**/* + - providers/tests/apache/flink/**/* provider:apache-hdfs: - - airflow/providers/apache/hdfs/**/* + - providers/src/airflow/providers/apache/hdfs/**/* - docs/apache-airflow-providers-apache-hdfs/**/* - - tests/providers/apache/hdfs/**/* + - providers/tests/apache/hdfs/**/* provider:apache-hive: - - airflow/providers/apache/hive/**/* + - providers/src/airflow/providers/apache/hive/**/* - docs/apache-airflow-providers-apache-hive/**/* - - tests/providers/apache/hive/**/* - - tests/system/providers/apache/hive/**/* + - providers/tests/apache/hive/**/* + - providers/tests/system/apache/hive/**/* provider:apache-iceberg: - - airflow/providers/apache/iceberg/**/* + - providers/src/airflow/providers/apache/iceberg/**/* - docs/apache-airflow-providers-apache-iceberg/**/* - - tests/providers/apache/iceberg/**/* - - tests/system/providers/apache/iceberg/**/* + - providers/tests/apache/iceberg/**/* + - providers/tests/system/apache/iceberg/**/* provider:apache-impala: - - airflow/providers/apache/impala/**/* + - providers/src/airflow/providers/apache/impala/**/* - docs/apache-airflow-providers-apache-impala/**/* - - tests/providers/apache/impala/**/* + - providers/tests/apache/impala/**/* provider:apache-kafka: - - airflow/providers/apache/kafka/**/* + - providers/src/airflow/providers/apache/kafka/**/* - docs/apache-airflow-providers-apache-kafka/**/* - - tests/providers/apache/kafka/**/* - - tests/system/providers/apache/kafka/**/* + - providers/tests/apache/kafka/**/* + - providers/tests/system/apache/kafka/**/* provider:apache-kylin: - - airflow/providers/apache/kylin/**/* + - providers/src/airflow/providers/apache/kylin/**/* - docs/apache-airflow-providers-apache-kylin/**/* - - tests/providers/apache/kylin/**/* - - tests/system/providers/apache/kylin/**/* + - providers/tests/apache/kylin/**/* + - providers/tests/system/apache/kylin/**/* provider:apache-livy: - - airflow/providers/apache/livy/**/* + - providers/src/airflow/providers/apache/livy/**/* - docs/apache-airflow-providers-apache-livy/**/* - - tests/providers/apache/livy/**/* - - tests/system/providers/apache/livy/**/* + - providers/tests/apache/livy/**/* + - providers/tests/system/apache/livy/**/* provider:apache-pig: - - airflow/providers/apache/pig/**/* + - providers/src/airflow/providers/apache/pig/**/* - docs/apache-airflow-providers-apache-pig/**/* - - tests/providers/apache/pig/**/* - - tests/system/providers/apache/pig/**/* + - providers/tests/apache/pig/**/* + - providers/tests/system/apache/pig/**/* provider:apache-pinot: - - airflow/providers/apache/pinot/**/* + - providers/src/airflow/providers/apache/pinot/**/* - docs/apache-airflow-providers-apache-pinot/**/* - - tests/providers/apache/pinot/**/* - - tests/system/providers/apache/pinot/**/* + - providers/tests/apache/pinot/**/* + - providers/tests/system/apache/pinot/**/* provider:apache-spark: - - airflow/providers/apache/spark/**/* + - providers/src/airflow/providers/apache/spark/**/* - docs/apache-airflow-providers-apache-spark/**/* - - tests/providers/apache/spark/**/* - - tests/system/providers/apache/spark/**/* + - providers/tests/apache/spark/**/* + - providers/tests/system/apache/spark/**/* provider:apprise: - - airflow/providers/apprise/**/* + - providers/src/airflow/providers/apprise/**/* - docs/apache-airflow-providers-apprise/**/* - - tests/providers/apprise/**/* + - providers/tests/apprise/**/* provider:arangodb: - - airflow/providers/arangodb/**/* + - providers/src/airflow/providers/arangodb/**/* - docs/apache-airflow-providers-arangodb/**/* - - tests/providers/arangodb/**/* + - providers/tests/arangodb/**/* provider:asana: - - airflow/providers/asana/**/* + - providers/src/airflow/providers/asana/**/* - docs/apache-airflow-providers-asana/**/* - - tests/providers/asana/**/* - - tests/system/providers/asana/**/* + - providers/tests/asana/**/* + - providers/tests/system/asana/**/* provider:atlassian-jira: - - airflow/providers/atlassian/jira/**/* + - providers/src/airflow/providers/atlassian/jira/**/* - docs/apache-airflow-providers-atlassian-jira/**/* - - tests/providers/atlassian/jira/**/* + - providers/tests/atlassian/jira/**/* provider:celery: - - airflow/providers/celery/**/* + - providers/src/airflow/providers/celery/**/* - docs/apache-airflow-providers-celery/**/* - - tests/providers/celery/**/* + - providers/tests/celery/**/* provider:cloudant: - - airflow/providers/cloudant/**/* + - providers/src/airflow/providers/cloudant/**/* - docs/apache-airflow-providers-cloudant/**/* - - tests/providers/cloudant/**/* + - providers/tests/cloudant/**/* provider:cncf-kubernetes: - airflow/**/kubernetes_*.py - airflow/example_dags/example_kubernetes_executor.py - - airflow/providers/cncf/kubernetes/**/* - - airflow/providers/celery/executors/celery_kubernetes_executor.py + - providers/src/airflow/providers/cncf/kubernetes/**/* + - providers/src/airflow/providers/celery/executors/celery_kubernetes_executor.py - docs/apache-airflow-providers-cncf-kubernetes/**/* - kubernetes_tests/**/* - - tests/providers/cncf/kubernetes/**/* - - tests/system/providers/cncf/kubernetes/**/* + - providers/tests/cncf/kubernetes/**/* + - providers/tests/system/cncf/kubernetes/**/* provider:cohere: - - airflow/providers/cohere/**/* + - providers/src/airflow/providers/cohere/**/* - docs/apache-airflow-providers-cohere/**/* - - tests/providers/cohere/**/* - - tests/system/providers/cohere/**/* + - providers/tests/cohere/**/* + - providers/tests/system/cohere/**/* provider:common-compat: - - airflow/providers/common/compat/**/* + - providers/src/airflow/providers/common/compat/**/* - docs/apache-airflow-providers-common-compat/**/* - - tests/providers/common/compat/**/* + - providers/tests/common/compat/**/* provider:common-io: - - airflow/providers/common/io/**/* + - providers/src/airflow/providers/common/io/**/* - docs/apache-airflow-providers-common-io/**/* - - tests/system/providers/common/io/**/* + - providers/tests/system/common/io/**/* provider:common-sql: - - airflow/providers/common/sql/**/* + - providers/src/airflow/providers/common/sql/**/* - docs/apache-airflow-providers-common-sql/**/* - - tests/providers/common/sql/**/* - - tests/system/providers/common/sql/**/* + - providers/tests/common/sql/**/* + - providers/tests/system/common/sql/**/* provider:standard: - - airflow/providers/standard/**/* + - providers/src/airflow/providers/standard/**/* - docs/apache-airflow-providers-standard/**/* - - tests/providers/standard/**/* + - providers/tests/standard/**/* provider:databricks: - - airflow/providers/databricks/**/* + - providers/src/airflow/providers/databricks/**/* - docs/apache-airflow-providers-databricks/**/* - - tests/providers/databricks/**/* - - tests/system/providers/databricks/**/* + - providers/tests/databricks/**/* + - providers/tests/system/databricks/**/* provider:datadog: - - airflow/providers/datadog/**/* + - providers/src/airflow/providers/datadog/**/* - docs/apache-airflow-providers-datadog/**/* - - tests/providers/datadog/**/* + - providers/tests/datadog/**/* provider:dbt-cloud: - - airflow/providers/dbt/cloud/**/* + - providers/src/airflow/providers/dbt/cloud/**/* - docs/apache-airflow-providers-dbt-cloud/**/* - - tests/providers/dbt/cloud/**/* - - tests/system/providers/dbt/cloud/**/* + - providers/tests/dbt/cloud/**/* + - providers/tests/system/dbt/cloud/**/* provider:dingding: - - airflow/providers/dingding/**/* + - providers/src/airflow/providers/dingding/**/* - docs/apache-airflow-providers-dingding/**/* - - tests/providers/dingding/**/* - - tests/system/providers/dingding/**/* + - providers/tests/dingding/**/* + - providers/tests/system/dingding/**/* provider:discord: - - airflow/providers/discord/**/* + - providers/src/airflow/providers/discord/**/* - docs/apache-airflow-providers-discord/**/* - - tests/providers/discord/**/* + - providers/tests/discord/**/* provider:docker: - - airflow/providers/docker/**/* + - providers/src/airflow/providers/docker/**/* - docs/apache-airflow-providers-docker/**/* - - tests/providers/docker/**/* - - tests/system/providers/docker/**/* + - providers/tests/docker/**/* + - providers/tests/system/docker/**/* provider:elasticsearch: - - airflow/providers/elasticsearch/**/* + - providers/src/airflow/providers/elasticsearch/**/* - docs/apache-airflow-providers-elasticsearch/**/* - - tests/providers/elasticsearch/**/* - - tests/system/providers/elasticsearch/**/* + - providers/tests/elasticsearch/**/* + - providers/tests/system/elasticsearch/**/* provider:exasol: - - airflow/providers/exasol/**/* + - providers/src/airflow/providers/exasol/**/* - docs/apache-airflow-providers-exasol/**/* - - tests/providers/exasol/**/* + - providers/tests/exasol/**/* provider:fab: - - airflow/providers/fab/**/* + - providers/src/airflow/providers/fab/**/* - docs/apache-airflow-providers-fab/**/* - - tests/providers/fab/**/* + - providers/tests/fab/**/* provider:facebook: - - airflow/providers/facebook/**/* + - providers/src/airflow/providers/facebook/**/* - docs/apache-airflow-providers-facebook/**/* - - tests/providers/facebook/**/* + - providers/tests/facebook/**/* provider:ftp: - - airflow/providers/ftp/**/* + - providers/src/airflow/providers/ftp/**/* - docs/apache-airflow-providers-ftp/**/* - - tests/providers/ftp/**/* - - tests/system/providers/ftp/**/* + - providers/tests/ftp/**/* + - providers/tests/system/ftp/**/* provider:github: - - airflow/providers/github/**/* + - providers/src/airflow/providers/github/**/* - docs/apache-airflow-providers-github/**/* - - tests/providers/github/**/* - - tests/system/providers/github/**/* + - providers/tests/github/**/* + - providers/tests/system/github/**/* provider:google: - - airflow/providers/google/**/* + - providers/src/airflow/providers/google/**/* - docs/apache-airflow-providers-google/**/* - - tests/providers/google/**/* - - tests/system/providers/google/**/* + - providers/tests/google/**/* + - providers/tests/system/google/**/* provider:grpc: - - airflow/providers/grpc/**/* + - providers/src/airflow/providers/grpc/**/* - docs/apache-airflow-providers-grpc/**/* - - tests/providers/grpc/**/* + - providers/tests/grpc/**/* provider:hashicorp: - - airflow/providers/hashicorp/**/* + - providers/src/airflow/providers/hashicorp/**/* - docs/apache-airflow-providers-hashicorp/**/* - - tests/providers/hashicorp/**/* + - providers/tests/hashicorp/**/* provider:http: - - airflow/providers/http/**/* + - providers/src/airflow/providers/http/**/* - docs/apache-airflow-providers-http/**/* - - tests/providers/http/**/* - - tests/system/providers/http/**/* + - providers/tests/http/**/* + - providers/tests/system/http/**/* provider:imap: - - airflow/providers/imap/**/* + - providers/src/airflow/providers/imap/**/* - docs/apache-airflow-providers-imap/**/* - - tests/providers/imap/**/* + - providers/tests/imap/**/* provider:influxdb: - - airflow/providers/influxdb/**/* + - providers/src/airflow/providers/influxdb/**/* - docs/apache-airflow-providers-influxdb/**/* - - tests/providers/influxdb/**/* - - tests/system/providers/influxdb/**/* + - providers/tests/influxdb/**/* + - providers/tests/system/influxdb/**/* provider:jdbc: - - airflow/providers/jdbc/**/* + - providers/src/airflow/providers/jdbc/**/* - docs/apache-airflow-providers-jdbc/**/* - - tests/providers/jdbc/**/* - - tests/system/providers/jdbc/**/* + - providers/tests/jdbc/**/* + - providers/tests/system/jdbc/**/* provider:jenkins: - - airflow/providers/jenkins/**/* + - providers/src/airflow/providers/jenkins/**/* - docs/apache-airflow-providers-jenkins/**/* - - tests/providers/jenkins/**/* - - tests/system/providers/jenkins/**/* + - providers/tests/jenkins/**/* + - providers/tests/system/jenkins/**/* provider:microsoft-azure: - - airflow/providers/microsoft/azure/**/* - - tests/providers/microsoft/azure/**/* + - providers/src/airflow/providers/microsoft/azure/**/* + - providers/tests/microsoft/azure/**/* - docs/apache-airflow-providers-microsoft-azure/**/* - - tests/system/providers/microsoft/azure/**/* + - providers/tests/system/microsoft/azure/**/* provider:microsoft-mssql: - - airflow/providers/microsoft/mssql/**/* + - providers/src/airflow/providers/microsoft/mssql/**/* - docs/apache-airflow-providers-microsoft-mssql/**/* - - tests/providers/microsoft/mssql/**/* - - tests/system/providers/microsoft/mssql/**/* + - providers/tests/microsoft/mssql/**/* + - providers/tests/system/microsoft/mssql/**/* provider:microsoft-psrp: - - airflow/providers/microsoft/psrp/**/* + - providers/src/airflow/providers/microsoft/psrp/**/* - docs/apache-airflow-providers-microsoft-psrp/**/* - - tests/providers/microsoft/psrp/**/* + - providers/tests/microsoft/psrp/**/* provider:microsoft-winrm: - - airflow/providers/microsoft/winrm/**/* + - providers/src/airflow/providers/microsoft/winrm/**/* - docs/apache-airflow-providers-microsoft-winrm/**/* - - tests/providers/microsoft/winrm/**/* - - tests/system/providers/microsoft/winrm/**/* + - providers/tests/microsoft/winrm/**/* + - providers/tests/system/microsoft/winrm/**/* provider:mongo: - - airflow/providers/mongo/**/* + - providers/src/airflow/providers/mongo/**/* - docs/apache-airflow-providers-mongo/**/* - - tests/providers/mongo/**/* + - providers/tests/mongo/**/* provider:mysql: - - airflow/providers/mysql/**/* + - providers/src/airflow/providers/mysql/**/* - docs/apache-airflow-providers-mysql/**/* - - tests/providers/mysql/**/* - - tests/system/providers/mysql/**/* + - providers/tests/mysql/**/* + - providers/tests/system/mysql/**/* provider:neo4j: - - airflow/providers/neo4j/**/* + - providers/src/airflow/providers/neo4j/**/* - docs/apache-airflow-providers-neo4j/**/* - - tests/providers/neo4j/**/* - - tests/system/providers/neo4j/**/* + - providers/tests/neo4j/**/* + - providers/tests/system/neo4j/**/* provider:odbc: - - airflow/providers/odbc/**/* + - providers/src/airflow/providers/odbc/**/* - docs/apache-airflow-providers-odbc/**/* - - tests/providers/odbc/**/* + - providers/tests/odbc/**/* provider:openai: - - airflow/providers/openai/**/* + - providers/src/airflow/providers/openai/**/* - docs/apache-airflow-providers-openai/**/* - - tests/providers/openai/**/* - - tests/system/providers/openai/**/* + - providers/tests/openai/**/* + - providers/tests/system/openai/**/* provider:openfaas: - - airflow/providers/openfaas/**/* + - providers/src/airflow/providers/openfaas/**/* - docs/apache-airflow-providers-openfaas/**/* - - tests/providers/openfaas/**/* + - providers/tests/openfaas/**/* provider:openlineage: - - airflow/providers/openlineage/**/* + - providers/src/airflow/providers/openlineage/**/* - docs/apache-airflow-providers-openlineage/**/* - - tests/providers/openlineage/**/* + - providers/tests/openlineage/**/* provider:opensearch: - - airflow/providers/opensearch/**/* + - providers/src/airflow/providers/opensearch/**/* - docs/apache-airflow-providers-opensearch/**/* - - tests/providers/opensearch/**/* - - tests/system/providers/opensearch/**/* + - providers/tests/opensearch/**/* + - providers/tests/system/opensearch/**/* provider:opsgenie: - - airflow/providers/opsgenie/**/* + - providers/src/airflow/providers/opsgenie/**/* - docs/apache-airflow-providers-opsgenie/**/* - - tests/providers/opsgenie/**/* - - tests/system/providers/opsgenie/**/* + - providers/tests/opsgenie/**/* + - providers/tests/system/opsgenie/**/* provider:Oracle: - - airflow/providers/oracle/**/* + - providers/src/airflow/providers/oracle/**/* - docs/apache-airflow-providers-oracle/**/* - - tests/providers/oracle/**/* + - providers/tests/oracle/**/* provider:pagerduty: - - airflow/providers/pagerduty/**/* + - providers/src/airflow/providers/pagerduty/**/* - docs/apache-airflow-providers-pagerduty/**/* - - tests/providers/pagerduty/**/* + - providers/tests/pagerduty/**/* provider:papermill: - - airflow/providers/papermill/**/* + - providers/src/airflow/providers/papermill/**/* - docs/apache-airflow-providers-papermill/**/* - - tests/providers/papermill/**/* - - tests/system/providers/papermill/**/* + - providers/tests/papermill/**/* + - providers/tests/system/papermill/**/* provider:pgvector: - - airflow/providers/pgvector/**/* + - providers/src/airflow/providers/pgvector/**/* - docs/apache-airflow-providers-pgvector/**/* - - tests/providers/pgvector/**/* - - tests/system/providers/pgvector/**/* + - providers/tests/pgvector/**/* + - providers/tests/system/pgvector/**/* provider:pinecone: - - airflow/providers/pinecone/**/* + - providers/src/airflow/providers/pinecone/**/* - docs/apache-airflow-providers-pinecone/**/* - - tests/providers/pinecone/**/* - - tests/system/providers/pinecone/**/* + - providers/tests/pinecone/**/* + - providers/tests/system/pinecone/**/* provider:postgres: - - airflow/providers/postgres/**/* + - providers/src/airflow/providers/postgres/**/* - docs/apache-airflow-providers-postgres/**/* - - tests/providers/postgres/**/* - - tests/system/providers/postgres/**/* + - providers/tests/postgres/**/* + - providers/tests/system/postgres/**/* provider:presto: - - airflow/providers/presto/**/* + - providers/src/airflow/providers/presto/**/* - docs/apache-airflow-providers-presto/**/* - - tests/providers/presto/**/* - - tests/system/providers/presto/**/* + - providers/tests/presto/**/* + - providers/tests/system/presto/**/* provider:qdrant: - - airflow/providers/qdrant/**/* + - providers/src/airflow/providers/qdrant/**/* - docs/apache-airflow-providers-qdrant/**/* - - tests/providers/qdrant/**/* - - tests/system/providers/qdrant/**/* + - providers/tests/qdrant/**/* + - providers/tests/system/qdrant/**/* provider:redis: - - airflow/providers/redis/**/* + - providers/src/airflow/providers/redis/**/* - docs/apache-airflow-providers-redis/**/* - - tests/providers/redis/**/* + - providers/tests/redis/**/* provider:salesforce: - - airflow/providers/salesforce/**/* + - providers/src/airflow/providers/salesforce/**/* - docs/apache-airflow-providers-salesforce/**/* - - tests/providers/salesforce/**/* - - tests/system/providers/salesforce/**/* + - providers/tests/salesforce/**/* + - providers/tests/system/salesforce/**/* provider:samba: - - airflow/providers/samba/**/* + - providers/src/airflow/providers/samba/**/* - docs/apache-airflow-providers-samba/**/* - - tests/providers/samba/**/* + - providers/tests/samba/**/* provider:segment: - - airflow/providers/segment/**/* + - providers/src/airflow/providers/segment/**/* - docs/apache-airflow-providers-segment/**/* - - tests/providers/segment/**/* + - providers/tests/segment/**/* provider:sendgrid: - - airflow/providers/segment/**/* + - providers/src/airflow/providers/segment/**/* - docs/apache-airflow-providers-segment/**/* - - tests/providers/segment/**/* + - providers/tests/segment/**/* provider:sftp: - - airflow/providers/sftp/**/* + - providers/src/airflow/providers/sftp/**/* - docs/apache-airflow-providers-sftp/**/* - - tests/providers/sftp/**/* + - providers/tests/sftp/**/* provider:singularity: - - airflow/providers/singularity/**/* + - providers/src/airflow/providers/singularity/**/* - docs/apache-airflow-providers-singularity/**/* - - tests/providers/singularity/**/* - - tests/system/providers/singularity/**/* + - providers/tests/singularity/**/* + - providers/tests/system/singularity/**/* provider:slack: - - airflow/providers/slack/**/* + - providers/src/airflow/providers/slack/**/* - docs/apache-airflow-providers-slack/**/* - - tests/providers/slack/**/* - - tests/system/providers/slack/**/* + - providers/tests/slack/**/* + - providers/tests/system/slack/**/* provider:smtp: - - airflow/providers/smtp/**/* + - providers/src/airflow/providers/smtp/**/* - docs/apache-airflow-providers-smtp/**/* - - tests/providers/smtp/**/* + - providers/tests/smtp/**/* provider:snowflake: - - airflow/providers/snowflake/**/* + - providers/src/airflow/providers/snowflake/**/* - docs/apache-airflow-providers-snowflake/**/* - - tests/providers/snowflake/**/* - - tests/system/providers/snowflake/**/* + - providers/tests/snowflake/**/* + - providers/tests/system/snowflake/**/* provider:sqlite: - - airflow/providers/sqlite/**/* + - providers/src/airflow/providers/sqlite/**/* - docs/apache-airflow-providers-sqlite/**/* - - tests/providers/sqlite/**/* - - tests/system/providers/sqlite/**/* + - providers/tests/sqlite/**/* + - providers/tests/system/sqlite/**/* provider:ssh: - - airflow/providers/ssh/**/* + - providers/src/airflow/providers/ssh/**/* - docs/apache-airflow-providers-ssh/**/* - - tests/providers/ssh/**/* + - providers/tests/ssh/**/* provider:tableau: - - airflow/providers/tableau/**/* + - providers/src/airflow/providers/tableau/**/* - docs/apache-airflow-providers-tableau/**/* - - tests/providers/tableau/**/* - - tests/system/providers/tableau/**/* + - providers/tests/tableau/**/* + - providers/tests/system/tableau/**/* provider:telegram: - - airflow/providers/telegram/**/* + - providers/src/airflow/providers/telegram/**/* - docs/apache-airflow-providers-telegram/**/* - - tests/providers/telegram/**/* - - tests/system/providers/telegram/**/* + - providers/tests/telegram/**/* + - providers/tests/system/telegram/**/* provider:teradata: - - airflow/providers/teradata/**/* + - providers/src/airflow/providers/teradata/**/* - docs/apache-airflow-providers-teradata/**/* - - tests/providers/teradata/**/* - - tests/system/providers/teradata/**/* + - providers/tests/teradata/**/* + - providers/tests/system/teradata/**/* provider:trino: - - airflow/providers/trino/**/* + - providers/src/airflow/providers/trino/**/* - docs/apache-airflow-providers-trino/**/* - - tests/providers/trino/**/* - - tests/system/providers/trino/**/* + - providers/tests/trino/**/* + - providers/tests/system/trino/**/* provider:vertica: - - airflow/providers/vertica/**/* + - providers/src/airflow/providers/vertica/**/* - docs/apache-airflow-providers-vertica/**/* - - tests/providers/vertica/**/* + - providers/tests/vertica/**/* provider:weaviate: - - airflow/providers/weaviate/**/* + - providers/src/airflow/providers/weaviate/**/* - docs/apache-airflow-providers-weaviate/**/* - - tests/providers/weaviate/**/* - - tests/system/providers/weaviate/**/* + - providers/tests/weaviate/**/* + - providers/tests/system/weaviate/**/* provider:yandex: - - airflow/providers/yandex/**/* + - providers/src/airflow/providers/yandex/**/* - docs/apache-airflow-providers-yandex/**/* - - tests/providers/yandex/**/* - - tests/system/providers/yandex/**/* + - providers/tests/yandex/**/* + - providers/tests/system/yandex/**/* provider:ydb: - - airflow/providers/ydb/**/* + - providers/src/airflow/providers/ydb/**/* - docs/apache-airflow-providers-ydb/**/* - - tests/providers/ydb/**/* - - tests/system/providers/ydb/**/* + - providers/tests/ydb/**/* + - providers/tests/system/ydb/**/* provider:zendesk: - - airflow/providers/zendesk/**/* + - providers/src/airflow/providers/zendesk/**/* - docs/apache-airflow-providers-zendesk/**/* - - tests/providers/zendesk/**/* - - tests/system/providers/zendesk/**/* + - providers/tests/zendesk/**/* + - providers/tests/system/zendesk/**/* area:providers: - - airflow/providers/**/* + - providers/src/airflow/providers/**/* - docs/apache-airflow-providers-*/**/* - - tests/providers/**/* - - tests/system/providers/**/* + - providers/tests/**/* + - providers/tests/system/**/* area:API: - airflow/api/**/* @@ -611,10 +611,10 @@ labelPRBasedOnFilePath: - docs/apache-airflow/administration-and-deployment/lineage.rst area:Logging: - - airflow/providers/**/log/* + - providers/src/airflow/providers/**/log/* - airflow/utils/log/**/* - docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-*.rst - - tests/providers/**/log/* + - providers/tests/**/log/* - tests/utils/log/**/* area:Plugins: @@ -638,9 +638,9 @@ labelPRBasedOnFilePath: area:Secrets: - airflow/secrets/**/* - - airflow/providers/**/secrets/* + - providers/src/airflow/providers/**/secrets/* - tests/secrets/**/* - - tests/providers/**/secrets/* + - providers/tests/**/secrets/* - docs/apache-airflow/security/secrets/**/* area:Triggerer: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 49d6a7245bc88..509d7066f6d38 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -195,9 +195,10 @@ jobs: env: HATCH_ENV: "test" working-directory: ./clients/python - - name: "Prepare FAB provider packages: wheel" + - name: "Prepare FAB+standard provider packages: wheel" run: > - breeze release-management prepare-provider-packages fab --package-format wheel --skip-tag-check + breeze release-management prepare-provider-packages fab standard \ + --package-format wheel --skip-tag-check - name: "Install Airflow with fab for webserver tests" run: pip install . dist/apache_airflow_providers_fab-*.whl - name: "Prepare Standard provider packages: wheel" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce557dba431b6..748a85e860bb7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -172,7 +172,7 @@ repos: name: Check and update common.sql API stubs entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py language: python - files: ^scripts/ci/pre_commit/update_common_sql_api\.py|^airflow/providers/common/sql/.*\.pyi?$ + files: ^scripts/ci/pre_commit/update_common_sql_api\.py|^providers/src/airflow/providers/common/sql/.*\.pyi?$ additional_dependencies: ['rich>=12.4.4', 'mypy==1.9.0', 'black==23.10.0', 'jinja2'] pass_filenames: false require_serial: true @@ -225,7 +225,7 @@ repos: entry: ./scripts/ci/pre_commit/check_deferrable_default.py pass_filenames: false additional_dependencies: ["libcst>=1.1.0"] - files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$ + files: ^(providers/src/)?airflow/.*/(sensors|operators)/.*\.py$ - repo: https://github.com/asottile/blacken-docs rev: 1.18.0 hooks: @@ -326,7 +326,7 @@ repos: exclude: ^.*/.*_vendor/|^airflow/www/static/css/material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$|^.*pnpm-lock\.yaml$ args: - --ignore-words=docs/spelling_wordlist.txt - - --skip=airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md + - --skip=providers/src/airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md - --exclude-file=.codespellignorelines - repo: local # Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because @@ -343,7 +343,7 @@ repos: language: python entry: ./scripts/ci/pre_commit/validate_operators_init.py pass_filenames: true - files: ^airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ + files: ^providers/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ additional_dependencies: [ 'rich>=12.4.4' ] - id: ruff name: Run 'ruff' for extremely fast Python linting @@ -418,7 +418,7 @@ repos: language: python files: ^airflow/.*\.py$ require_serial: true - exclude: ^airflow/kubernetes/|^airflow/providers/ + exclude: ^airflow/kubernetes/|^providers/src/airflow/providers/ entry: ./scripts/ci/pre_commit/check_cncf_k8s_used_for_k8s_executor_only.py additional_dependencies: ['rich>=12.4.4'] - id: check-airflow-provider-compatibility @@ -426,7 +426,7 @@ repos: entry: ./scripts/ci/pre_commit/check_provider_airflow_compatibility.py language: python pass_filenames: true - files: ^airflow/providers/.*\.py$ + files: ^providers/src/airflow/providers/.*\.py$ additional_dependencies: ['rich>=12.4.4'] - id: check-google-re2-as-dependency name: Check google-re2 declared as dep @@ -435,7 +435,7 @@ repos: language: python pass_filenames: true require_serial: true - files: ^airflow/providers/.*\.py$ + files: ^providers/src/airflow/providers/.*\.py$ additional_dependencies: ['rich>=12.4.4'] - id: update-local-yml-file name: Update mounts in the local yml file @@ -449,13 +449,18 @@ repos: description: Check dependency of SQL Providers with common data structure entry: ./scripts/ci/pre_commit/check_common_sql_dependency.py language: python - files: ^airflow/providers/.*/hooks/.*\.py$ + files: ^providers/src/airflow/providers/.*/hooks/.*\.py$ additional_dependencies: ['rich>=12.4.4', 'pyyaml', 'packaging'] - id: update-providers-dependencies name: Update dependencies for provider packages entry: ./scripts/ci/pre_commit/update_providers_dependencies.py language: python - files: ^airflow/providers/.*\.py$|^airflow/providers/.*/provider\.yaml$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|^scripts/ci/pre_commit/update_providers_dependencies\.py$ + files: | + (?x) + ^providers/src/airflow/providers/.*\.py$ | + ^providers/src/airflow/providers/.*/provider\.yaml$ | + ^providers/tests/.*\.py$ | + ^scripts/ci/pre_commit/update_providers_dependencies\.py$ pass_filenames: false additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml', 'tomli'] - id: check-extra-packages-references @@ -477,7 +482,7 @@ repos: name: Update extras in documentation entry: ./scripts/ci/pre_commit/insert_extras.py language: python - files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^airflow/providers/.*/provider\.yaml$|^Dockerfile.* + files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^providers/src/airflow/providers/.*/provider\.yaml$|^Dockerfile.* pass_filenames: false additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0'] - id: check-extras-order @@ -510,9 +515,9 @@ repos: (?x) ^scripts/ci/pre_commit/version_heads_map\.py$| ^airflow/migrations/versions/.*$|^airflow/migrations/versions| - ^airflow/providers/fab/migrations/versions/.*$|^airflow/providers/fab/migrations/versions| + ^providers/src/airflow/providers/fab/migrations/versions/.*$|^providers/src/airflow/providers/fab/migrations/versions| ^airflow/utils/db.py$| - ^airflow/providers/fab/auth_manager/models/db.py$ + ^providers/src/airflow/providers/fab/auth_manager/models/db.py$ additional_dependencies: ['packaging','google-re2'] - id: update-version name: Update versions in docs @@ -533,11 +538,11 @@ repos: language: pygrep entry: > (?i) - .*https://github.*[0-9]/tests/system/providers| - .*https://github.*/main/tests/system/providers| - .*https://github.*/master/tests/system/providers| - .*https://github.*/main/airflow/providers/.*/example_dags/| - .*https://github.*/master/airflow/providers/.*/example_dags/ + .*https://github.*[0-9]/providers/tests/system/| + .*https://github.*/main/providers/tests/system/| + .*https://github.*/master/providers/tests/system/| + .*https://github.*/main/providers/src/airflow/providers/.*/example_dags/| + .*https://github.*/master/providers/src/airflow/providers/.*/example_dags/ pass_filenames: true files: ^docs/apache-airflow-providers-.*\.rst - id: check-safe-filter-usage-in-html @@ -560,7 +565,7 @@ repos: description: Use AirflowProviderDeprecationWarning in providers entry: "^\\s*DeprecationWarning*" pass_filenames: true - files: ^airflow/providers/.*\.py$ + files: ^providers/src/airflow/providers/.*\.py$ - id: check-urlparse-usage-in-code language: pygrep name: Don't use urlparse in code @@ -601,28 +606,28 @@ repos: ^airflow/config_templates/| ^airflow/models/baseoperator.py$| ^airflow/operators/__init__.py$| - ^airflow/providers/amazon/aws/hooks/emr.py$| - ^airflow/providers/amazon/aws/operators/emr.py$| - ^airflow/providers/apache/cassandra/hooks/cassandra.py$| - ^airflow/providers/apache/hive/operators/hive_stats.py$| - ^airflow/providers/apache/hive/transfers/vertica_to_hive.py$| - ^airflow/providers/apache/spark/decorators/| - ^airflow/providers/apache/spark/hooks/| - ^airflow/providers/apache/spark/operators/| - ^airflow/providers/exasol/hooks/exasol.py$| - ^airflow/providers/fab/auth_manager/security_manager/| - ^airflow/providers/google/cloud/hooks/bigquery.py$| - ^airflow/providers/google/cloud/operators/cloud_build.py$| - ^airflow/providers/google/cloud/operators/dataproc.py$| - ^airflow/providers/google/cloud/operators/mlengine.py$| - ^airflow/providers/microsoft/azure/hooks/cosmos.py$| - ^airflow/providers/microsoft/winrm/hooks/winrm.py$| + ^providers/src/airflow/providers/amazon/aws/hooks/emr.py$| + ^providers/src/airflow/providers/amazon/aws/operators/emr.py$| + ^providers/src/airflow/providers/apache/cassandra/hooks/cassandra.py$| + ^providers/src/airflow/providers/apache/hive/operators/hive_stats.py$| + ^providers/src/airflow/providers/apache/hive/transfers/vertica_to_hive.py$| + ^providers/src/airflow/providers/apache/spark/decorators/| + ^providers/src/airflow/providers/apache/spark/hooks/| + ^providers/src/airflow/providers/apache/spark/operators/| + ^providers/src/airflow/providers/exasol/hooks/exasol.py$| + ^providers/src/airflow/providers/fab/auth_manager/security_manager/| + ^providers/src/airflow/providers/google/cloud/hooks/bigquery.py$| + ^providers/src/airflow/providers/google/cloud/operators/cloud_build.py$| + ^providers/src/airflow/providers/google/cloud/operators/dataproc.py$| + ^providers/src/airflow/providers/google/cloud/operators/mlengine.py$| + ^providers/src/airflow/providers/microsoft/azure/hooks/cosmos.py$| + ^providers/src/airflow/providers/microsoft/winrm/hooks/winrm.py$| ^airflow/www/fab_security/manager.py$| ^docs/.*commits.rst$| ^docs/apache-airflow-providers-apache-cassandra/connections/cassandra.rst$| - ^airflow/providers/microsoft/winrm/operators/winrm.py$| - ^airflow/providers/opsgenie/hooks/opsgenie.py$| - ^airflow/providers/redis/provider.yaml$| + ^providers/src/airflow/providers/microsoft/winrm/operators/winrm.py$| + ^providers/src/airflow/providers/opsgenie/hooks/opsgenie.py$| + ^providers/src/airflow/providers/redis/provider.yaml$| ^airflow/serialization/serialized_objects.py$| ^airflow/ui/pnpm-lock.yaml$| ^airflow/utils/db.py$| @@ -647,6 +652,7 @@ repos: ^newsfragments/41761.significant.rst$| ^scripts/ci/pre_commit/vendor_k8s_json_schema.py$| ^tests/| + ^providers/tests/| ^.pre-commit-config\.yaml$| ^.*CHANGELOG\.(rst|txt)$| ^.*RELEASE_NOTES\.rst$| @@ -682,7 +688,7 @@ repos: ^airflow/decorators/.*$| ^airflow/hooks/.*$| ^airflow/operators/.*$| - ^airflow/providers/.*$| + ^providers/src/airflow/providers/.*$| ^airflow/sensors/.*$| ^dev/provider_packages/.*$ - id: check-base-operator-usage @@ -697,7 +703,7 @@ repos: ^airflow/decorators/.*$| ^airflow/hooks/.*$| ^airflow/operators/.*$| - ^airflow/providers/.*$| + ^providers/src/airflow/providers/.*$| ^airflow/sensors/.*$| ^dev/provider_packages/.*$ - id: check-base-operator-usage @@ -708,16 +714,16 @@ repos: pass_filenames: true files: > (?x) - ^airflow/providers/.*\.py$ - exclude: ^.*/.*_vendor/|airflow/providers/standard/operators/bash.py + ^providers/src/airflow/providers/.*\.py$ + exclude: ^.*/.*_vendor/|providers/src/airflow/providers/standard/operators/bash.py - id: check-get-lineage-collector-providers language: python name: Check providers import hook lineage code from compat description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of airflow.lineage.hook. entry: ./scripts/ci/pre_commit/check_get_lineage_collector_providers.py - files: ^airflow/providers/.*\.py$ - exclude: ^airflow/providers/common/compat/.*\.py$ + files: ^providers/src/airflow/providers/.*\.py$ + exclude: ^providers/src/airflow/providers/common/compat/.*\.py$ additional_dependencies: [ 'rich>=12.4.4' ] - id: check-decorated-operator-implements-custom-name name: Check @task decorator implements custom_operator_name @@ -730,7 +736,7 @@ repos: name: Verify usage of Airflow deprecation classes in core entry: category=DeprecationWarning|category=PendingDeprecationWarning files: \.py$ - exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$|^tests/.*$ + exclude: ^airflow/configuration\.py$|^providers/src/airflow/providers/|^scripts/in_container/verify_providers\.py$|^(providers/)?tests/.*$|^dev/tests_common/ pass_filenames: true - id: check-provide-create-sessions-imports language: pygrep @@ -772,7 +778,7 @@ repos: name: Check if aiobotocore is an optional dependency only entry: ./scripts/ci/pre_commit/check_aiobotocore_optional.py language: python - files: ^airflow/providers/.*/provider\.yaml$ + files: ^providers/src/airflow/providers/.*/provider\.yaml$ pass_filenames: true additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml'] require_serial: true @@ -877,12 +883,6 @@ repos: entry: ./scripts/ci/pre_commit/compile_www_assets_dev.py pass_filenames: false additional_dependencies: ['yarn@1.22.21'] - - id: check-providers-init-file-missing - name: Provider init file is missing - pass_filenames: false - always_run: true - entry: ./scripts/ci/pre_commit/check_providers_init.py - language: python - id: check-providers-subpackages-init-file-exist name: Provider subpackage init files are there pass_filenames: false @@ -944,8 +944,8 @@ repos: name: Check if system tests have required segments of code entry: ./scripts/ci/pre_commit/check_system_tests.py language: python - files: ^tests/system/.*/example_[^/]*\.py$ - exclude: ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ + files: ^(providers/)?tests/system/.*/example_[^/]*\.py$ + exclude: ^providers/tests/system/google/cloud/bigquery/example_bigquery_queries\.py$ pass_filenames: true additional_dependencies: ['rich>=12.4.4'] - id: generate-pypi-readme @@ -1177,12 +1177,21 @@ repos: entry: "^\\s*from re\\s|^\\s*import re\\s" pass_filenames: true files: \.py$ - exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^\w+_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$|^hatch_build.py$ + exclude: | + (?x) + ^airflow/utils/helpers.py$ | + ^providers/src/airflow/providers/ | + ^(providers/)?tests/ | + ^dev/.*\.py$ | + ^scripts/.*\.py$ | + ^\w+_tests/ | + ^docs/.*\.py$ | + ^hatch_build.py$ - id: check-provider-docs-valid name: Validate provider doc files entry: ./scripts/ci/pre_commit/check_provider_docs.py language: python - files: ^airflow/providers/.*/provider\.yaml|^docs/.* + files: ^providers/src/airflow/providers/.*/provider\.yaml|^docs/.* additional_dependencies: ['rich>=12.4.4', 'pyyaml', 'jinja2'] require_serial: true - id: bandit @@ -1273,9 +1282,20 @@ repos: - id: mypy-airflow name: Run mypy for airflow language: python - entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages + entry: ./scripts/ci/pre_commit/mypy.py files: \.py$ - exclude: ^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^scripts|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py|^clients/python/test_.*\.py|^performance + exclude: | + (?x) + ^.*/.*_vendor/ | + ^airflow/migrations | + ^providers/ | + ^dev | + ^scripts | + ^docs | + ^provider_packages | + ^performance/ | + ^tests/dags/test_imports.py | + ^clients/python/test_.*\.py require_serial: true additional_dependencies: ['rich>=12.4.4'] - id: mypy-airflow @@ -1291,7 +1311,7 @@ repos: name: Run mypy for providers language: python entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages - files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$ + files: ^providers/src/airflow/providers/.*\.py$|^providers/tests//.*\.py$ exclude: ^.*/.*_vendor/ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1299,7 +1319,7 @@ repos: stages: ['manual'] name: Run mypy for providers (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py airflow/providers + entry: ./scripts/ci/pre_commit/mypy_folder.py providers/src/airflow/providers pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1325,14 +1345,14 @@ repos: name: Validate provider.yaml files entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py language: python - files: ^airflow/providers/.*/provider\.yaml$ + files: ^providers/src/airflow/providers/.*/provider\.yaml$ additional_dependencies: ['rich>=12.4.4'] require_serial: true - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors language: python entry: ./scripts/ci/pre_commit/check_template_fields.py - files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$ + files: ^(providers/src/)?airflow/.*/(sensors|operators)/.*\.py$ additional_dependencies: [ 'rich>=12.4.4' ] require_serial: true - id: update-migration-references diff --git a/Dockerfile b/Dockerfile index 4525a717728a8..ce229c75facef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -877,8 +877,13 @@ function install_airflow() { # Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then + # We need _a _ file in there otherwise the editable install doesn't include anything in the .pth file + mkdir -p ./providers/src/airflow/providers/ + touch ./providers/src/airflow/providers/__init__.py + trap 'rm -f ./providers/src/airflow/providers/__init__.py 2>/dev/null' EXIT + # When installing from sources - we always use `--editable` mode - installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" + installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then diff --git a/Dockerfile.ci b/Dockerfile.ci index 304d2a4a2d46d..a3e982614de69 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -676,8 +676,13 @@ function install_airflow() { # Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then + # We need _a _ file in there otherwise the editable install doesn't include anything in the .pth file + mkdir -p ./providers/src/airflow/providers/ + touch ./providers/src/airflow/providers/__init__.py + trap 'rm -f ./providers/src/airflow/providers/__init__.py 2>/dev/null' EXIT + # When installing from sources - we always use `--editable` mode - installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" + installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then @@ -1350,6 +1355,7 @@ RUN bash /scripts/docker/install_pipx_tools.sh # We copy pyproject.toml and other files needed to perform setup of dependencies # So in case pyproject.toml changes we can install latest dependencies required. COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml +COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/ COPY generated/* ${AIRFLOW_SOURCES}/generated/ COPY constraints/* ${AIRFLOW_SOURCES}/constraints/ diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 246e0e95ec617..2dc425daa0549 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -113,7 +113,6 @@ clear_task_instances, ) from airflow.models.tasklog import LogTemplate -from airflow.providers.fab import __version__ as FAB_VERSION from airflow.secrets.local_filesystem import LocalFilesystemBackend from airflow.security import permissions from airflow.settings import json @@ -796,6 +795,9 @@ def _upgrade_outdated_dag_access_control(access_control=None): """Look for outdated dag level actions in DAG access_controls and replace them with updated actions.""" if access_control is None: return None + + from airflow.providers.fab import __version__ as FAB_VERSION + updated_access_control = {} for role, perms in access_control.items(): if packaging_version.parse(FAB_VERSION) >= packaging_version.parse("1.3.0"): diff --git a/airflow/providers/.gitignore b/airflow/providers/.gitignore deleted file mode 100644 index 9b4a1a9d8f3ed..0000000000000 --- a/airflow/providers/.gitignore +++ /dev/null @@ -1 +0,0 @@ -get_provider_info.py diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index 2c673063cb23e..1b1ca469f21d5 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -31,7 +31,7 @@ from dataclasses import dataclass from functools import wraps from time import perf_counter -from typing import TYPE_CHECKING, Any, Callable, MutableMapping, NamedTuple, NoReturn, TypeVar +from typing import TYPE_CHECKING, Any, Callable, MutableMapping, NamedTuple, TypeVar from packaging.utils import canonicalize_name @@ -362,7 +362,7 @@ def _correctness_check(provider_package: str, class_name: str, provider_info: Pr # We want to have better control over initialization of parameters and be able to debug and test it # So we add our own decorator -def provider_info_cache(cache_name: str) -> Callable[[Callable[PS, NoReturn]], Callable[PS, None]]: +def provider_info_cache(cache_name: str) -> Callable[[Callable[PS, None]], Callable[PS, None]]: """ Decorate and cache provider info. @@ -370,7 +370,7 @@ def provider_info_cache(cache_name: str) -> Callable[[Callable[PS, NoReturn]], C :param cache_name: Name of the cache """ - def provider_info_cache_decorator(func: Callable[PS, NoReturn]) -> Callable[PS, None]: + def provider_info_cache_decorator(func: Callable[PS, None]) -> Callable[PS, None]: @wraps(func) def wrapped_function(*args: PS.args, **kwargs: PS.kwargs) -> None: providers_manager_instance = args[0] diff --git a/airflow/settings.py b/airflow/settings.py index 7a805f64a29c7..a6adbbcf9ff77 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -377,7 +377,8 @@ def is_called_from_test_code(self) -> tuple[bool, traceback.FrameSummary | None] and not tb.filename == AIRFLOW_UTILS_SESSION_PATH ] if any( - filename.endswith("conftest.py") or filename.endswith("tests/test_utils/db.py") + filename.endswith("conftest.py") + or filename.endswith("dev/airflow_common_pytest/test_utils/db.py") for filename, _, _, _ in airflow_frames ): # This is a fixture call or testing utilities diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 422a9f027e1ed..f064a13a773b6 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -212,8 +212,6 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-provider-yaml-valid | Validate provider.yaml files | * | +-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-providers-init-file-missing | Provider init file is missing | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-providers-subpackages-init-file-exist | Provider subpackage init files are there | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-pydevd-left-in-code | Check for pydevd debug statements accidentally left | | diff --git a/contributing-docs/testing/system_tests.rst b/contributing-docs/testing/system_tests.rst index 867b89582f165..cc64d25e90cbc 100644 --- a/contributing-docs/testing/system_tests.rst +++ b/contributing-docs/testing/system_tests.rst @@ -35,7 +35,7 @@ Airflow system tests are pretty special because they serve three purposes: Old System Tests ---------------- -The system tests derive from the ``tests.test_utils.system_test_class.SystemTests`` class. +The system tests derive from the ``dev.tests_common.test_utils.system_test_class.SystemTests`` class. Old versions of System tests should also be marked with ``@pytest.marker.system(SYSTEM)`` where ``system`` designates the system to be tested (for example, ``google.cloud``). These tests are skipped by default. diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index dff3835a02fba..468f10b68d4c5 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -952,7 +952,7 @@ will ask you to rebuild the image if it is needed and some new dependencies shou .. code-block:: bash - breeze testing tests tests/providers/http/hooks/test_http.py tests/core/test_core.py --db-reset --log-cli-level=DEBUG + breeze testing tests providers/tests/http/hooks/test_http.py tests/core/test_core.py --db-reset --log-cli-level=DEBUG You can run the whole test suite without adding the test target: @@ -1146,7 +1146,7 @@ directly to the container. .. code-block:: bash - pytest tests/providers//test.py + pytest providers/tests//test.py 4. Iterate with the tests and providers. Both providers and tests are mounted from local sources so changes you do locally in both - tests and provider sources are immediately reflected inside the @@ -1171,7 +1171,7 @@ are not part of the public API. We deal with it in one of the following ways: 1) If the whole provider is supposed to only work for later airflow version, we remove the whole provider by excluding it from compatibility test configuration (see below) -2) Some compatibility shims are defined in ``tests/test_utils/compat.py`` - and they can be used to make the +2) Some compatibility shims are defined in ``dev.tests_common.test_utils/compat.py`` - and they can be used to make the tests compatible - for example importing ``ParseImportError`` after the exception has been renamed from ``ImportError`` and it would fail in Airflow 2.9, but we have a fallback import in ``compat.py`` that falls back to old import automatically, so all tests testing / expecting ``ParseImportError`` should import @@ -1184,7 +1184,7 @@ are not part of the public API. We deal with it in one of the following ways: .. code-block:: python - from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS + from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS @pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="The tests should be skipped for Airflow < 2.8") @@ -1197,6 +1197,9 @@ are not part of the public API. We deal with it in one of the following ways: .. code-block:: python + from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES + + @pytest.mark.skipif( RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES, reason="Plugin initialization is done early in case of packages" ) @@ -1280,7 +1283,7 @@ In case you want to reproduce canary run, you need to add ``--clean-airflow-inst .. code-block:: bash - pytest tests/providers//test.py + pytest providers/tests//test.py 7. Iterate with the tests diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 0ddded9468a55..d52aa78d7ec1f 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -203,32 +203,32 @@ Build documents. ╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--docs-only-dOnly build documentation.│ -│--spellcheck-only-sOnly run spell checking.│ -│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx    │ +│--docs-only-dOnly build documentation.│ +│--spellcheck-only-sOnly run spell checking.│ +│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx    │ │artifacts before the build - useful for a clean build.                           │ -│--one-pass-onlyBuilds documentation in one pass only. This is useful for debugging sphinx       │ +│--one-pass-onlyBuilds documentation in one pass only. This is useful for debugging sphinx       │ │errors.                                                                          │ -│--package-filterFilter(s) to use more than one can be specified. You can use glob pattern        │ +│--package-filterFilter(s) to use more than one can be specified. You can use glob pattern        │ │matching the full package name, for example `apache-airflow-providers-*`. Useful │ │when you want to selectseveral similarly named packages together.                │ │(TEXT)                                                                           │ -│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ -│--include-removed-providersWhether to include providers that are removed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ +│--include-removed-providersWhether to include providers that are removed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ │[default: autodetect]                                         │ -│--package-listOptional, contains comma-separated list of package ids that are processed for    │ +│--package-listOptional, contains comma-separated list of package ids that are processed for    │ │documentation building, and document publishing. It is an easier alternative to  │ │adding individual packages as arguments to every command. This overrides the     │ │packages passed as arguments.                                                    │ │(TEXT)                                                                           │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_compile-ui-assets.svg b/dev/breeze/doc/images/output_compile-ui-assets.svg index 4721274d9cf59..3b6304558479e 100644 --- a/dev/breeze/doc/images/output_compile-ui-assets.svg +++ b/dev/breeze/doc/images/output_compile-ui-assets.svg @@ -104,14 +104,14 @@ Compiles ui assets. ╭─ Compile ui assets flag ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│--devRun development version of assets compilation - it will not quit and automatically recompile assets│ +│--devRun development version of assets compilation - it will not quit and automatically recompile assets│ │on-the-fly when they are changed.                                                                  │ -│--force-cleanForce cleanup of compile assets before building them.│ +│--force-cleanForce cleanup of compile assets before building them.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index b2b90c9e28728..37feab7831e79 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -151,14 +151,14 @@ Command to add back references for documentation to make it backward compatible. ╭─ Add Back References to Docs ────────────────────────────────────────────────────────────────────────────────────────╮ -│*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(DIRECTORY)[required]│ -│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ -│--include-removed-providersWhether to include providers that are removed.│ +│*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(DIRECTORY)[required]│ +│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ +│--include-removed-providersWhether to include providers that are removed.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index f07aa12760d92..62ed9b25dda68 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -151,16 +151,16 @@ Generates content for issue to test the release. ╭─ Generate issue content flags ───────────────────────────────────────────────────────────────────────────────────────╮ -│--disable-progressDisable progress bar│ -│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ -│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env     │ +│--disable-progressDisable progress bar│ +│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ +│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env     │ │variable set. Can be generated with:                                                    │ │https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status     │ │(TEXT)                                                                                  │ -│--only-available-in-distOnly consider package ids with packages prepared in the dist folder│ +│--only-available-in-distOnly consider package ids with packages prepared in the dist folder│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index 19a549cc5a955..6bf07f9bc7f3d 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -187,28 +187,28 @@ Prepare CHANGELOG, README and COMMITS information for providers. ╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ -│--base-branchBase branch to use as diff for documentation generation (used for releasing from │ +│--base-branchBase branch to use as diff for documentation generation (used for releasing from │ │old branch)                                                                      │ │(TEXT)                                                                           │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ -│--include-removed-providersWhether to include providers that are removed.│ -│--non-interactiveRun in non-interactive mode. Provides random answers to the type of changes and  │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ +│--include-removed-providersWhether to include providers that are removed.│ +│--non-interactiveRun in non-interactive mode. Provides random answers to the type of changes and  │ │confirms releasefor providers prepared for release - useful to test the script in│ │non-interactive mode in CI.                                                      │ -│--only-min-version-updateOnly update minimum version in __init__.py files and regenerate corresponding    │ +│--only-min-version-updateOnly update minimum version in __init__.py files and regenerate corresponding    │ │documentation                                                                    │ -│--reapply-templates-onlyOnly reapply templates, do not bump version. Useful if templates were added and  │ +│--reapply-templates-onlyOnly reapply templates, do not bump version. Useful if templates were added and  │ │you need to regenerate documentation.                                            │ -│--skip-git-fetchSkips removal and recreation of `apache-https-for-providers` remote in git. By   │ +│--skip-git-fetchSkips removal and recreation of `apache-https-for-providers` remote in git. By   │ │default, the remote is recreated and fetched to make sure that it's up to date   │ │and that recent commits are not missing                                          │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg index 16e211ec4202d..3c4e468aba0c7 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg @@ -187,28 +187,28 @@ Prepare sdist/whl packages of Airflow Providers. ╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--clean-distClean dist directory before building packages. Useful when you want to build   │ +│--clean-distClean dist directory before building packages. Useful when you want to build   │ │multiple packages  in a clean environment                                      │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ │[default: apache/airflow]                       │ -│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ -│--include-removed-providersWhether to include providers that are removed.│ -│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ -│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ -│--skip-deleting-generated-filesSkip deleting files that were used to generate provider package. Useful for    │ +│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ +│--include-removed-providersWhether to include providers that are removed.│ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ +│--skip-deleting-generated-filesSkip deleting files that were used to generate provider package. Useful for    │ │debugging and developing changes to the build process.                         │ -│--skip-tag-checkSkip checking if the tag already exists in the remote repository│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--package-listOptional, contains comma-separated list of package ids that are processed for  │ +│--skip-tag-checkSkip checking if the tag already exists in the remote repository│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--package-listOptional, contains comma-separated list of package ids that are processed for  │ │documentation building, and document publishing. It is an easier alternative to│ │adding individual packages as arguments to every command. This overrides the   │ │packages passed as arguments.                                                  │ │(TEXT)                                                                         │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index db974991886e7..7063e3e6fbff0 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -208,33 +208,33 @@ Command to publish generated documentation to airflow-site ╭─ Publish Docs ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(DIRECTORY)[required]│ -│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ -│--include-removed-providersWhether to include providers that are removed.│ -│--override-versioned-sOverrides versioned directories.│ -│--package-filterFilter(s) to use more than one can be specified. You can use glob pattern     │ +│*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(DIRECTORY)[required]│ +│--include-not-ready-providersWhether to include providers that are not yet ready to be released.│ +│--include-removed-providersWhether to include providers that are removed.│ +│--override-versioned-sOverrides versioned directories.│ +│--package-filterFilter(s) to use more than one can be specified. You can use glob pattern     │ │matching the full package name, for example `apache-airflow-providers-*`.     │ │Useful when you want to selectseveral similarly named packages together.      │ │(TEXT)                                                                        │ -│--package-listOptional, contains comma-separated list of package ids that are processed for │ +│--package-listOptional, contains comma-separated list of package ids that are processed for │ │documentation building, and document publishing. It is an easier alternative  │ │to adding individual packages as arguments to every command. This overrides   │ │the packages passed as arguments.                                             │ │(TEXT)                                                                        │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index 4f29325c201c1..4b29f11dd3fd9 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -183,9 +183,9 @@ Generate requirements for selected provider. ╭─ Generate provider requirements flags ───────────────────────────────────────────────────────────────────────────────╮ -│--pythonPython version to update sbom from. (defaults to all historical python versions)│ +│--pythonPython version to update sbom from. (defaults to all historical python versions)│ │(3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12)                                    │ -│--provider-idProvider id to generate the requirements for                                                  │ +│--provider-idProvider id to generate the requirements for                                                  │ │(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid |  │ │apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka |    │ │apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apprise | arangodb |  │ @@ -198,25 +198,25 @@ │| redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |       │ │snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica |       │ │weaviate | yandex | ydb | zendesk)                                                            │ -│--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported   │ +│--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported   │ │value to account for the most recent version of the provider                                  │ │(TEXT)                                                                                        │ -│--forceForce update providers requirements even if they already exist.│ +│--forceForce update providers requirements even if they already exist.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 47e73326a4af2..71feee5875aa1 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -174,7 +174,7 @@ Check that all parameters are put in groups. ╭─ Check all params in groups flags ───────────────────────────────────────────────────────────────────────────────────╮ -│--commandCommand(s) to regenerate images for (optional, might be repeated)                                      │ +│--commandCommand(s) to regenerate images for (optional, might be repeated)                                      │ │(build-docs | ci | ci-image | ci-image:build | ci-image:pull | ci-image:verify |                       │ │ci:find-backtracking-candidates | ci:fix-ownership | ci:free-space | ci:get-workflow-info |            │ │ci:resource-check | ci:selective-check | cleanup | compile-ui-assets | compile-www-assets | down | exec│ @@ -202,9 +202,9 @@ │testing:tests)                                                                                         │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg index bd11568d5a174..83018c3d8d39f 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg @@ -186,8 +186,8 @@ Regenerate breeze command images. ╭─ Image regeneration option ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--forceForces regeneration of all images│ -│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ +│--forceForces regeneration of all images│ +│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ │(build-docs | ci | ci-image | ci-image:build | ci-image:pull | ci-image:verify |                    │ │ci:find-backtracking-candidates | ci:fix-ownership | ci:free-space | ci:get-workflow-info |         │ │ci:resource-check | ci:selective-check | cleanup | compile-ui-assets | compile-www-assets | down |  │ @@ -214,13 +214,13 @@ │| setup:version | shell | start-airflow | static-checks | testing | testing:db-tests |              │ │testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:non-db-tests│ │| testing:tests)                                                                                    │ -│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ -│together with --command flag or --force.                                                            │ +│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ +│together with --command flag or --force.                                                            │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index ade76225d105a..e3c89b304289b 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -351,9 +351,9 @@ │check-persist-credentials-disabled-in-github-workflows |                         │ │check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ │check-provider-docs-valid | check-provider-yaml-valid |                          │ -│check-providers-init-file-missing | check-providers-subpackages-init-file-exist |│ -│check-pydevd-left-in-code | check-revision-heads-map |                           │ -│check-safe-filter-usage-in-html | check-sql-dependency-common-data-structure |   │ +│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ +│check-revision-heads-map | check-safe-filter-usage-in-html |                     │ +│check-sql-dependency-common-data-structure |                                     │ │check-start-date-not-used-in-defaults | check-system-tests-present |             │ │check-system-tests-tocs | check-taskinstance-tis-attrs |                         │ │check-template-context-variable-in-sync | check-template-fields-valid |          │ diff --git a/dev/breeze/doc/images/output_static-checks.txt b/dev/breeze/doc/images/output_static-checks.txt index 0b4fae743bd10..e917996931d60 100644 --- a/dev/breeze/doc/images/output_static-checks.txt +++ b/dev/breeze/doc/images/output_static-checks.txt @@ -1 +1 @@ -769905ba0e5eea7f79c37b2d047699e0 +08a7e37cd651e4d1eb702cb347d9b061 diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index c6cc343ae4aee..f97b6b573a463 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -607,7 +607,7 @@ def _rebuild_k8s_image( COPY --chown=airflow:0 airflow/example_dags/ /opt/airflow/dags/ -COPY --chown=airflow:0 airflow/providers/cncf/kubernetes/kubernetes_executor_templates/ /opt/airflow/pod_templates/ +COPY --chown=airflow:0 providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/ /opt/airflow/pod_templates/ ENV GUNICORN_CMD_ARGS='--preload' AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL=0 """ diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 46ab710dbfa7b..61b260bea09ae 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -791,7 +791,7 @@ def _run_test_command( perform_environment_checks() if skip_providers: ignored_path_list = [ - f"--ignore=tests/providers/{provider_id.replace('.','/')}" + f"--ignore=providers/tests/{provider_id.replace('.','/')}" for provider_id in skip_providers.split(" ") ] extra_pytest_args = (*extra_pytest_args, *ignored_path_list) diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 03873ed8ac7bf..e684acd24ad2d 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -71,7 +71,6 @@ "check-provide-create-sessions-imports", "check-provider-docs-valid", "check-provider-yaml-valid", - "check-providers-init-file-missing", "check-providers-subpackages-init-file-exist", "check-pydevd-left-in-code", "check-revision-heads-map", diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py index 88ad3e8c8cf3a..ebe8940294966 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py @@ -37,7 +37,7 @@ render_template, tag_exists_for_provider, ) -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_SRC, AIRFLOW_SOURCES_ROOT from airflow_breeze.utils.run_utils import run_command LICENCE_RST = """ @@ -83,7 +83,7 @@ def copy_provider_sources_to_target(provider_id: str) -> Path: rmtree(target_provider_root_path, ignore_errors=True) target_provider_root_path.mkdir(parents=True) source_provider_sources_path = get_source_package_path(provider_id) - relative_provider_path = source_provider_sources_path.relative_to(AIRFLOW_SOURCES_ROOT) + relative_provider_path = source_provider_sources_path.relative_to(AIRFLOW_PROVIDERS_SRC) target_providers_sub_folder = target_provider_root_path / relative_provider_path get_console().print( f"[info]Copying provider sources: {source_provider_sources_path} -> {target_providers_sub_folder}" diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index ef7779afd9749..618d064e0e41f 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -23,6 +23,7 @@ import os import re import sys +from functools import lru_cache from subprocess import DEVNULL, CalledProcessError, CompletedProcess from typing import TYPE_CHECKING @@ -91,6 +92,7 @@ ("generated", "/opt/airflow/generated"), ("hooks", "/opt/airflow/hooks"), ("logs", "/root/airflow/logs"), + ("providers", "/opt/airflow/providers"), ("pyproject.toml", "/opt/airflow/pyproject.toml"), ("scripts", "/opt/airflow/scripts"), ("scripts/docker/entrypoint_ci.sh", "/entrypoint"), @@ -502,6 +504,7 @@ def check_executable_entrypoint_permissions(quiet: bool = False): get_console().print("[success]Executable permissions on entrypoints are OK[/]") +@lru_cache def perform_environment_checks(quiet: bool = False): check_docker_is_running() check_docker_version(quiet) diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 5c22c41e0ba6a..5ee2c2a3edfc3 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -36,7 +36,7 @@ ) from airflow_breeze.utils.console import get_console from airflow_breeze.utils.path_utils import ( - AIRFLOW_PROVIDERS_ROOT, + AIRFLOW_PROVIDERS_NS_PACKAGE, BREEZE_SOURCES_ROOT, DOCS_ROOT, GENERATED_PROVIDER_PACKAGES_DIR, @@ -382,7 +382,7 @@ def find_matching_long_package_names( def get_source_package_path(provider_id: str) -> Path: - return AIRFLOW_PROVIDERS_ROOT.joinpath(*provider_id.split(".")) + return AIRFLOW_PROVIDERS_NS_PACKAGE.joinpath(*provider_id.split(".")) def get_documentation_package_path(provider_id: str) -> Path: diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 064a61771fb5b..4e510cb768009 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -281,9 +281,11 @@ def find_airflow_sources_root_to_operate_on() -> Path: AIRFLOW_SOURCES_ROOT = find_airflow_sources_root_to_operate_on().resolve() AIRFLOW_WWW_DIR = AIRFLOW_SOURCES_ROOT / "airflow" / "www" AIRFLOW_UI_DIR = AIRFLOW_SOURCES_ROOT / "airflow" / "ui" -TESTS_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "tests" / "providers" -SYSTEM_TESTS_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "tests" / "system" / "providers" -AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" +AIRFLOW_PROVIDERS_PROJECT = AIRFLOW_SOURCES_ROOT / "providers" +AIRFLOW_PROVIDERS_SRC = AIRFLOW_PROVIDERS_PROJECT / "src" +AIRFLOW_PROVIDERS_NS_PACKAGE = AIRFLOW_PROVIDERS_SRC / "airflow" / "providers" +TESTS_PROVIDERS_ROOT = AIRFLOW_PROVIDERS_PROJECT / "tests" +SYSTEM_TESTS_PROVIDERS_ROOT = AIRFLOW_PROVIDERS_PROJECT / "tests" / "system" DOCS_ROOT = AIRFLOW_SOURCES_ROOT / "docs" BUILD_CACHE_DIR = AIRFLOW_SOURCES_ROOT / ".build" GENERATED_DIR = AIRFLOW_SOURCES_ROOT / "generated" diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py index cad78f1e6d2d9..72d07601601ab 100644 --- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py +++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py @@ -23,7 +23,7 @@ from airflow_breeze.utils.console import get_console from airflow_breeze.utils.github import get_tag_date -from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_ROOT, PROVIDER_DEPENDENCIES_JSON_FILE_PATH +from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_NS_PACKAGE, PROVIDER_DEPENDENCIES_JSON_FILE_PATH DEPENDENCIES = json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text()) @@ -67,7 +67,7 @@ def generate_providers_metadata_for_package( ) -> dict[str, dict[str, str]]: get_console().print(f"[info]Generating metadata for {provider_id}") provider_yaml_dict = yaml.safe_load( - (AIRFLOW_PROVIDERS_ROOT.joinpath(*provider_id.split(".")) / "provider.yaml").read_text() + (AIRFLOW_PROVIDERS_NS_PACKAGE.joinpath(*provider_id.split(".")) / "provider.yaml").read_text() ) provider_metadata: dict[str, dict[str, str]] = {} last_airflow_version = START_AIRFLOW_VERSION_FROM diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py index 8c5d63748cb74..bb5ccb1e82906 100644 --- a/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py @@ -19,14 +19,19 @@ import json import os -from glob import glob from pathlib import Path from typing import Any +from airflow_breeze.utils.path_utils import ( + AIRFLOW_PROVIDERS_NS_PACKAGE, + AIRFLOW_SOURCES_ROOT, + SYSTEM_TESTS_PROVIDERS_ROOT, +) + CONSOLE_WIDTH = 180 -ROOT_DIR = Path(__file__).parents[5].resolve() -PROVIDER_DATA_SCHEMA_PATH = ROOT_DIR / "airflow" / "provider.yaml.schema.json" + +PROVIDER_DATA_SCHEMA_PATH = AIRFLOW_SOURCES_ROOT / "airflow" / "provider.yaml.schema.json" def _load_schema() -> dict[str, Any]: @@ -36,22 +41,17 @@ def _load_schema() -> dict[str, Any]: def _filepath_to_module(filepath: str): - return str(Path(filepath).relative_to(ROOT_DIR)).replace("/", ".") + # TODO: handle relative to providers project + return str(Path(filepath).relative_to(AIRFLOW_SOURCES_ROOT)).replace("/", ".") def _filepath_to_system_tests(filepath: str): - return str( - ROOT_DIR - / "tests" - / "system" - / "providers" - / Path(filepath).relative_to(ROOT_DIR / "airflow" / "providers") - ) + return str(SYSTEM_TESTS_PROVIDERS_ROOT / Path(filepath).relative_to(AIRFLOW_PROVIDERS_NS_PACKAGE)) def get_provider_yaml_paths(): """Returns list of provider.yaml files""" - return sorted(glob(f"{ROOT_DIR}/airflow/providers/**/provider.yaml", recursive=True)) + return sorted(AIRFLOW_PROVIDERS_NS_PACKAGE.glob("**/provider.yaml")) def pretty_format_path(path: str, start: str) -> str: diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 840bf2fcad6d2..23fe5b6e1965e 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -25,7 +25,7 @@ from airflow_breeze.global_constants import PIP_VERSION from airflow_breeze.utils.console import Output, get_console from airflow_breeze.utils.packages import get_excluded_provider_folders, get_suspended_provider_folders -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, TESTS_PROVIDERS_ROOT from airflow_breeze.utils.run_utils import run_command from airflow_breeze.utils.virtualenv_utils import create_temp_venv @@ -113,9 +113,9 @@ def get_ignore_switches_for_provider(provider_folders: list[str]) -> list[str]: for providers in provider_folders: args.extend( [ - f"--ignore=tests/providers/{providers}", - f"--ignore=tests/system/providers/{providers}", - f"--ignore=tests/integration/providers/{providers}", + f"--ignore=providers/tests/{providers}", + f"--ignore=providers/tests/system/{providers}", + f"--ignore=providers/tests/integration/{providers}", ] ) return args @@ -161,7 +161,7 @@ def get_excluded_provider_args(python_version: str) -> list[str]: "tests/operators/test_python.py::TestPythonVirtualenvOperator::test_airflow_context", "--assert=plain", ], - "Providers": ["tests/providers"], + "Providers": ["providers/tests"], "PythonVenv": [ "tests/operators/test_python.py::TestPythonVirtualenvOperator", ], @@ -182,11 +182,10 @@ def get_excluded_provider_args(python_version: str) -> list[str]: NO_RECURSE_DIRS = [ "tests/_internals", "tests/dags_with_system_exit", - "tests/test_utils", "tests/dags_corrupted", "tests/dags", - "tests/system/providers/google/cloud/dataproc/resources", - "tests/system/providers/google/cloud/gcs/resources", + "providers/tests/system/google/cloud/dataproc/resources", + "providers/tests/system/google/cloud/gcs/resources", ] @@ -255,16 +254,16 @@ def convert_test_type_to_pytest_args( providers_with_exclusions = TEST_TYPE_MAP_TO_PYTEST_ARGS["Providers"].copy() for excluded_provider in excluded_provider_list: providers_with_exclusions.append( - "--ignore=tests/providers/" + excluded_provider.replace(".", "/") + "--ignore=providers/tests/" + excluded_provider.replace(".", "/") ) return providers_with_exclusions if test_type.startswith(PROVIDERS_LIST_PREFIX): provider_list = test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",") providers_to_test = [] for provider in provider_list: - provider_path = "tests/providers/" + provider.replace(".", "/") - if (AIRFLOW_SOURCES_ROOT / provider_path).is_dir(): - providers_to_test.append(provider_path) + provider_path = TESTS_PROVIDERS_ROOT.joinpath(provider.replace(".", "/")) + if provider_path.is_dir(): + providers_to_test.append(provider_path.relative_to(AIRFLOW_SOURCES_ROOT).as_posix()) else: get_console().print( f"[error]Provider directory {provider_path} does not exist for {provider}. " @@ -346,9 +345,9 @@ def generate_args_for_pytest( if run_db_tests_only: args.append("--run-db-tests-only") if test_type != "System": - args.append(f"--ignore={SYSTEM_TESTS}") + args.append(f"--ignore-glob=*/{SYSTEM_TESTS}") if test_type != "Integration": - args.append(f"--ignore={INTEGRATION_TESTS}") + args.append(f"--ignore-glob=*/{INTEGRATION_TESTS}") if test_type != "Helm": # do not produce warnings output for helm tests args.append(f"--warning-output-path={warnings_file}") @@ -402,12 +401,12 @@ def convert_parallel_types_to_folders( ) ) # leave only folders, strip --pytest-args that exclude some folders with `-' prefix - folders = [arg for arg in args if arg.startswith("test")] - # remove specific provider sub-folders if "tests/providers" is already in the list + folders = [arg for arg in args if arg.startswith("test") or arg.startswith("providers/tests")] + # remove specific provider sub-folders if "providers/tests" is already in the list # This workarounds pytest issues where it will only run tests from specific subfolders # if both parent and child folders are in the list # The issue in Pytest (changed behaviour in Pytest 8.2 is tracked here # https://github.com/pytest-dev/pytest/issues/12605 - if "tests/providers" in folders: - folders = [folder for folder in folders if not folder.startswith("tests/providers/")] + if "providers/tests" in folders: + folders = [folder for folder in folders if not folder.startswith("providers/tests/")] return folders diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 385cd6f14411a..dfc24993f274a 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -59,7 +59,7 @@ from airflow_breeze.utils.kubernetes_utils import get_kubernetes_python_combos from airflow_breeze.utils.packages import get_available_packages from airflow_breeze.utils.path_utils import ( - AIRFLOW_PROVIDERS_ROOT, + AIRFLOW_PROVIDERS_NS_PACKAGE, AIRFLOW_SOURCES_ROOT, DOCS_DIR, SYSTEM_TESTS_PROVIDERS_ROOT, @@ -177,7 +177,7 @@ def __hash__(self): r"^\.github/SECURITY\.rst$", r"^airflow/.*\.py$", r"^chart", - r"^providers", + r"^providers/src/", r"^tests/system", r"^CHANGELOG\.txt", r"^airflow/config_templates/config\.yml", @@ -200,9 +200,9 @@ def __hash__(self): FileGroupForCi.KUBERNETES_FILES: [ r"^chart", r"^kubernetes_tests", - r"^airflow/providers/cncf/kubernetes/", - r"^tests/providers/cncf/kubernetes/", - r"^tests/system/providers/cncf/kubernetes/", + r"^providers/src/airflow/providers/cncf/kubernetes/", + r"^providers/tests/cncf/kubernetes/", + r"^providers/tests/system/cncf/kubernetes/", ], FileGroupForCi.ALL_PYTHON_FILES: [ r".*\.py$", @@ -211,9 +211,9 @@ def __hash__(self): r".*\.py$", ], FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES: [ - r"^airflow/providers/.*\.py$", - r"^tests/providers/.*\.py$", - r"^tests/system/providers/.*\.py$", + r"^providers/src/airflow/providers/.*\.py$", + r"^providers/tests/.*\.py$", + r"^providers/tests/system/.*\.py$", ], FileGroupForCi.ALL_DOCS_PYTHON_FILES: [ r"^docs/.*\.py$", @@ -225,6 +225,8 @@ def __hash__(self): r"^.pre-commit-config.yaml$", r"^airflow", r"^chart", + r"^providers/src/", + r"^providers/tests/", r"^tests", r"^kubernetes_tests", ], @@ -239,7 +241,7 @@ def __hash__(self): ], FileGroupForCi.TESTS_UTILS_FILES: [ r"^tests/utils/", - r"^tests/test_utils/", + r"^dev/tests_common/.*\.py$", ], } ) @@ -249,12 +251,12 @@ def __hash__(self): FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES: [ r"^.*/.*_vendor/.*", r"^airflow/migrations/.*", - r"^airflow/providers/.*", + r"^providers/src/airflow/providers/.*", r"^dev/.*", r"^docs/.*", r"^provider_packages/.*", - r"^tests/providers/.*", - r"^tests/system/providers/.*", + r"^providers/tests/.*", + r"^providers/tests/system/.*", r"^tests/dags/test_imports.py", ] } @@ -286,9 +288,9 @@ def __hash__(self): r"^tests/operators/", ], SelectiveUnitTestTypes.PROVIDERS: [ - r"^airflow/providers/", - r"^tests/system/providers/", - r"^tests/providers/", + r"^providers/src/airflow/providers/", + r"^providers/tests/system/", + r"^providers/tests/", ], SelectiveUnitTestTypes.SERIALIZATION: [ r"^airflow/serialization/", @@ -308,7 +310,7 @@ def __hash__(self): def find_provider_affected(changed_file: str, include_docs: bool) -> str | None: file_path = AIRFLOW_SOURCES_ROOT / changed_file # is_relative_to is only available in Python 3.9 - we should simplify this check when we are Python 3.9+ - for provider_root in (TESTS_PROVIDERS_ROOT, SYSTEM_TESTS_PROVIDERS_ROOT, AIRFLOW_PROVIDERS_ROOT): + for provider_root in (TESTS_PROVIDERS_ROOT, SYSTEM_TESTS_PROVIDERS_ROOT, AIRFLOW_PROVIDERS_NS_PACKAGE): try: file_path.relative_to(provider_root) relative_base_path = provider_root @@ -329,7 +331,7 @@ def find_provider_affected(changed_file: str, include_docs: bool) -> str | None: if parent_dir_path == relative_base_path: break relative_path = parent_dir_path.relative_to(relative_base_path) - if (AIRFLOW_PROVIDERS_ROOT / relative_path / "provider.yaml").exists(): + if (AIRFLOW_PROVIDERS_NS_PACKAGE / relative_path / "provider.yaml").exists(): return str(parent_dir_path.relative_to(relative_base_path)).replace(os.sep, ".") # If we got here it means that some "common" files were modified. so we need to test all Providers return "Providers" diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index a3126b3f55499..e828f24cf1bb0 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -47,7 +47,7 @@ get_suspended_provider_ids, validate_provider_info_with_runtime_schema, ) -from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_ROOT, AIRFLOW_SOURCES_ROOT, DOCS_ROOT +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCS_ROOT def test_get_available_packages(): @@ -151,7 +151,9 @@ def test_find_matching_long_package_name_bad_filter(): def test_get_source_package_path(): - assert get_source_package_path("apache.hdfs") == AIRFLOW_PROVIDERS_ROOT / "apache" / "hdfs" + assert get_source_package_path("apache.hdfs") == AIRFLOW_SOURCES_ROOT.joinpath( + "providers", "src", "airflow", "providers", "apache", "hdfs" + ) def test_get_documentation_package_path(): @@ -318,9 +320,12 @@ def test_get_provider_details(): assert provider_details.provider_id == "asana" assert provider_details.full_package_name == "airflow.providers.asana" assert provider_details.pypi_package_name == "apache-airflow-providers-asana" - assert ( - provider_details.source_provider_package_path - == AIRFLOW_SOURCES_ROOT / "airflow" / "providers" / "asana" + assert provider_details.source_provider_package_path == AIRFLOW_SOURCES_ROOT.joinpath( + "providers", + "src", + "airflow", + "providers", + "asana", ) assert ( provider_details.documentation_provider_package_path == DOCS_ROOT / "apache-airflow-providers-asana" @@ -496,7 +501,7 @@ def test_provider_jinja_context(): "RELEASE_NO_LEADING_ZEROS": version, "VERSION_SUFFIX": ".rc1", "PROVIDER_DESCRIPTION": "Amazon integration (including `Amazon Web Services (AWS) `__).\n", - "CHANGELOG_RELATIVE_PATH": "../../airflow/providers/amazon", + "CHANGELOG_RELATIVE_PATH": "../../providers/src/airflow/providers/amazon", "SUPPORTED_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12"], "PLUGINS": [], "MIN_AIRFLOW_VERSION": "2.8.0", diff --git a/dev/breeze/tests/test_provider_documentation.py b/dev/breeze/tests/test_provider_documentation.py index e2de9fee9fbf3..db770b7856a2c 100644 --- a/dev/breeze/tests/test_provider_documentation.py +++ b/dev/breeze/tests/test_provider_documentation.py @@ -217,7 +217,7 @@ def test_convert_git_changes_to_table(input: str, output: str, markdown: bool, c def test_verify_changelog_exists(): assert ( _verify_changelog_exists("asana") - == AIRFLOW_SOURCES_ROOT / "airflow" / "providers" / "asana" / "CHANGELOG.rst" + == AIRFLOW_SOURCES_ROOT / "providers" / "src" / "airflow" / "providers" / "asana" / "CHANGELOG.rst" ) diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index 7ecbbf4b5bf3c..de636a9893d27 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -74,7 +74,7 @@ ), ( "Providers", - ["tests/providers"], + ["providers/tests"], False, ), ( @@ -84,26 +84,26 @@ ), ( "Providers[amazon]", - ["tests/providers/amazon"], + ["providers/tests/amazon"], False, ), ( "Providers[common.io]", - ["tests/providers/common/io"], + ["providers/tests/common/io"], False, ), ( "Providers[amazon,google,apache.hive]", - ["tests/providers/amazon", "tests/providers/google", "tests/providers/apache/hive"], + ["providers/tests/amazon", "providers/tests/google", "providers/tests/apache/hive"], False, ), ( "Providers[-amazon,google,microsoft.azure]", [ - "tests/providers", - "--ignore=tests/providers/amazon", - "--ignore=tests/providers/google", - "--ignore=tests/providers/microsoft/azure", + "providers/tests", + "--ignore=providers/tests/amazon", + "--ignore=providers/tests/google", + "--ignore=providers/tests/microsoft/azure", ], False, ), @@ -269,7 +269,7 @@ def test_pytest_args_for_helm_test_types(helm_test_package: str, pytest_args: li "tests/models", "tests/ti_deps", "tests/utils", - "tests/providers", + "providers/tests", ], False, ), @@ -282,7 +282,7 @@ def test_pytest_args_for_helm_test_types(helm_test_package: str, pytest_args: li "tests/models", "tests/ti_deps", "tests/utils", - "tests/providers/amazon", + "providers/tests/amazon", ], False, ), @@ -295,8 +295,8 @@ def test_pytest_args_for_helm_test_types(helm_test_package: str, pytest_args: li "tests/models", "tests/ti_deps", "tests/utils", - "tests/providers/amazon", - "tests/providers/google", + "providers/tests/amazon", + "providers/tests/google", ], False, ), @@ -309,7 +309,7 @@ def test_pytest_args_for_helm_test_types(helm_test_package: str, pytest_args: li "tests/models", "tests/ti_deps", "tests/utils", - "tests/providers", + "providers/tests", ], False, ), @@ -334,7 +334,7 @@ def test_pytest_args_for_helm_test_types(helm_test_package: str, pytest_args: li "tests/models", "tests/ti_deps", "tests/utils", - "tests/providers", + "providers/tests", ], False, ), diff --git a/dev/breeze/tests/test_run_test_args.py b/dev/breeze/tests/test_run_test_args.py index 66d57c8d7822e..58e341e4dfa12 100644 --- a/dev/breeze/tests/test_run_test_args.py +++ b/dev/breeze/tests/test_run_test_args.py @@ -75,7 +75,7 @@ def test_irregular_provider_with_extra_ignore_should_be_valid_cmd(mock_run_comma _run_test( shell_params=ShellParams(test_type="Providers"), - extra_pytest_args=(f"--ignore=tests/providers/{fake_provider_name}",), + extra_pytest_args=(f"--ignore=providers/tests/{fake_provider_name}",), python_version="3.9", output=None, test_timeout=60, @@ -88,12 +88,12 @@ def test_irregular_provider_with_extra_ignore_should_be_valid_cmd(mock_run_comma arg_str = " ".join(run_cmd_call.args[0]) # The command pattern we look for is " \ - # <*other args we don't care about*> --ignore tests/providers/ \ - # --ignore tests/system/providers/ --ignore tests/integration/providers/" + # <*other args we don't care about*> --ignore providers/tests/ \ + # --ignore providers/tests/system/ --ignore providers/tests/integration/" # (the container id is simply to anchor the pattern so we know where we are starting; _run_tests should # be refactored to make arg testing easier but until then we have to regex-test the entire command string match_pattern = re.compile( - f" airflow tests/providers .+ --ignore=tests/providers/{fake_provider_name} --ignore=tests/system/providers/{fake_provider_name} --ignore=tests/integration/providers/{fake_provider_name}" + f" airflow providers/tests .+ --ignore=providers/tests/{fake_provider_name} --ignore=providers/tests/system/{fake_provider_name} --ignore=providers/tests/integration/{fake_provider_name}" ) assert match_pattern.search(arg_str) @@ -104,7 +104,7 @@ def test_primary_test_arg_is_excluded_by_extra_pytest_arg(mock_run_command): test_provider_not_skipped = "ftp" _run_test( shell_params=ShellParams(test_type=f"Providers[{test_provider},{test_provider_not_skipped}]"), - extra_pytest_args=(f"--ignore=tests/providers/{test_provider}",), + extra_pytest_args=(f"--ignore=providers/tests/{test_provider}",), python_version="3.9", output=None, test_timeout=60, @@ -116,13 +116,13 @@ def test_primary_test_arg_is_excluded_by_extra_pytest_arg(mock_run_command): arg_str = " ".join(run_cmd_call.args[0]) # The command pattern we look for is " --verbosity=0 \ - # <*other args we don't care about*> --ignore=tests/providers/" - # The tests/providers/http argument has been eliminated by the code that preps the args; this is a bug, + # <*other args we don't care about*> --ignore=providers/tests/" + # The providers/tests/http argument has been eliminated by the code that preps the args; this is a bug, # bc without a directory or module arg, pytest tests everything (which we don't want!) # We check "--verbosity=0" to ensure nothing is between the airflow container id and the verbosity arg, # IOW that the primary test arg is removed match_pattern = re.compile( - f"airflow tests/providers/{test_provider_not_skipped} --verbosity=0 .+ --ignore=tests/providers/{test_provider}" + f"airflow providers/tests/{test_provider_not_skipped} --verbosity=0 .+ --ignore=providers/tests/{test_provider}" ) assert match_pattern.search(arg_str) @@ -135,7 +135,7 @@ def test_test_is_skipped_if_all_are_ignored(mock_run_command): ] # "Providers[]" scans the source tree so we need to use a real provider id _run_test( shell_params=ShellParams(test_type=f"Providers[{','.join(test_providers)}]"), - extra_pytest_args=[f"--ignore=tests/providers/{provider}" for provider in test_providers], + extra_pytest_args=[f"--ignore=providers/tests/{provider}" for provider in test_providers], python_version="3.9", output=None, test_timeout=60, diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 3b58a45ae1b9a..be3922d461963 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -87,7 +87,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): print_in_color("\nOutput received:") print_in_color(received_output_as_dict) print_in_color() - assert received_value == expected_value + assert received_value == expected_value, f"Correct value for {expected_key!r}" else: print( f"\n[red]ERROR: The key '{expected_key}' missing but " @@ -320,7 +320,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ( "airflow/api/file.py", - "tests/providers/postgres/file.py", + "providers/tests/postgres/file.py", ), { "affected-providers-list-as-string": "amazon common.compat common.sql fab google openlineage " @@ -353,7 +353,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ), ( pytest.param( - ("tests/providers/apache/beam/file.py",), + ("providers/tests/apache/beam/file.py",), { "affected-providers-list-as-string": "apache.beam google", "all-python-versions": "['3.9']", @@ -410,7 +410,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ( "chart/aaaa.txt", - "tests/providers/postgres/file.py", + "providers/tests/postgres/file.py", ), { "affected-providers-list-as-string": "amazon common.sql google openlineage " @@ -445,7 +445,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ( "INTHEWILD.md", "chart/aaaa.txt", - "tests/providers/http/file.py", + "providers/tests/http/file.py", ), { "affected-providers-list-as-string": "amazon apache.livy " @@ -482,7 +482,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ( "INTHEWILD.md", "chart/aaaa.txt", - "tests/providers/airbyte/file.py", + "providers/tests/airbyte/file.py", ), { "affected-providers-list-as-string": "airbyte", @@ -514,7 +514,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ( "INTHEWILD.md", "chart/aaaa.txt", - "tests/system/utils/file.py", + "foo/other.py", ), { "affected-providers-list-as-string": None, @@ -595,7 +595,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ) ), pytest.param( - ("airflow/providers/amazon/__init__.py",), + ("providers/src/airflow/providers/amazon/__init__.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " "common.compat common.sql exasol ftp google http imap microsoft.azure " @@ -623,7 +623,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Providers tests run including amazon tests if amazon provider files changed", ), pytest.param( - ("tests/providers/airbyte/__init__.py",), + ("providers/tests/airbyte/__init__.py",), { "affected-providers-list-as-string": "airbyte", "all-python-versions": "['3.9']", @@ -647,7 +647,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Providers tests run without amazon tests if no amazon file changed", ), pytest.param( - ("airflow/providers/amazon/file.py",), + ("providers/src/airflow/providers/amazon/file.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " "common.compat common.sql exasol ftp google http imap microsoft.azure " @@ -677,8 +677,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ( "tests/always/test_project_structure.py", - "tests/providers/common/io/operators/__init__.py", - "tests/providers/common/io/operators/test_file_transfer.py", + "providers/tests/common/io/operators/__init__.py", + "providers/tests/common/io/operators/test_file_transfer.py", ), { "affected-providers-list-as-string": "common.compat common.io openlineage", @@ -703,9 +703,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Only Always and common providers tests should run when only common.io and tests/always changed", ), pytest.param( - ("airflow/providers/standard/operators/bash.py",), + ("providers/src/airflow/providers/standard/operators/bash.py",), { - "affected-providers-list-as-string": "celery edge standard", + "affected-providers-list-as-string": "standard", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "python-versions": "['3.9']", @@ -720,14 +720,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always Core Providers[celery,edge,standard] Serialization", + "parallel-test-types-list-as-string": "Always Core Providers[standard] Serialization", "needs-mypy": "true", "mypy-folders": "['providers']", }, id="Providers standard tests and Serialization tests to run when airflow bash.py changed", ), pytest.param( - ("tests/operators/bash.py",), + ("providers/tests/standard/operators/bash.py",), { "affected-providers-list-as-string": None, "all-python-versions": "['3.9']", @@ -741,12 +741,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "run-kubernetes-tests": "false", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always Core Operators Serialization", + "parallel-test-types-list-as-string": "Always Core Providers[standard] Serialization", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-folders": "['providers']", }, id="Force Core and Serialization tests to run when tests bash changed", ), @@ -776,6 +776,32 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="All tests should be run when tests/utils/ change", ) ), + ( + pytest.param( + ("dev/tests_common/__init__.py",), + { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "python-versions": "['3.9']", + "python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "run-amazon-tests": "true", + "docs-build": "true", + "full-tests-needed": "true", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "upgrade-to-newer-dependencies": "false", + "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, + "needs-mypy": "true", + "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + }, + id="All tests should be run when dev/tests_common/ change", + ) + ), ], ) def test_expected_output_pull_request_main( @@ -1124,7 +1150,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ ), ( pytest.param( - ("INTHEWILD.md", "tests/providers/asana.py"), + ("INTHEWILD.md", "providers/tests/asana.py"), ("full tests needed",), "v2-7-stable", { @@ -1200,7 +1226,7 @@ def test_expected_output_full_tests_needed( pytest.param( ( "chart/aaaa.txt", - "tests/providers/google/file.py", + "providers/tests/google/file.py", ), { "affected-providers-list-as-string": "amazon apache.beam apache.cassandra cncf.kubernetes " @@ -1229,7 +1255,7 @@ def test_expected_output_full_tests_needed( ( "airflow/cli/test.py", "chart/aaaa.txt", - "tests/providers/google/file.py", + "providers/tests/google/file.py", ), { "affected-providers-list-as-string": "amazon apache.beam apache.cassandra " @@ -1257,7 +1283,7 @@ def test_expected_output_full_tests_needed( pytest.param( ( "airflow/file.py", - "tests/providers/google/file.py", + "providers/tests/google/file.py", ), { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, @@ -1347,7 +1373,7 @@ def test_expected_output_pull_request_v2_7( ( "airflow/cli/test.py", "chart/aaaa.txt", - "tests/providers/google/file.py", + "providers/tests/google/file.py", ), { "affected-providers-list-as-string": "amazon apache.beam apache.cassandra " @@ -1619,7 +1645,7 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): id="pyproject.toml changed but no dependency change", ), pytest.param( - ("airflow/providers/microsoft/azure/provider.yaml",), + ("providers/src/airflow/providers/microsoft/azure/provider.yaml",), { "upgrade-to-newer-dependencies": "false", }, @@ -1677,9 +1703,9 @@ def test_upgrade_to_newer_dependencies( id="Google provider docs changed", ), pytest.param( - ("airflow/providers/common/sql/common_sql_python.py",), + ("providers/src/airflow/providers/common/sql/common_sql_python.py",), { - "docs-list-as-string": "apache-airflow amazon apache.drill apache.druid apache.hive " + "docs-list-as-string": "amazon apache.drill apache.druid apache.hive " "apache.impala apache.pinot common.sql databricks elasticsearch " "exasol google jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", @@ -1719,8 +1745,8 @@ def test_upgrade_to_newer_dependencies( id="Only Airflow docs changed", ), pytest.param( - ("airflow/providers/celery/file.py",), - {"docs-list-as-string": "apache-airflow celery cncf.kubernetes standard"}, + ("providers/src/airflow/providers/celery/file.py",), + {"docs-list-as-string": "celery cncf.kubernetes"}, id="Celery python files changed", ), pytest.param( @@ -2195,7 +2221,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs id="Airflow mypy checks on airflow files with model changes.", ), pytest.param( - ("airflow/providers/a_file.py",), + ("providers/src/airflow/providers/a_file.py",), { "needs-mypy": "true", "mypy-folders": "['providers']", diff --git a/dev/example_dags/update_example_dags_paths.py b/dev/example_dags/update_example_dags_paths.py index 2104721d1bb2b..ac8cf4a0feec4 100755 --- a/dev/example_dags/update_example_dags_paths.py +++ b/dev/example_dags/update_example_dags_paths.py @@ -38,10 +38,10 @@ EXAMPLE_DAGS_URL_MATCHER = re.compile( - r"^(.*)(https://github.com/apache/airflow/tree/(.*)/airflow/providers/(.*)/example_dags)(/?\".*)$" + r"^(.*)(https://github.com/apache/airflow/tree/(.*)/providers/src/airflow/providers/(.*)/example_dags)(/?\".*)$" ) SYSTEM_TESTS_URL_MATCHER = re.compile( - r"^(.*)(https://github.com/apache/airflow/tree/(.*)/tests/system/providers/(.*))(/?\".*)$" + r"^(.*)(https://github.com/apache/airflow/tree/(.*)/providers/tests/system/(.*))(/?\".*)$" ) @@ -67,11 +67,11 @@ def replace_match(file: str, line: str, provider: str, version: str) -> str | No continue system_tests_url = ( f"https://github.com/apache/airflow/tree/providers-{provider}/{version}" - f"/tests/system/providers/{url_path_to_dir}" + f"/providers/tests/system/{url_path_to_dir}" ) example_dags_url = ( f"https://github.com/apache/airflow/tree/providers-{provider}/{version}" - f"/airflow/providers/{url_path_to_dir}/example_dags" + f"/providers/src/airflow/providers/{url_path_to_dir}/example_dags" ) if check_if_url_exists(system_tests_url) and index == 1: new_line = re.sub(matcher, r"\1" + system_tests_url + r"\5", line) diff --git a/dev/perf/scheduler_dag_execution_timing.py b/dev/perf/scheduler_dag_execution_timing.py index b11c73073df92..fc5c21bda7aa7 100755 --- a/dev/perf/scheduler_dag_execution_timing.py +++ b/dev/perf/scheduler_dag_execution_timing.py @@ -107,7 +107,7 @@ def get_executor_under_test(dotted_path): from airflow.executors.executor_loader import ExecutorLoader if dotted_path == "MockExecutor": - from tests.test_utils.mock_executor import MockExecutor as executor + from dev.tests_common.test_utils.mock_executor import MockExecutor as executor else: executor = ExecutorLoader.load_executor(dotted_path) diff --git a/scripts/ci/pre_commit/check_providers_init.py b/dev/tests_common/__init__.py old mode 100755 new mode 100644 similarity index 62% rename from scripts/ci/pre_commit/check_providers_init.py rename to dev/tests_common/__init__.py index 33def71253f34..bf2b7d4f3e2a4 --- a/scripts/ci/pre_commit/check_providers_init.py +++ b/dev/tests_common/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,14 +17,14 @@ # under the License. from __future__ import annotations -import sys +import os from pathlib import Path -AIRFLOW_SOURCES = Path(__file__).parents[3] -PROVIDERS_INIT_FILE = AIRFLOW_SOURCES / "airflow" / "providers" / "__init__.py" - -print(f"Checking if {PROVIDERS_INIT_FILE} exists.") -if PROVIDERS_INIT_FILE.exists(): - print(f"\033[0;31mERROR: {PROVIDERS_INIT_FILE} file should not exist. Deleting it.\033[0m\n") - PROVIDERS_INIT_FILE.unlink() - sys.exit(1) +# This constant is set to True if tests are run with Airflow installed from Packages rather than running +# the tests within Airflow sources. While most tests in CI are run using Airflow sources, there are +# also compatibility tests that only use `tests` package and run against installed packages of Airflow in +# for supported Airflow versions. +RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES = ( + "USE_AIRFLOW_VERSION" in os.environ + or not (Path(__file__).parents[2] / "airflow" / "__init__.py").exists() +) diff --git a/airflow/providers/airbyte/sensors/__init__.py b/dev/tests_common/_internals/__init__.py similarity index 100% rename from airflow/providers/airbyte/sensors/__init__.py rename to dev/tests_common/_internals/__init__.py diff --git a/tests/_internals/capture_warnings.py b/dev/tests_common/_internals/capture_warnings.py similarity index 100% rename from tests/_internals/capture_warnings.py rename to dev/tests_common/_internals/capture_warnings.py diff --git a/tests/_internals/forbidden_warnings.py b/dev/tests_common/_internals/forbidden_warnings.py similarity index 90% rename from tests/_internals/forbidden_warnings.py rename to dev/tests_common/_internals/forbidden_warnings.py index 324d2ff6f9824..ce24878510667 100644 --- a/tests/_internals/forbidden_warnings.py +++ b/dev/tests_common/_internals/forbidden_warnings.py @@ -17,32 +17,43 @@ from __future__ import annotations +import os +from collections.abc import Sequence from pathlib import Path import pytest import yaml -TESTS_DIR = Path(__file__).parents[1].resolve() - class ForbiddenWarningsPlugin: """Internal plugin for restricting warnings during the tests run.""" node_key: str = "forbidden_warnings_node" - deprecations_ignore: Path = (TESTS_DIR / "deprecations_ignore.yml").resolve(strict=True) + deprecations_ignore: Sequence[str | os.PathLike] def __init__(self, config: pytest.Config, forbidden_warnings: tuple[str, ...]): + # Set by a pytest_configure hook in conftest + deprecations_ignore = config.inicfg["airflow_deprecations_ignore"] + if isinstance(deprecations_ignore, (str, os.PathLike)): + self.deprecations_ignore = [deprecations_ignore] + else: + self.deprecations_ignore = deprecations_ignore + excluded_cases = { # Skip: Integration and System Tests "tests/integration/", "tests/system/", + "providers/tests/integration/", + "providers/tests/system/", # Skip: DAGs for tests "tests/dags/", "tests/dags_corrupted/", "tests/dags_with_system_exit/", } - with self.deprecations_ignore.open() as fp: - excluded_cases.update(yaml.safe_load(fp)) + for path in self.deprecations_ignore: + path = Path(path).resolve() + with path.open() as fp: + excluded_cases.update(yaml.safe_load(fp)) self.config = config self.forbidden_warnings = forbidden_warnings diff --git a/dev/tests_common/pyproject.toml b/dev/tests_common/pyproject.toml new file mode 100644 index 0000000000000..092ad7c7c5f5b --- /dev/null +++ b/dev/tests_common/pyproject.toml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "airflow-dev-tests-common" +version = "0.1.0" +description = "" +classifiers = [ + "Private :: Do Not Upload", +] + +[tool.hatch.publish.index] +# Lets make doubly sure this never goes to PyPi +disable = true + +[tool.hatch.build.targets.wheel] +include = ["**/*.py"] + +[tool.hatch.build.targets.wheel.sources] +"" = "dev/tests_common" diff --git a/dev/tests_common/pytest_plugin.py b/dev/tests_common/pytest_plugin.py new file mode 100644 index 0000000000000..5694fe1b3403f --- /dev/null +++ b/dev/tests_common/pytest_plugin.py @@ -0,0 +1,1436 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import json +import os +import platform +import re +import subprocess +import sys +from contextlib import ExitStack, suppress +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +import time_machine + +if TYPE_CHECKING: + from itsdangerous import URLSafeSerializer + + from dev.tests_common._internals.capture_warnings import CaptureWarningsPlugin # noqa: F401 + from dev.tests_common._internals.forbidden_warnings import ForbiddenWarningsPlugin # noqa: F401 + +# https://docs.pytest.org/en/stable/reference/reference.html#stash +capture_warnings_key = pytest.StashKey["CaptureWarningsPlugin"]() +forbidden_warnings_key = pytest.StashKey["ForbiddenWarningsPlugin"]() + +keep_env_variables = "--keep-env-variables" in sys.argv + +if not keep_env_variables: + # Clear all Environment Variables that might have side effect, + # For example, defined in /files/airflow-breeze-config/variables.env + _AIRFLOW_CONFIG_PATTERN = re.compile(r"^AIRFLOW__(.+)__(.+)$") + _KEEP_CONFIGS_SETTINGS: dict[str, dict[str, set[str]]] = { + # Keep always these configurations + "always": { + "database": {"sql_alchemy_conn"}, + "core": {"sql_alchemy_conn"}, + "celery": {"result_backend", "broker_url"}, + }, + # Keep per enabled integrations + "celery": {"celery": {"*"}, "celery_broker_transport_options": {"*"}}, + "kerberos": {"kerberos": {"*"}}, + } + if os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true": + _KEEP_CONFIGS_SETTINGS["always"].update( + { + "core": { + "internal_api_url", + "fernet_key", + "database_access_isolation", + "internal_api_secret_key", + "internal_api_clock_grace", + }, + } + ) + _ENABLED_INTEGRATIONS = {e.split("_", 1)[-1].lower() for e in os.environ if e.startswith("INTEGRATION_")} + _KEEP_CONFIGS: dict[str, set[str]] = {} + for keep_settings_key in ("always", *_ENABLED_INTEGRATIONS): + if keep_settings := _KEEP_CONFIGS_SETTINGS.get(keep_settings_key): + for section, options in keep_settings.items(): + if section not in _KEEP_CONFIGS: + _KEEP_CONFIGS[section] = options + else: + _KEEP_CONFIGS[section].update(options) + for env_key in os.environ.copy(): + if m := _AIRFLOW_CONFIG_PATTERN.match(env_key): + section, option = m.group(1).lower(), m.group(2).lower() + if not (ko := _KEEP_CONFIGS.get(section)) or not ("*" in ko or option in ko): + del os.environ[env_key] + +SUPPORTED_DB_BACKENDS = ("sqlite", "postgres", "mysql") + +# A bit of a Hack - but we need to check args before they are parsed by pytest in order to +# configure the DB before Airflow gets initialized (which happens at airflow import time). +# Using env variables also handles the case, when python-xdist is used - python-xdist spawns separate +# processes and does not pass all args to them (it's done via env variables) so we are doing the +# same here and detect whether `--skip-db-tests` or `--run-db-tests-only` is passed to pytest +# and set env variables so the processes spawned by python-xdist can read the status from there +skip_db_tests = "--skip-db-tests" in sys.argv or os.environ.get("_AIRFLOW_SKIP_DB_TESTS") == "true" +run_db_tests_only = ( + "--run-db-tests-only" in sys.argv or os.environ.get("_AIRFLOW_RUN_DB_TESTS_ONLY") == "true" +) + +if skip_db_tests: + if run_db_tests_only: + raise Exception("You cannot specify both --skip-db-tests and --run-db-tests-only together") + # Make sure sqlalchemy will not be usable for pure unit tests even if initialized + os.environ["AIRFLOW__CORE__SQL_ALCHEMY_CONN"] = "bad_schema:///" + os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"] = "bad_schema:///" + os.environ["_IN_UNIT_TESTS"] = "true" + # Set it here to pass the flag to python-xdist spawned processes + os.environ["_AIRFLOW_SKIP_DB_TESTS"] = "true" + +if run_db_tests_only: + # Set it here to pass the flag to python-xdist spawned processes + os.environ["_AIRFLOW_RUN_DB_TESTS_ONLY"] = "true" + +_airflow_sources = os.getenv("AIRFLOW_SOURCES", None) +AIRFLOW_SOURCES_ROOT_DIR = ( + Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[2] +).resolve() +AIRFLOW_TESTS_DIR = AIRFLOW_SOURCES_ROOT_DIR / "tests" + +os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = os.fspath(AIRFLOW_TESTS_DIR / "plugins") +os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = os.fspath(AIRFLOW_TESTS_DIR / "dags") +os.environ["AIRFLOW__CORE__UNIT_TEST_MODE"] = "True" +os.environ["AWS_DEFAULT_REGION"] = os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" +os.environ["CREDENTIALS_DIR"] = os.environ.get("CREDENTIALS_DIR") or "/files/airflow-breeze-config/keys" +os.environ["AIRFLOW_ENABLE_AIP_44"] = os.environ.get("AIRFLOW_ENABLE_AIP_44") or "true" + +if platform.system() == "Darwin": + # mocks from unittest.mock work correctly in subprocesses only if they are created by "fork" method + # but macOS uses "spawn" by default + os.environ["AIRFLOW__CORE__MP_START_METHOD"] = "fork" + + +@pytest.fixture +def reset_db(): + """Resets Airflow db.""" + + from airflow.utils import db + + db.resetdb() + + +ALLOWED_TRACE_SQL_COLUMNS = ["num", "time", "trace", "sql", "parameters", "count"] + + +@pytest.fixture(autouse=True) +def trace_sql(request): + from dev.tests_common.test_utils.perf.perf_kit.sqlalchemy import ( # isort: skip + count_queries, + trace_queries, + ) + + """Displays queries from the tests to console.""" + trace_sql_option = request.config.option.trace_sql + if not trace_sql_option: + yield + return + + terminal_reporter = request.config.pluginmanager.getplugin("terminalreporter") + # if no terminal reporter plugin is present, nothing we can do here; + # this can happen when this function executes in a worker node + # when using pytest-xdist, for example + if terminal_reporter is None: + yield + return + + columns = [col.strip() for col in trace_sql_option.split(",")] + + def pytest_print(text): + return terminal_reporter.write_line(text) + + with ExitStack() as exit_stack: + if columns == ["num"]: + # It is very unlikely that the user wants to display only numbers, but probably + # the user just wants to count the queries. + exit_stack.enter_context(count_queries(print_fn=pytest_print)) + elif any(c in columns for c in ["time", "trace", "sql", "parameters"]): + exit_stack.enter_context( + trace_queries( + display_num="num" in columns, + display_time="time" in columns, + display_trace="trace" in columns, + display_sql="sql" in columns, + display_parameters="parameters" in columns, + print_fn=pytest_print, + ) + ) + + yield + + +@pytest.fixture(autouse=True, scope="session") +def set_db_isolation_mode(): + if os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true": + from airflow.api_internal.internal_api_call import InternalApiConfig + + InternalApiConfig.set_use_internal_api("tests", allow_tests_to_use_db=True) + + +def skip_if_database_isolation_mode(item): + if os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true": + for _ in item.iter_markers(name="skip_if_database_isolation_mode"): + pytest.skip("This test is skipped because it is not allowed in database isolation mode.") + + +def pytest_addoption(parser: pytest.Parser): + """Add options parser for custom plugins.""" + group = parser.getgroup("airflow") + group.addoption( + "--with-db-init", + action="store_true", + dest="db_init", + help="Forces database initialization before tests", + ) + group.addoption( + "--integration", + action="append", + dest="integration", + metavar="INTEGRATIONS", + help="only run tests matching integration specified: " + "[cassandra,kerberos,mongo,celery,statsd,trino]. ", + ) + group.addoption( + "--keep-env-variables", + action="store_true", + dest="keep_env_variables", + help="do not clear environment variables that might have side effect while running tests", + ) + group.addoption( + "--skip-db-tests", + action="store_true", + dest="skip_db_tests", + help="skip tests that require database", + ) + group.addoption( + "--run-db-tests-only", + action="store_true", + dest="run_db_tests_only", + help="only run tests requiring database", + ) + group.addoption( + "--backend", + action="store", + dest="backend", + metavar="BACKEND", + help="only run tests matching the backend: [sqlite,postgres,mysql].", + ) + group.addoption( + "--system", + action="append", + dest="system", + metavar="SYSTEMS", + help="only run tests matching the system specified [google.cloud, google.marketing_platform]", + ) + group.addoption( + "--include-long-running", + action="store_true", + dest="include_long_running", + help="Includes long running tests (marked with long_running marker). They are skipped by default.", + ) + group.addoption( + "--include-quarantined", + action="store_true", + dest="include_quarantined", + help="Includes quarantined tests (marked with quarantined marker). They are skipped by default.", + ) + group.addoption( + "--exclude-virtualenv-operator", + action="store_true", + dest="exclude_virtualenv_operator", + help="Excludes virtualenv operators tests (marked with virtualenv_test marker).", + ) + group.addoption( + "--exclude-external-python-operator", + action="store_true", + dest="exclude_external_python_operator", + help="Excludes external python operator tests (marked with external_python_test marker).", + ) + allowed_trace_sql_columns_list = ",".join(ALLOWED_TRACE_SQL_COLUMNS) + group.addoption( + "--trace-sql", + action="store", + dest="trace_sql", + help=( + "Trace SQL statements. As an argument, you must specify the columns to be " + f"displayed as a comma-separated list. Supported values: [f{allowed_trace_sql_columns_list}]" + ), + metavar="COLUMNS", + ) + group.addoption( + "--no-db-cleanup", + action="store_false", + dest="db_cleanup", + help="Disable DB clear before each test module.", + ) + group.addoption( + "--disable-forbidden-warnings", + action="store_true", + dest="disable_forbidden_warnings", + help="Disable raising an error if forbidden warnings detected.", + ) + group.addoption( + "--disable-capture-warnings", + action="store_true", + dest="disable_capture_warnings", + help="Disable internal capture warnings.", + ) + group.addoption( + "--warning-output-path", + action="store", + dest="warning_output_path", + metavar="PATH", + help=( + "Path for resulting captured warnings. Absolute or relative to the `tests` directory. " + "If not provided or environment variable `CAPTURE_WARNINGS_OUTPUT` not set " + "then 'warnings.txt' will be used." + ), + ) + parser.addini( + name="forbidden_warnings", + type="linelist", + help="List of internal Airflow warnings which are prohibited during tests execution.", + ) + + +@pytest.fixture(autouse=True, scope="session") +def initialize_airflow_tests(request): + """Helper that setups Airflow testing environment.""" + print(" AIRFLOW ".center(60, "=")) + + from dev.tests_common.test_utils.db import initial_db_init + + # Setup test environment for breeze + home = os.path.expanduser("~") + airflow_home = os.environ.get("AIRFLOW_HOME") or os.path.join(home, "airflow") + + print(f"Home of the user: {home}\nAirflow home {airflow_home}") + + # Initialize Airflow db if required + lock_file = os.path.join(airflow_home, ".airflow_db_initialised") + if not skip_db_tests: + if request.config.option.db_init: + from dev.tests_common.test_utils.db import initial_db_init + + print("Initializing the DB - forced with --with-db-init switch.") + initial_db_init() + elif not os.path.exists(lock_file): + print( + "Initializing the DB - first time after entering the container.\n" + "You can force re-initialization the database by adding --with-db-init switch to run-tests." + ) + initial_db_init() + # Create pid file + with open(lock_file, "w+"): + pass + else: + print( + "Skipping initializing of the DB as it was initialized already.\n" + "You can re-initialize the database by adding --with-db-init flag when running tests." + ) + integration_kerberos = os.environ.get("INTEGRATION_KERBEROS") + if integration_kerberos == "true": + # Initialize kerberos + kerberos = os.environ.get("KRB5_KTNAME") + if kerberos: + subprocess.check_call(["kinit", "-kt", kerberos, "bob@EXAMPLE.COM"]) + else: + print("Kerberos enabled! Please setup KRB5_KTNAME environment variable") + sys.exit(1) + + +def pytest_configure(config: pytest.Config) -> None: + # Ensure that the airflow sources dir is at the end of the sys path if it's not already there. Needed to + # run import from `providers/tests/` + desired = AIRFLOW_SOURCES_ROOT_DIR.as_posix() + for path in sys.path: + if path == desired: + break + else: + sys.path.append(desired) + + if (backend := config.getoption("backend", default=None)) and backend not in SUPPORTED_DB_BACKENDS: + msg = ( + f"Provided DB backend {backend!r} not supported, " + f"expected one of: {', '.join(map(repr, SUPPORTED_DB_BACKENDS))}" + ) + pytest.exit(msg, returncode=6) + + config.addinivalue_line("markers", "integration(name): mark test to run with named integration") + config.addinivalue_line("markers", "backend(name): mark test to run with named backend") + config.addinivalue_line("markers", "system(name): mark test to run with named system") + config.addinivalue_line("markers", "platform(name): mark test to run with specific platform/environment") + config.addinivalue_line("markers", "long_running: mark test that run for a long time (many minutes)") + config.addinivalue_line( + "markers", "quarantined: mark test that are in quarantine (i.e. flaky, need to be isolated and fixed)" + ) + config.addinivalue_line( + "markers", "credential_file(name): mark tests that require credential file in CREDENTIALS_DIR" + ) + config.addinivalue_line( + "markers", "need_serialized_dag: mark tests that require dags in serialized form to be present" + ) + config.addinivalue_line( + "markers", + "db_test: mark tests that require database to be present", + ) + config.addinivalue_line( + "markers", + "non_db_test_override: you can mark individual tests with this marker to override the db_test marker", + ) + config.addinivalue_line( + "markers", + "virtualenv_operator: virtualenv operator tests are 'long', we should run them separately", + ) + config.addinivalue_line( + "markers", + "external_python_operator: external python operator tests are 'long', we should run them separately", + ) + config.addinivalue_line("markers", "enable_redact: do not mock redact secret masker") + config.addinivalue_line("markers", "skip_if_database_isolation_mode: skip if DB isolation is enabled") + + os.environ["_AIRFLOW__SKIP_DATABASE_EXECUTOR_COMPATIBILITY_CHECK"] = "1" + + # Setup internal warnings plugins + if "ignore" in sys.warnoptions: + config.option.disable_forbidden_warnings = True + config.option.disable_capture_warnings = True + if not config.pluginmanager.get_plugin("warnings"): + # Internal forbidden warnings plugin depends on builtin pytest warnings plugin + config.option.disable_forbidden_warnings = True + + forbidden_warnings: list[str] | None = config.getini("forbidden_warnings") + if not config.option.disable_forbidden_warnings and forbidden_warnings: + from dev.tests_common._internals.forbidden_warnings import ForbiddenWarningsPlugin + + forbidden_warnings_plugin = ForbiddenWarningsPlugin( + config=config, + forbidden_warnings=tuple(map(str.strip, forbidden_warnings)), + ) + config.pluginmanager.register(forbidden_warnings_plugin) + config.stash[forbidden_warnings_key] = forbidden_warnings_plugin + + if not config.option.disable_capture_warnings: + from dev.tests_common._internals.capture_warnings import CaptureWarningsPlugin + + capture_warnings_plugin = CaptureWarningsPlugin( + config=config, output_path=config.getoption("warning_output_path", default=None) + ) + config.pluginmanager.register(capture_warnings_plugin) + config.stash[capture_warnings_key] = capture_warnings_plugin + + +def pytest_unconfigure(config: pytest.Config) -> None: + os.environ.pop("_AIRFLOW__SKIP_DATABASE_EXECUTOR_COMPATIBILITY_CHECK", None) + if forbidden_warnings_plugin := config.stash.get(forbidden_warnings_key, None): + del config.stash[forbidden_warnings_key] + config.pluginmanager.unregister(forbidden_warnings_plugin) + if capture_warnings_plugin := config.stash.get(capture_warnings_key, None): + del config.stash[capture_warnings_key] + config.pluginmanager.unregister(capture_warnings_plugin) + + +def skip_if_not_marked_with_integration(selected_integrations, item): + for marker in item.iter_markers(name="integration"): + integration_name = marker.args[0] + if integration_name in selected_integrations or "all" in selected_integrations: + return + pytest.skip( + f"The test is skipped because it does not have the right integration marker. " + f"Only tests marked with pytest.mark.integration(INTEGRATION) are run with INTEGRATION " + f"being one of {selected_integrations}. {item}" + ) + + +def skip_if_not_marked_with_backend(selected_backend, item): + for marker in item.iter_markers(name="backend"): + backend_names = marker.args + if selected_backend in backend_names: + return + pytest.skip( + f"The test is skipped because it does not have the right backend marker. " + f"Only tests marked with pytest.mark.backend('{selected_backend}') are run: {item}" + ) + + +def skip_if_platform_doesnt_match(marker): + allowed_platforms = ("linux", "breeze") + if not (args := marker.args): + pytest.fail(f"No platform specified, expected one of: {', '.join(map(repr, allowed_platforms))}") + elif not all(a in allowed_platforms for a in args): + pytest.fail( + f"Allowed platforms {', '.join(map(repr, allowed_platforms))}; " + f"but got: {', '.join(map(repr, args))}" + ) + if "linux" in args: + if not sys.platform.startswith("linux"): + pytest.skip("Test expected to run on Linux platform.") + if "breeze" in args: + if not os.path.isfile("/.dockerenv") or os.environ.get("BREEZE", "").lower() != "true": + raise pytest.skip( + "Test expected to run into Airflow Breeze container. " + "Maybe because it is to dangerous to run it outside." + ) + + +def skip_if_not_marked_with_system(selected_systems, item): + for marker in item.iter_markers(name="system"): + systems_name = marker.args[0] + if systems_name in selected_systems or "all" in selected_systems: + return + pytest.skip( + f"The test is skipped because it does not have the right system marker. " + f"Only tests marked with pytest.mark.system(SYSTEM) are run with SYSTEM " + f"being one of {selected_systems}. {item}" + ) + + +def skip_system_test(item): + for marker in item.iter_markers(name="system"): + pytest.skip( + f"The test is skipped because it has system marker. System tests are only run when " + f"--system flag with the right system ({marker.args[0]}) is passed to pytest. {item}" + ) + + +def skip_long_running_test(item): + for _ in item.iter_markers(name="long_running"): + pytest.skip( + f"The test is skipped because it has long_running marker. " + f"And --include-long-running flag is not passed to pytest. {item}" + ) + + +def skip_quarantined_test(item): + for _ in item.iter_markers(name="quarantined"): + pytest.skip( + f"The test is skipped because it has quarantined marker. " + f"And --include-quarantined flag is not passed to pytest. {item}" + ) + + +def skip_virtualenv_operator_test(item): + for _ in item.iter_markers(name="virtualenv_operator"): + pytest.skip( + f"The test is skipped because it has virtualenv_operator marker. " + f"And --exclude-virtualenv-operator flag is not passed to pytest. {item}" + ) + + +def skip_external_python_operator_test(item): + for _ in item.iter_markers(name="external_python_operator"): + pytest.skip( + f"The test is skipped because it has external_python_operator marker. " + f"And --exclude-external-python-operator flag is not passed to pytest. {item}" + ) + + +def skip_db_test(item): + if next(item.iter_markers(name="db_test"), None): + if next(item.iter_markers(name="non_db_test_override"), None): + # non_db_test can override the db_test set for example on module or class level + return + else: + pytest.skip( + f"The test is skipped as it is DB test " + f"and --skip-db-tests is flag is passed to pytest. {item}" + ) + if next(item.iter_markers(name="backend"), None): + # also automatically skip tests marked with `backend` marker as they are implicitly + # db tests + pytest.skip( + f"The test is skipped as it is DB test " + f"and --skip-db-tests is flag is passed to pytest. {item}" + ) + + +def only_run_db_test(item): + if next(item.iter_markers(name="db_test"), None) and not next( + item.iter_markers(name="non_db_test_override"), None + ): + # non_db_test at individual level can override the db_test set for example on module or class level + return + else: + if next(item.iter_markers(name="backend"), None): + # Also do not skip the tests marked with `backend` marker - as it is implicitly a db test + return + pytest.skip( + f"The test is skipped as it is not a DB tests " + f"and --run-db-tests-only flag is passed to pytest. {item}" + ) + + +def skip_if_integration_disabled(marker, item): + integration_name = marker.args[0] + environment_variable_name = "INTEGRATION_" + integration_name.upper() + environment_variable_value = os.environ.get(environment_variable_name) + if not environment_variable_value or environment_variable_value != "true": + pytest.skip( + f"The test requires {integration_name} integration started and " + f"{environment_variable_name} environment variable to be set to true (it is '{environment_variable_value}')." + f" It can be set by specifying '--integration {integration_name}' at breeze startup" + f": {item}" + ) + + +def skip_if_wrong_backend(marker: pytest.Mark, item: pytest.Item) -> None: + if not (backend_names := marker.args): + reason = ( + "`pytest.mark.backend` expect to get at least one of the following backends: " + f"{', '.join(map(repr, SUPPORTED_DB_BACKENDS))}." + ) + pytest.fail(reason) + elif unsupported_backends := list(filter(lambda b: b not in SUPPORTED_DB_BACKENDS, backend_names)): + reason = ( + "Airflow Tests supports only the following backends in `pytest.mark.backend` marker: " + f"{', '.join(map(repr, SUPPORTED_DB_BACKENDS))}, " + f"but got {', '.join(map(repr, unsupported_backends))}." + ) + pytest.fail(reason) + + env_name = "BACKEND" + if not (backend := os.environ.get(env_name)) or backend not in backend_names: + reason = ( + f"The test {item.nodeid!r} requires one of {', '.join(map(repr, backend_names))} backend started " + f"and {env_name!r} environment variable to be set (currently it set to {backend!r}). " + f"It can be set by specifying backend at breeze startup." + ) + pytest.skip(reason) + + +def skip_if_credential_file_missing(item): + for marker in item.iter_markers(name="credential_file"): + credential_file = marker.args[0] + credential_path = os.path.join(os.environ.get("CREDENTIALS_DIR"), credential_file) + if not os.path.exists(credential_path): + pytest.skip(f"The test requires credential file {credential_path}: {item}") + + +def pytest_runtest_setup(item): + selected_integrations_list = item.config.option.integration + selected_systems_list = item.config.option.system + + include_long_running = item.config.option.include_long_running + include_quarantined = item.config.option.include_quarantined + exclude_virtualenv_operator = item.config.option.exclude_virtualenv_operator + exclude_external_python_operator = item.config.option.exclude_external_python_operator + + for marker in item.iter_markers(name="integration"): + skip_if_integration_disabled(marker, item) + if selected_integrations_list: + skip_if_not_marked_with_integration(selected_integrations_list, item) + if selected_systems_list: + skip_if_not_marked_with_system(selected_systems_list, item) + else: + skip_system_test(item) + for marker in item.iter_markers(name="platform"): + skip_if_platform_doesnt_match(marker) + for marker in item.iter_markers(name="backend"): + skip_if_wrong_backend(marker, item) + skip_if_database_isolation_mode(item) + selected_backend = item.config.option.backend + if selected_backend: + skip_if_not_marked_with_backend(selected_backend, item) + if not include_long_running: + skip_long_running_test(item) + if not include_quarantined: + skip_quarantined_test(item) + if exclude_virtualenv_operator: + skip_virtualenv_operator_test(item) + if exclude_external_python_operator: + skip_external_python_operator_test(item) + if skip_db_tests: + skip_db_test(item) + if run_db_tests_only: + only_run_db_test(item) + skip_if_credential_file_missing(item) + + +@pytest.fixture +def frozen_sleep(monkeypatch): + """Use time-machine to "stub" sleep. + + This means the ``sleep()`` takes no time, but ``datetime.now()`` appears to move forwards. + + If your module under test does ``import time`` and then ``time.sleep``: + + .. code-block:: python + + def test_something(frozen_sleep): + my_mod.fn_under_test() + + If your module under test does ``from time import sleep`` then you will + have to mock that sleep function directly: + + .. code-block:: python + + def test_something(frozen_sleep, monkeypatch): + monkeypatch.setattr("my_mod.sleep", frozen_sleep) + my_mod.fn_under_test() + """ + traveller = None + + def fake_sleep(seconds): + nonlocal traveller + utcnow = datetime.now(tz=timezone.utc) + if traveller is not None: + traveller.stop() + traveller = time_machine.travel(utcnow + timedelta(seconds=seconds)) + traveller.start() + + monkeypatch.setattr("time.sleep", fake_sleep) + yield fake_sleep + + if traveller is not None: + traveller.stop() + + +@pytest.fixture +def dag_maker(request): + """Fixture to help create DAG, DagModel, and SerializedDAG automatically. + + You have to use the dag_maker as a context manager and it takes + the same argument as DAG:: + + with dag_maker(dag_id="mydag") as dag: + task1 = EmptyOperator(task_id="mytask") + task2 = EmptyOperator(task_id="mytask2") + + If the DagModel you want to use needs different parameters than the one + automatically created by the dag_maker, you have to update the DagModel as below:: + + dag_maker.dag_model.is_active = False + session.merge(dag_maker.dag_model) + session.commit() + + For any test you use the dag_maker, make sure to create a DagRun:: + + dag_maker.create_dagrun() + + The dag_maker.create_dagrun takes the same arguments as dag.create_dagrun + + If you want to operate on serialized DAGs, then either pass + ``serialized=True`` to the ``dag_maker()`` call, or you can mark your + test/class/file with ``@pytest.mark.need_serialized_dag(True)``. In both of + these cases the ``dag`` returned by the context manager will be a + lazily-evaluated proxy object to the SerializedDAG. + """ + import lazy_object_proxy + + # IMPORTANT: Delay _all_ imports from `airflow.*` to _inside a method_. + # This fixture is "called" early on in the pytest collection process, and + # if we import airflow.* here the wrong (non-test) config will be loaded + # and "baked" in to various constants + + want_serialized = False + + # Allow changing default serialized behaviour with `@pytest.mark.need_serialized_dag` or + # `@pytest.mark.need_serialized_dag(False)` + serialized_marker = request.node.get_closest_marker("need_serialized_dag") + if serialized_marker: + (want_serialized,) = serialized_marker.args or (True,) + + from airflow.utils.log.logging_mixin import LoggingMixin + + class DagFactory(LoggingMixin): + _own_session = False + + def __init__(self): + from airflow.models import DagBag + + # Keep all the serialized dags we've created in this test + self.dagbag = DagBag(os.devnull, include_examples=False, read_dags_from_db=False) + + def __enter__(self): + self.dag.__enter__() + if self.want_serialized: + return lazy_object_proxy.Proxy(self._serialized_dag) + return self.dag + + def _serialized_dag(self): + return self.serialized_model.dag + + def get_serialized_data(self): + try: + data = self.serialized_model.data + except AttributeError: + raise RuntimeError("DAG serialization not requested") + if isinstance(data, str): + return json.loads(data) + return data + + def _bag_dag_compat(self, dag): + # This is a compatibility shim for the old bag_dag method in Airflow <3.0 + # TODO: Remove this when we drop support for Airflow <3.0 in Providers + if hasattr(dag, "parent_dag"): + return self.dagbag.bag_dag(dag, root_dag=dag) + return self.dagbag.bag_dag(dag) + + def __exit__(self, type, value, traceback): + from airflow.models import DagModel + from airflow.models.serialized_dag import SerializedDagModel + + dag = self.dag + dag.__exit__(type, value, traceback) + if type is not None: + return + + dag.clear(session=self.session) + dag.sync_to_db(processor_subdir=self.processor_subdir, session=self.session) + self.dag_model = self.session.get(DagModel, dag.dag_id) + + if self.want_serialized: + self.serialized_model = SerializedDagModel( + dag, processor_subdir=self.dag_model.processor_subdir + ) + self.session.merge(self.serialized_model) + serialized_dag = self._serialized_dag() + self._bag_dag_compat(serialized_dag) + self.session.flush() + else: + self._bag_dag_compat(self.dag) + + def create_dagrun(self, **kwargs): + from airflow.utils import timezone + from airflow.utils.state import State + from airflow.utils.types import DagRunType + + from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + + if AIRFLOW_V_3_0_PLUS: + from airflow.utils.types import DagRunTriggeredByType + + dag = self.dag + kwargs = { + "state": State.RUNNING, + "start_date": self.start_date, + "session": self.session, + **kwargs, + } + # Need to provide run_id if the user does not either provide one + # explicitly, or pass run_type for inference in dag.create_dagrun(). + if "run_id" not in kwargs and "run_type" not in kwargs: + kwargs["run_id"] = "test" + + if "run_type" not in kwargs: + kwargs["run_type"] = DagRunType.from_run_id(kwargs["run_id"]) + if kwargs.get("execution_date") is None: + if kwargs["run_type"] == DagRunType.MANUAL: + kwargs["execution_date"] = self.start_date + else: + kwargs["execution_date"] = dag.next_dagrun_info(None).logical_date + if "data_interval" not in kwargs: + logical_date = timezone.coerce_datetime(kwargs["execution_date"]) + if kwargs["run_type"] == DagRunType.MANUAL: + data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date) + else: + data_interval = dag.infer_automated_data_interval(logical_date) + kwargs["data_interval"] = data_interval + if AIRFLOW_V_3_0_PLUS and "triggered_by" not in kwargs: + kwargs["triggered_by"] = DagRunTriggeredByType.TEST + + self.dag_run = dag.create_dagrun(**kwargs) + for ti in self.dag_run.task_instances: + ti.refresh_from_task(dag.get_task(ti.task_id)) + if self.want_serialized: + self.session.commit() + return self.dag_run + + def create_dagrun_after(self, dagrun, **kwargs): + next_info = self.dag.next_dagrun_info(self.dag.get_run_data_interval(dagrun)) + if next_info is None: + raise ValueError(f"cannot create run after {dagrun}") + return self.create_dagrun( + execution_date=next_info.logical_date, + data_interval=next_info.data_interval, + **kwargs, + ) + + def __call__( + self, + dag_id="test_dag", + schedule=timedelta(days=1), + serialized=want_serialized, + fileloc=None, + processor_subdir=None, + session=None, + **kwargs, + ): + from airflow import settings + from airflow.models.dag import DAG + from airflow.utils import timezone + + if session is None: + self._own_session = True + session = settings.Session() + + self.kwargs = kwargs + self.session = session + self.start_date = self.kwargs.get("start_date", None) + default_args = kwargs.get("default_args", None) + if default_args and not self.start_date: + if "start_date" in default_args: + self.start_date = default_args.get("start_date") + if not self.start_date: + if hasattr(request.module, "DEFAULT_DATE"): + self.start_date = getattr(request.module, "DEFAULT_DATE") + else: + DEFAULT_DATE = timezone.datetime(2016, 1, 1) + self.start_date = DEFAULT_DATE + self.kwargs["start_date"] = self.start_date + # Set schedule argument to explicitly set value, or a default if no + # other scheduling arguments are set. + self.dag = DAG(dag_id, schedule=schedule, **self.kwargs) + self.dag.fileloc = fileloc or request.module.__file__ + self.want_serialized = serialized + self.processor_subdir = processor_subdir + + return self + + def cleanup(self): + from airflow.models import DagModel, DagRun, TaskInstance, XCom + from airflow.models.serialized_dag import SerializedDagModel + from airflow.models.taskmap import TaskMap + from airflow.utils.retries import run_with_db_retries + + from dev.tests_common.test_utils.compat import AssetEvent + + for attempt in run_with_db_retries(logger=self.log): + with attempt: + dag_ids = list(self.dagbag.dag_ids) + if not dag_ids: + return + # To isolate problems here with problems from elsewhere on the session object + self.session.rollback() + + self.session.query(SerializedDagModel).filter( + SerializedDagModel.dag_id.in_(dag_ids) + ).delete(synchronize_session=False) + self.session.query(DagRun).filter(DagRun.dag_id.in_(dag_ids)).delete( + synchronize_session=False, + ) + self.session.query(TaskInstance).filter(TaskInstance.dag_id.in_(dag_ids)).delete( + synchronize_session=False, + ) + self.session.query(XCom).filter(XCom.dag_id.in_(dag_ids)).delete( + synchronize_session=False, + ) + self.session.query(DagModel).filter(DagModel.dag_id.in_(dag_ids)).delete( + synchronize_session=False, + ) + self.session.query(TaskMap).filter(TaskMap.dag_id.in_(dag_ids)).delete( + synchronize_session=False, + ) + self.session.query(AssetEvent).filter(AssetEvent.source_dag_id.in_(dag_ids)).delete( + synchronize_session=False, + ) + self.session.commit() + if self._own_session: + self.session.expunge_all() + + factory = DagFactory() + + try: + yield factory + finally: + factory.cleanup() + with suppress(AttributeError): + del factory.session + + +@pytest.fixture +def create_dummy_dag(dag_maker): + """Create a `DAG` with a single `EmptyOperator` task. + + DagRun and DagModel is also created. + + Apart from the already existing arguments, any other argument in kwargs + is passed to the DAG and not to the EmptyOperator task. + + If you have an argument that you want to pass to the EmptyOperator that + is not here, please use `default_args` so that the DAG will pass it to the + Task:: + + dag, task = create_dummy_dag(default_args={"start_date": timezone.datetime(2016, 1, 1)}) + + You cannot be able to alter the created DagRun or DagModel, use `dag_maker` fixture instead. + """ + from airflow.operators.empty import EmptyOperator + from airflow.utils.types import DagRunType + + def create_dag( + dag_id="dag", + task_id="op1", + task_display_name=None, + max_active_tis_per_dag=16, + max_active_tis_per_dagrun=None, + pool="default_pool", + executor_config=None, + trigger_rule="all_done", + on_success_callback=None, + on_execute_callback=None, + on_failure_callback=None, + on_retry_callback=None, + email=None, + with_dagrun_type=DagRunType.SCHEDULED, + **kwargs, + ): + op_kwargs = {} + from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + + if AIRFLOW_V_2_9_PLUS: + op_kwargs["task_display_name"] = task_display_name + with dag_maker(dag_id, **kwargs) as dag: + op = EmptyOperator( + task_id=task_id, + max_active_tis_per_dag=max_active_tis_per_dag, + max_active_tis_per_dagrun=max_active_tis_per_dagrun, + executor_config=executor_config or {}, + on_success_callback=on_success_callback, + on_execute_callback=on_execute_callback, + on_failure_callback=on_failure_callback, + on_retry_callback=on_retry_callback, + email=email, + pool=pool, + trigger_rule=trigger_rule, + **op_kwargs, + ) + if with_dagrun_type is not None: + dag_maker.create_dagrun(run_type=with_dagrun_type) + return dag, op + + return create_dag + + +if TYPE_CHECKING: + from airflow.models.taskinstance import TaskInstance + + +@pytest.fixture +def create_task_instance(dag_maker, create_dummy_dag): + """Create a TaskInstance, and associated DB rows (DagRun, DagModel, etc). + + Uses ``create_dummy_dag`` to create the dag structure. + """ + from airflow.operators.empty import EmptyOperator + + def maker( + execution_date=None, + dagrun_state=None, + state=None, + run_id=None, + run_type=None, + data_interval=None, + external_executor_id=None, + dag_id="dag", + task_id="op1", + task_display_name=None, + max_active_tis_per_dag=16, + max_active_tis_per_dagrun=None, + pool="default_pool", + executor_config=None, + trigger_rule="all_done", + on_success_callback=None, + on_execute_callback=None, + on_failure_callback=None, + on_retry_callback=None, + email=None, + map_index=-1, + **kwargs, + ) -> TaskInstance: + from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + + if AIRFLOW_V_3_0_PLUS: + from airflow.utils.types import DagRunTriggeredByType + + if execution_date is None: + from airflow.utils import timezone + + execution_date = timezone.utcnow() + with dag_maker(dag_id, **kwargs): + op_kwargs = {} + from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + + if AIRFLOW_V_2_9_PLUS: + op_kwargs["task_display_name"] = task_display_name + task = EmptyOperator( + task_id=task_id, + max_active_tis_per_dag=max_active_tis_per_dag, + max_active_tis_per_dagrun=max_active_tis_per_dagrun, + executor_config=executor_config or {}, + on_success_callback=on_success_callback, + on_execute_callback=on_execute_callback, + on_failure_callback=on_failure_callback, + on_retry_callback=on_retry_callback, + email=email, + pool=pool, + trigger_rule=trigger_rule, + **op_kwargs, + ) + + dagrun_kwargs = { + "execution_date": execution_date, + "state": dagrun_state, + } + dagrun_kwargs.update({"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {}) + if run_id is not None: + dagrun_kwargs["run_id"] = run_id + if run_type is not None: + dagrun_kwargs["run_type"] = run_type + if data_interval is not None: + dagrun_kwargs["data_interval"] = data_interval + dagrun = dag_maker.create_dagrun(**dagrun_kwargs) + (ti,) = dagrun.task_instances + ti.task = task + ti.state = state + ti.external_executor_id = external_executor_id + ti.map_index = map_index + + dag_maker.session.flush() + return ti + + return maker + + +@pytest.fixture +def create_serialized_task_instance_of_operator(dag_maker): + def _create_task_instance( + operator_class, + *, + dag_id, + execution_date=None, + session=None, + **operator_kwargs, + ) -> TaskInstance: + with dag_maker(dag_id=dag_id, serialized=True, session=session): + operator_class(**operator_kwargs) + if execution_date is None: + dagrun_kwargs = {} + else: + dagrun_kwargs = {"execution_date": execution_date} + (ti,) = dag_maker.create_dagrun(**dagrun_kwargs).task_instances + return ti + + return _create_task_instance + + +@pytest.fixture +def create_task_instance_of_operator(dag_maker): + def _create_task_instance( + operator_class, + *, + dag_id, + execution_date=None, + session=None, + **operator_kwargs, + ) -> TaskInstance: + with dag_maker(dag_id=dag_id, session=session, serialized=True): + operator_class(**operator_kwargs) + if execution_date is None: + dagrun_kwargs = {} + else: + dagrun_kwargs = {"execution_date": execution_date} + (ti,) = dag_maker.create_dagrun(**dagrun_kwargs).task_instances + return ti + + return _create_task_instance + + +@pytest.fixture +def create_task_of_operator(dag_maker): + def _create_task_of_operator(operator_class, *, dag_id, session=None, **operator_kwargs): + with dag_maker(dag_id=dag_id, session=session): + task = operator_class(**operator_kwargs) + return task + + return _create_task_of_operator + + +@pytest.fixture +def session(): + from airflow.utils.session import create_session + + with create_session() as session: + yield session + session.rollback() + + +@pytest.fixture +def get_test_dag(): + def _get(dag_id: str): + from airflow.models.dagbag import DagBag + from airflow.models.serialized_dag import SerializedDagModel + + dag_file = AIRFLOW_TESTS_DIR / "dags" / f"{dag_id}.py" + dagbag = DagBag(dag_folder=dag_file, include_examples=False) + + dag = dagbag.get_dag(dag_id) + dag.sync_to_db() + SerializedDagModel.write_dag(dag) + + return dag + + return _get + + +@pytest.fixture +def create_log_template(request): + from airflow import settings + from airflow.models.tasklog import LogTemplate + + session = settings.Session() + + def _create_log_template(filename_template, elasticsearch_id=""): + log_template = LogTemplate(filename=filename_template, elasticsearch_id=elasticsearch_id) + session.add(log_template) + session.commit() + + def _delete_log_template(): + from airflow.models import DagRun, TaskInstance + + session.query(TaskInstance).delete() + session.query(DagRun).delete() + session.delete(log_template) + session.commit() + + request.addfinalizer(_delete_log_template) + + return _create_log_template + + +@pytest.fixture +def reset_logging_config(): + import logging.config + + from airflow import settings + from airflow.utils.module_loading import import_string + + logging_config = import_string(settings.LOGGING_CLASS_PATH) + logging.config.dictConfig(logging_config) + + +@pytest.fixture(scope="session", autouse=True) +def suppress_info_logs_for_dag_and_fab(): + import logging + + from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + + dag_logger = logging.getLogger("airflow.models.dag") + dag_logger.setLevel(logging.WARNING) + + if AIRFLOW_V_2_9_PLUS: + fab_logger = logging.getLogger("airflow.providers.fab.auth_manager.security_manager.override") + fab_logger.setLevel(logging.WARNING) + else: + fab_logger = logging.getLogger("airflow.www.fab_security") + fab_logger.setLevel(logging.WARNING) + + +@pytest.fixture(scope="module", autouse=True) +def _clear_db(request): + """Clear DB before each test module run.""" + from dev.tests_common.test_utils.db import clear_all, initial_db_init + + if not request.config.option.db_cleanup: + return + if skip_db_tests: + return + from airflow.configuration import conf + + sql_alchemy_conn = conf.get("database", "sql_alchemy_conn") + if sql_alchemy_conn.startswith("sqlite"): + sql_alchemy_file = sql_alchemy_conn.replace("sqlite:///", "") + if not os.path.exists(sql_alchemy_file): + print(f"The sqlite file `{sql_alchemy_file}` does not exist. Attempt to initialize it.") + initial_db_init() + + dist_option = getattr(request.config.option, "dist", "no") + if dist_option != "no" or hasattr(request.config, "workerinput"): + # Skip if pytest-xdist detected (controller or worker) + return + try: + clear_all() + except Exception as ex: + exc_name_parts = [type(ex).__name__] + exc_module = type(ex).__module__ + if exc_module != "builtins": + exc_name_parts.insert(0, exc_module) + extra_msg = "" if request.config.option.db_init else ", try to run with flag --with-db-init" + pytest.exit(f"Unable clear test DB{extra_msg}, got error {'.'.join(exc_name_parts)}: {ex}") + + +@pytest.fixture(autouse=True) +def clear_lru_cache(): + from airflow.executors.executor_loader import ExecutorLoader + from airflow.utils.entry_points import _get_grouped_entry_points + + ExecutorLoader.validate_database_executor_compatibility.cache_clear() + try: + _get_grouped_entry_points.cache_clear() + try: + yield + finally: + _get_grouped_entry_points.cache_clear() + finally: + ExecutorLoader.validate_database_executor_compatibility.cache_clear() + + +@pytest.fixture(autouse=True) +def refuse_to_run_test_from_wrongly_named_files(request: pytest.FixtureRequest): + filepath = request.node.path + is_system_test: bool = "tests/system/" in os.fspath(filepath) + test_name = request.node.name + if request.node.cls: + test_name = f"{request.node.cls.__name__}.{test_name}" + if is_system_test and not filepath.name.startswith(("example_", "test_")): + pytest.fail( + f"All test method files in tests/system must start with 'example_' or 'test_'. " + f"Seems that {os.fspath(filepath)!r} contains {test_name!r} that looks like a test case. " + f"Please rename the file to follow the example_* or test_* pattern if you want to run the tests " + f"in it." + ) + elif not is_system_test and not filepath.name.startswith("test_"): + pytest.fail( + f"All test method files in tests/ must start with 'test_'. Seems that {os.fspath(filepath)!r} " + f"contains {test_name!r} that looks like a test case. Please rename the file to " + f"follow the test_* pattern if you want to run the tests in it." + ) + + +@pytest.fixture(autouse=True) +def initialize_providers_manager(): + from airflow.providers_manager import ProvidersManager + + ProvidersManager().initialize_providers_configuration() + + +@pytest.fixture(autouse=True) +def close_all_sqlalchemy_sessions(): + from sqlalchemy.orm import close_all_sessions + + with suppress(Exception): + close_all_sessions() + yield + with suppress(Exception): + close_all_sessions() + + +@pytest.fixture +def cleanup_providers_manager(): + from airflow.providers_manager import ProvidersManager + + ProvidersManager()._cleanup() + ProvidersManager().initialize_providers_configuration() + try: + yield + finally: + ProvidersManager()._cleanup() + + +@pytest.fixture(autouse=True) +def _disable_redact(request: pytest.FixtureRequest, mocker): + """Disable redacted text in tests, except specific.""" + from airflow import settings + + if next(request.node.iter_markers("enable_redact"), None): + with pytest.MonkeyPatch.context() as mp_ctx: + mp_ctx.setattr(settings, "MASK_SECRETS_IN_LOGS", True) + yield + return + + mocked_redact = mocker.patch("airflow.utils.log.secrets_masker.SecretsMasker.redact") + mocked_redact.side_effect = lambda item, name=None, max_depth=None: item + with pytest.MonkeyPatch.context() as mp_ctx: + mp_ctx.setattr(settings, "MASK_SECRETS_IN_LOGS", False) + yield + return + + +@pytest.fixture +def providers_src_folder() -> Path: + import airflow.providers + + return Path(airflow.providers.__path__[0]).parents[1] + + +@pytest.fixture +def hook_lineage_collector(): + from airflow.lineage import hook + + hook._hook_lineage_collector = None + hook._hook_lineage_collector = hook.HookLineageCollector() + yield hook.get_hook_lineage_collector() + hook._hook_lineage_collector = None + + +@pytest.fixture +def clean_dags_and_dagruns(): + """Fixture that cleans the database before and after every test.""" + from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs + + clear_db_runs() + clear_db_dags() + yield # Test runs here + clear_db_dags() + clear_db_runs() + + +@pytest.fixture(scope="session") +def app(): + from dev.tests_common.test_utils.config import conf_vars + + with conf_vars({("fab", "auth_rate_limited"): "False"}): + from airflow.www import app + + yield app.create_app(testing=True) + + +@pytest.fixture +def secret_key() -> str: + """Return secret key configured.""" + from airflow.configuration import conf + + the_key = conf.get("webserver", "SECRET_KEY") + if the_key is None: + raise RuntimeError( + "The secret key SHOULD be configured as `[webserver] secret_key` in the " + "configuration/environment at this stage! " + ) + return the_key + + +@pytest.fixture +def url_safe_serializer(secret_key) -> URLSafeSerializer: + from itsdangerous import URLSafeSerializer + + return URLSafeSerializer(secret_key) diff --git a/tests/test_utils/README.md b/dev/tests_common/test_utils/README.md similarity index 67% rename from tests/test_utils/README.md rename to dev/tests_common/test_utils/README.md index cdde5b83a59bd..2f5e694be5e3a 100644 --- a/tests/test_utils/README.md +++ b/dev/tests_common/test_utils/README.md @@ -15,6 +15,14 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ---> + --> + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Utilities for use in tests.](#utilities-for-use-in-tests) + + # Utilities for use in tests. diff --git a/tests/system/providers/google/marketing_platform/__init__.py b/dev/tests_common/test_utils/__init__.py similarity index 87% rename from tests/system/providers/google/marketing_platform/__init__.py rename to dev/tests_common/test_utils/__init__.py index 217e5db960782..e440178fae6c7 100644 --- a/tests/system/providers/google/marketing_platform/__init__.py +++ b/dev/tests_common/test_utils/__init__.py @@ -15,3 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + +from pathlib import Path + +AIRFLOW_MAIN_FOLDER = Path(__file__).parents[3] diff --git a/tests/test_utils/api_connexion_utils.py b/dev/tests_common/test_utils/api_connexion_utils.py similarity index 93% rename from tests/test_utils/api_connexion_utils.py rename to dev/tests_common/test_utils/api_connexion_utils.py index 48869ee48078d..2d273af2e6878 100644 --- a/tests/test_utils/api_connexion_utils.py +++ b/dev/tests_common/test_utils/api_connexion_utils.py @@ -20,7 +20,8 @@ from typing import TYPE_CHECKING from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP -from tests.test_utils.compat import ignore_provider_compatibility_error + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES @@ -48,11 +49,7 @@ def create_user_scope(app, username, **kwargs): It will create a user and provide it for the fixture via YIELD (generator) then will tidy up once test is complete """ - from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( - create_user as create_user_fab, - ) - - test_user = create_user_fab(app, username, **kwargs) + test_user = create_user(app, username, **kwargs) try: yield test_user diff --git a/tests/test_utils/asserts.py b/dev/tests_common/test_utils/asserts.py similarity index 100% rename from tests/test_utils/asserts.py rename to dev/tests_common/test_utils/asserts.py diff --git a/tests/test_utils/azure_system_helpers.py b/dev/tests_common/test_utils/azure_system_helpers.py similarity index 97% rename from tests/test_utils/azure_system_helpers.py rename to dev/tests_common/test_utils/azure_system_helpers.py index 033399235a294..b1a46617c082f 100644 --- a/tests/test_utils/azure_system_helpers.py +++ b/dev/tests_common/test_utils/azure_system_helpers.py @@ -28,8 +28,9 @@ from airflow.models import Connection from airflow.providers.microsoft.azure.hooks.fileshare import AzureFileShareHook from airflow.utils.process_utils import patch_environ -from tests.test_utils import AIRFLOW_MAIN_FOLDER -from tests.test_utils.system_tests_class import SystemTest + +from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from dev.tests_common.test_utils.system_tests_class import SystemTest AZURE_DAG_FOLDER = os.path.join( AIRFLOW_MAIN_FOLDER, "airflow", "providers", "microsoft", "azure", "example_dags" diff --git a/tests/test_utils/compat.py b/dev/tests_common/test_utils/compat.py similarity index 100% rename from tests/test_utils/compat.py rename to dev/tests_common/test_utils/compat.py diff --git a/tests/test_utils/config.py b/dev/tests_common/test_utils/config.py similarity index 100% rename from tests/test_utils/config.py rename to dev/tests_common/test_utils/config.py diff --git a/tests/test_utils/db.py b/dev/tests_common/test_utils/db.py similarity index 89% rename from tests/test_utils/db.py rename to dev/tests_common/test_utils/db.py index a5dd94e2d009d..4b2dede05ead7 100644 --- a/tests/test_utils/db.py +++ b/dev/tests_common/test_utils/db.py @@ -17,6 +17,8 @@ # under the License. from __future__ import annotations +import os + from airflow.jobs.job import Job from airflow.models import ( Connection, @@ -42,7 +44,8 @@ from airflow.security.permissions import RESOURCE_DAG_PREFIX from airflow.utils.db import add_default_pool_if_not_exists, create_default_connections, reflect_tables from airflow.utils.session import create_session -from tests.test_utils.compat import ( + +from dev.tests_common.test_utils.compat import ( AIRFLOW_V_2_10_PLUS, AssetDagRunQueue, AssetEvent, @@ -53,6 +56,26 @@ ) +def initial_db_init(): + from flask import Flask + + from airflow.configuration import conf + from airflow.utils import db + from airflow.www.extensions.init_appbuilder import init_appbuilder + from airflow.www.extensions.init_auth_manager import get_auth_manager + + from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + + db.resetdb() + db.bootstrap_dagbag() + # minimal app to add roles + flask_app = Flask(__name__) + flask_app.config["SQLALCHEMY_DATABASE_URI"] = conf.get("database", "SQL_ALCHEMY_CONN") + init_appbuilder(flask_app) + if AIRFLOW_V_2_8_PLUS: + get_auth_manager().init() + + def clear_db_runs(): with create_session() as session: session.query(Job).delete() @@ -83,7 +106,7 @@ def clear_db_assets(): session.query(DagScheduleAssetReference).delete() session.query(TaskOutletAssetReference).delete() if AIRFLOW_V_2_10_PLUS: - from tests.test_utils.compat import AssetAliasModel + from dev.tests_common.test_utils.compat import AssetAliasModel session.query(AssetAliasModel).delete() @@ -250,3 +273,7 @@ def clear_all(): clear_db_pools() clear_db_connections(add_default_connections_back=True) clear_dag_specific_permissions() + + +def is_db_isolation_mode(): + return os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true" diff --git a/tests/test_utils/decorators.py b/dev/tests_common/test_utils/decorators.py similarity index 100% rename from tests/test_utils/decorators.py rename to dev/tests_common/test_utils/decorators.py diff --git a/tests/test_utils/fake_datetime.py b/dev/tests_common/test_utils/fake_datetime.py similarity index 100% rename from tests/test_utils/fake_datetime.py rename to dev/tests_common/test_utils/fake_datetime.py diff --git a/tests/test_utils/gcp_system_helpers.py b/dev/tests_common/test_utils/gcp_system_helpers.py similarity index 90% rename from tests/test_utils/gcp_system_helpers.py rename to dev/tests_common/test_utils/gcp_system_helpers.py index 0e681d64c49b4..e17679bd8eb56 100644 --- a/tests/test_utils/gcp_system_helpers.py +++ b/dev/tests_common/test_utils/gcp_system_helpers.py @@ -20,6 +20,7 @@ import os import tempfile from contextlib import contextmanager +from pathlib import Path from tempfile import TemporaryDirectory from typing import Sequence from unittest import mock @@ -27,27 +28,20 @@ import pytest from google.auth.environment_vars import CLOUD_SDK_CONFIG_DIR, CREDENTIALS +import airflow.providers.google from airflow.providers.google.cloud.utils.credentials_provider import provide_gcp_conn_and_credentials -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY, GCP_SECRET_MANAGER_KEY -from tests.test_utils import AIRFLOW_MAIN_FOLDER -from tests.test_utils.logging_command_executor import CommandExecutor -from tests.test_utils.system_tests_class import SystemTest -CLOUD_DAG_FOLDER = os.path.join( - AIRFLOW_MAIN_FOLDER, "airflow", "providers", "google", "cloud", "example_dags" -) -MARKETING_DAG_FOLDER = os.path.join( - AIRFLOW_MAIN_FOLDER, "airflow", "providers", "google", "marketing_platform", "example_dags" -) -GSUITE_DAG_FOLDER = os.path.join( - AIRFLOW_MAIN_FOLDER, "airflow", "providers", "google", "suite", "example_dags" -) -FIREBASE_DAG_FOLDER = os.path.join( - AIRFLOW_MAIN_FOLDER, "airflow", "providers", "google", "firebase", "example_dags" -) -LEVELDB_DAG_FOLDER = os.path.join( - AIRFLOW_MAIN_FOLDER, "airflow", "providers", "google", "leveldb", "example_dags" -) +from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from dev.tests_common.test_utils.logging_command_executor import CommandExecutor +from dev.tests_common.test_utils.system_tests_class import SystemTest +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY, GCP_SECRET_MANAGER_KEY + +GCP_DIR = Path(airflow.providers.google.__file__).parent +CLOUD_DAG_FOLDER = GCP_DIR.joinpath("cloud", "example_dags") +MARKETING_DAG_FOLDER = GCP_DIR.joinpath("marketing_platform", "example_dags") +GSUITE_DAG_FOLDER = GCP_DIR.joinpath("suite", "example_dags") +FIREBASE_DAG_FOLDER = GCP_DIR.joinpath("firebase", "example_dags") +LEVELDB_DAG_FOLDER = GCP_DIR.joinpath("leveldb", "example_dags") POSTGRES_LOCAL_EXECUTOR = os.path.join( AIRFLOW_MAIN_FOLDER, "tests", "test_utils", "postgres_local_executor.cfg" ) diff --git a/tests/test_utils/get_all_tests.py b/dev/tests_common/test_utils/get_all_tests.py similarity index 100% rename from tests/test_utils/get_all_tests.py rename to dev/tests_common/test_utils/get_all_tests.py diff --git a/tests/test_utils/hdfs_utils.py b/dev/tests_common/test_utils/hdfs_utils.py similarity index 100% rename from tests/test_utils/hdfs_utils.py rename to dev/tests_common/test_utils/hdfs_utils.py diff --git a/tests/test_utils/logging_command_executor.py b/dev/tests_common/test_utils/logging_command_executor.py similarity index 100% rename from tests/test_utils/logging_command_executor.py rename to dev/tests_common/test_utils/logging_command_executor.py diff --git a/tests/test_utils/mapping.py b/dev/tests_common/test_utils/mapping.py similarity index 100% rename from tests/test_utils/mapping.py rename to dev/tests_common/test_utils/mapping.py diff --git a/tests/test_utils/mock_executor.py b/dev/tests_common/test_utils/mock_executor.py similarity index 100% rename from tests/test_utils/mock_executor.py rename to dev/tests_common/test_utils/mock_executor.py diff --git a/tests/test_utils/mock_operators.py b/dev/tests_common/test_utils/mock_operators.py similarity index 98% rename from tests/test_utils/mock_operators.py rename to dev/tests_common/test_utils/mock_operators.py index cd816707a59f5..0df0afec824c3 100644 --- a/tests/test_utils/mock_operators.py +++ b/dev/tests_common/test_utils/mock_operators.py @@ -23,7 +23,8 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.xcom import XCom -from tests.test_utils.compat import BaseOperatorLink + +from dev.tests_common.test_utils.compat import BaseOperatorLink if TYPE_CHECKING: import jinja2 diff --git a/tests/test_utils/mock_plugins.py b/dev/tests_common/test_utils/mock_plugins.py similarity index 100% rename from tests/test_utils/mock_plugins.py rename to dev/tests_common/test_utils/mock_plugins.py diff --git a/tests/test_utils/mock_security_manager.py b/dev/tests_common/test_utils/mock_security_manager.py similarity index 92% rename from tests/test_utils/mock_security_manager.py rename to dev/tests_common/test_utils/mock_security_manager.py index d95d077a9882e..6b9f45e3d8410 100644 --- a/tests/test_utils/mock_security_manager.py +++ b/dev/tests_common/test_utils/mock_security_manager.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0", __file__): from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride diff --git a/airflow/providers/airbyte/triggers/__init__.py b/dev/tests_common/test_utils/operators/__init__.py similarity index 100% rename from airflow/providers/airbyte/triggers/__init__.py rename to dev/tests_common/test_utils/operators/__init__.py diff --git a/tests/test_utils/operators/postgres_local_executor.cfg b/dev/tests_common/test_utils/operators/postgres_local_executor.cfg similarity index 100% rename from tests/test_utils/operators/postgres_local_executor.cfg rename to dev/tests_common/test_utils/operators/postgres_local_executor.cfg diff --git a/airflow/providers/alibaba/cloud/log/__init__.py b/dev/tests_common/test_utils/perf/__init__.py similarity index 100% rename from airflow/providers/alibaba/cloud/log/__init__.py rename to dev/tests_common/test_utils/perf/__init__.py diff --git a/tests/test_utils/perf/perf_kit/__init__.py b/dev/tests_common/test_utils/perf/perf_kit/__init__.py similarity index 100% rename from tests/test_utils/perf/perf_kit/__init__.py rename to dev/tests_common/test_utils/perf/perf_kit/__init__.py diff --git a/tests/test_utils/perf/perf_kit/memory.py b/dev/tests_common/test_utils/perf/perf_kit/memory.py similarity index 100% rename from tests/test_utils/perf/perf_kit/memory.py rename to dev/tests_common/test_utils/perf/perf_kit/memory.py diff --git a/tests/test_utils/perf/perf_kit/python.py b/dev/tests_common/test_utils/perf/perf_kit/python.py similarity index 100% rename from tests/test_utils/perf/perf_kit/python.py rename to dev/tests_common/test_utils/perf/perf_kit/python.py diff --git a/tests/test_utils/perf/perf_kit/repeat_and_time.py b/dev/tests_common/test_utils/perf/perf_kit/repeat_and_time.py similarity index 100% rename from tests/test_utils/perf/perf_kit/repeat_and_time.py rename to dev/tests_common/test_utils/perf/perf_kit/repeat_and_time.py diff --git a/tests/test_utils/perf/perf_kit/sqlalchemy.py b/dev/tests_common/test_utils/perf/perf_kit/sqlalchemy.py similarity index 100% rename from tests/test_utils/perf/perf_kit/sqlalchemy.py rename to dev/tests_common/test_utils/perf/perf_kit/sqlalchemy.py diff --git a/tests/test_utils/permissions.py b/dev/tests_common/test_utils/permissions.py similarity index 100% rename from tests/test_utils/permissions.py rename to dev/tests_common/test_utils/permissions.py diff --git a/tests/test_utils/providers.py b/dev/tests_common/test_utils/providers.py similarity index 100% rename from tests/test_utils/providers.py rename to dev/tests_common/test_utils/providers.py diff --git a/tests/test_utils/remote_user_api_auth_backend.py b/dev/tests_common/test_utils/remote_user_api_auth_backend.py similarity index 100% rename from tests/test_utils/remote_user_api_auth_backend.py rename to dev/tests_common/test_utils/remote_user_api_auth_backend.py diff --git a/tests/test_utils/reset_warning_registry.py b/dev/tests_common/test_utils/reset_warning_registry.py similarity index 100% rename from tests/test_utils/reset_warning_registry.py rename to dev/tests_common/test_utils/reset_warning_registry.py diff --git a/tests/test_utils/salesforce_system_helpers.py b/dev/tests_common/test_utils/salesforce_system_helpers.py similarity index 100% rename from tests/test_utils/salesforce_system_helpers.py rename to dev/tests_common/test_utils/salesforce_system_helpers.py diff --git a/tests/test_utils/sftp_system_helpers.py b/dev/tests_common/test_utils/sftp_system_helpers.py similarity index 100% rename from tests/test_utils/sftp_system_helpers.py rename to dev/tests_common/test_utils/sftp_system_helpers.py diff --git a/tests/system/utils/__init__.py b/dev/tests_common/test_utils/system_tests.py similarity index 100% rename from tests/system/utils/__init__.py rename to dev/tests_common/test_utils/system_tests.py diff --git a/tests/test_utils/system_tests_class.py b/dev/tests_common/test_utils/system_tests_class.py similarity index 97% rename from tests/test_utils/system_tests_class.py rename to dev/tests_common/test_utils/system_tests_class.py index cfd72174e1778..836782b8584c9 100644 --- a/tests/test_utils/system_tests_class.py +++ b/dev/tests_common/test_utils/system_tests_class.py @@ -29,8 +29,9 @@ from airflow.configuration import AIRFLOW_HOME, AirflowConfigParser, get_airflow_config from airflow.exceptions import AirflowException from airflow.models.dagbag import DagBag -from tests.test_utils import AIRFLOW_MAIN_FOLDER -from tests.test_utils.logging_command_executor import get_executor + +from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from dev.tests_common.test_utils.logging_command_executor import get_executor DEFAULT_DAG_FOLDER = os.path.join(AIRFLOW_MAIN_FOLDER, "airflow", "example_dags") diff --git a/tests/test_utils/terraform.py b/dev/tests_common/test_utils/terraform.py similarity index 95% rename from tests/test_utils/terraform.py rename to dev/tests_common/test_utils/terraform.py index bbb68b60c1a8b..b600ef6643b19 100644 --- a/tests/test_utils/terraform.py +++ b/dev/tests_common/test_utils/terraform.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests.test_utils.system_tests_class import SystemTest +from dev.tests_common.test_utils.system_tests_class import SystemTest class Terraform(SystemTest): diff --git a/tests/test_utils/timetables.py b/dev/tests_common/test_utils/timetables.py similarity index 100% rename from tests/test_utils/timetables.py rename to dev/tests_common/test_utils/timetables.py diff --git a/tests/system/utils/watcher.py b/dev/tests_common/test_utils/watcher.py similarity index 100% rename from tests/system/utils/watcher.py rename to dev/tests_common/test_utils/watcher.py diff --git a/tests/test_utils/www.py b/dev/tests_common/test_utils/www.py similarity index 100% rename from tests/test_utils/www.py rename to dev/tests_common/test_utils/www.py diff --git a/docs/apache-airflow-providers-airbyte/changelog.rst b/docs/apache-airflow-providers-airbyte/changelog.rst index 2e7bb99a7f6b5..6ac2033101e62 100644 --- a/docs/apache-airflow-providers-airbyte/changelog.rst +++ b/docs/apache-airflow-providers-airbyte/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/airbyte/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/airbyte/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-airbyte/index.rst b/docs/apache-airflow-providers-airbyte/index.rst index a967e36ff4faa..c4aff1b11543f 100644 --- a/docs/apache-airflow-providers-airbyte/index.rst +++ b/docs/apache-airflow-providers-airbyte/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/airbyte/index> + System Tests <_api/tests/system/airbyte/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-airbyte/operators/airbyte.rst b/docs/apache-airflow-providers-airbyte/operators/airbyte.rst index d07301dc42b84..45632be1e3d9e 100644 --- a/docs/apache-airflow-providers-airbyte/operators/airbyte.rst +++ b/docs/apache-airflow-providers-airbyte/operators/airbyte.rst @@ -47,14 +47,14 @@ This Operator will initiate the Airbyte job, and the Operator manages the job st An example using the synchronous way: -.. exampleinclude:: /../../tests/system/providers/airbyte/example_airbyte_trigger_job.py +.. exampleinclude:: /../../providers/tests/system/airbyte/example_airbyte_trigger_job.py :language: python :start-after: [START howto_operator_airbyte_synchronous] :end-before: [END howto_operator_airbyte_synchronous] An example using the async way: -.. exampleinclude:: /../../tests/system/providers/airbyte/example_airbyte_trigger_job.py +.. exampleinclude:: /../../providers/tests/system/airbyte/example_airbyte_trigger_job.py :language: python :start-after: [START howto_operator_airbyte_asynchronous] :end-before: [END howto_operator_airbyte_asynchronous] diff --git a/docs/apache-airflow-providers-alibaba/changelog.rst b/docs/apache-airflow-providers-alibaba/changelog.rst index 5cb1e797e2ae3..f4c779dbf5011 100644 --- a/docs/apache-airflow-providers-alibaba/changelog.rst +++ b/docs/apache-airflow-providers-alibaba/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/alibaba/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/alibaba/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-alibaba/index.rst b/docs/apache-airflow-providers-alibaba/index.rst index ab2b244861b16..4529cd7c02bb3 100644 --- a/docs/apache-airflow-providers-alibaba/index.rst +++ b/docs/apache-airflow-providers-alibaba/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/alibaba/index> + System Tests <_api/tests/system/alibaba/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst b/docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst index ac3f0638ad24c..952eb6c2c6037 100644 --- a/docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst +++ b/docs/apache-airflow-providers-alibaba/operators/analyticdb_spark.rst @@ -39,7 +39,7 @@ Defining tasks In the following code we submit Spark Pi and Spark Logistic regression applications. -.. exampleinclude:: /../../tests/system/providers/alibaba/example_adb_spark_batch.py +.. exampleinclude:: /../../providers/tests/system/alibaba/example_adb_spark_batch.py :language: python :start-after: [START howto_operator_adb_spark_batch] :end-before: [END howto_operator_adb_spark_batch] diff --git a/docs/apache-airflow-providers-alibaba/operators/oss.rst b/docs/apache-airflow-providers-alibaba/operators/oss.rst index d09949be0be06..17d3829e37427 100644 --- a/docs/apache-airflow-providers-alibaba/operators/oss.rst +++ b/docs/apache-airflow-providers-alibaba/operators/oss.rst @@ -45,7 +45,7 @@ Defining tasks In the following code we create a new bucket and then delete the bucket. -.. exampleinclude:: /../../tests/system/providers/alibaba/example_oss_bucket.py +.. exampleinclude:: /../../providers/tests/system/alibaba/example_oss_bucket.py :language: python :start-after: [START howto_operator_oss_bucket] :end-before: [END howto_operator_oss_bucket] diff --git a/docs/apache-airflow-providers-amazon/changelog.rst b/docs/apache-airflow-providers-amazon/changelog.rst index abd12152cfaeb..8138c8db39b24 100644 --- a/docs/apache-airflow-providers-amazon/changelog.rst +++ b/docs/apache-airflow-providers-amazon/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/amazon/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/amazon/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-amazon/example-dags.rst b/docs/apache-airflow-providers-amazon/example-dags.rst index c01939ad25afd..2762597cb4ba3 100644 --- a/docs/apache-airflow-providers-amazon/example-dags.rst +++ b/docs/apache-airflow-providers-amazon/example-dags.rst @@ -20,4 +20,4 @@ Example DAGs You can learn how to use Amazon AWS integrations by analyzing the source code of the example DAGs: -* `Amazon AWS `__ +* `Amazon AWS `__ diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index 6d354b1a5f894..870bbe5626329 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -57,7 +57,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/amazon/index> + System Tests <_api/tests/system/amazon/index> System Tests Dashboard .. toctree:: diff --git a/docs/apache-airflow-providers-amazon/operators/appflow.rst b/docs/apache-airflow-providers-amazon/operators/appflow.rst index c28cfb001b5a0..d54d79df75cbb 100644 --- a/docs/apache-airflow-providers-amazon/operators/appflow.rst +++ b/docs/apache-airflow-providers-amazon/operators/appflow.rst @@ -49,7 +49,7 @@ Run Flow To run an AppFlow flow keeping as is, use: :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow_run.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow_run.py :language: python :dedent: 4 :start-after: [START howto_operator_appflow_run] @@ -66,7 +66,7 @@ Run Flow Full To run an AppFlow flow removing all filters, use: :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunFullOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py :language: python :dedent: 4 :start-after: [START howto_operator_appflow_run_full] @@ -83,7 +83,7 @@ Run Flow Daily To run an AppFlow flow filtering daily records, use: :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunDailyOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py :language: python :dedent: 4 :start-after: [START howto_operator_appflow_run_daily] @@ -100,7 +100,7 @@ Run Flow Before To run an AppFlow flow filtering future records and selecting the past ones, use: :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunBeforeOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py :language: python :dedent: 4 :start-after: [START howto_operator_appflow_run_before] @@ -117,7 +117,7 @@ Run Flow After To run an AppFlow flow filtering past records and selecting the future ones, use: :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunAfterOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py :language: python :dedent: 4 :start-after: [START howto_operator_appflow_run_after] @@ -134,7 +134,7 @@ Skipping Tasks For Empty Runs To skip tasks when some AppFlow run return zero records, use: :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRecordsShortCircuitOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py :language: python :dedent: 4 :start-after: [START howto_operator_appflow_shortcircuit] diff --git a/docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst b/docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst index e789290a6ba0f..295da63f9170a 100644 --- a/docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst +++ b/docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst @@ -48,9 +48,9 @@ to run a query in Amazon Athena. In the following example, we query an existing Athena table and send the results to an existing Amazon S3 bucket. For more examples of how to use this operator, please -see the `Sample DAG `__. +see the `Sample DAG `__. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_athena.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_athena.py :language: python :start-after: [START howto_operator_athena] :dedent: 4 @@ -67,7 +67,7 @@ Wait on Amazon Athena query results Use the :class:`~airflow.providers.amazon.aws.sensors.athena.AthenaSensor` to wait for the results of a query in Amazon Athena. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_athena.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_athena.py :language: python :start-after: [START howto_sensor_athena] :dedent: 4 diff --git a/docs/apache-airflow-providers-amazon/operators/athena/athena_sql.rst b/docs/apache-airflow-providers-amazon/operators/athena/athena_sql.rst index c29f89063fcdd..9f92e17e010e9 100644 --- a/docs/apache-airflow-providers-amazon/operators/athena/athena_sql.rst +++ b/docs/apache-airflow-providers-amazon/operators/athena/athena_sql.rst @@ -41,7 +41,7 @@ The generic ``SQLExecuteQueryOperator`` can be used to execute SQL queries again To execute a single SQL query against an Amazon Athena without bringing back the results to Airflow, please use ``AthenaOperator`` instead. -.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_execute_query.py +.. exampleinclude:: /../../providers/tests/system/common/sql/example_sql_execute_query.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_execute_query] @@ -51,7 +51,7 @@ Also, if you need to do simple data quality tests with Amazon Athena, you can us The below example demonstrates how to instantiate the SQLTableCheckOperator task. -.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_column_table_check.py +.. exampleinclude:: /../../providers/tests/system/common/sql/example_sql_column_table_check.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_table_check] diff --git a/docs/apache-airflow-providers-amazon/operators/batch.rst b/docs/apache-airflow-providers-amazon/operators/batch.rst index 4cc2a2b0cced4..efb71fafe8043 100644 --- a/docs/apache-airflow-providers-amazon/operators/batch.rst +++ b/docs/apache-airflow-providers-amazon/operators/batch.rst @@ -40,7 +40,7 @@ Submit a new AWS Batch job To submit a new AWS Batch job and monitor it until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.operators.batch.BatchOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_batch.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_batch.py :language: python :dedent: 4 :start-after: [START howto_operator_batch] @@ -54,7 +54,7 @@ Create an AWS Batch compute environment To create a new AWS Batch compute environment you can use :class:`~airflow.providers.amazon.aws.operators.batch.BatchCreateComputeEnvironmentOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_batch.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_batch.py :language: python :dedent: 4 :start-after: [START howto_operator_batch_create_compute_environment] @@ -71,7 +71,7 @@ Wait on an AWS Batch job state To wait on the state of an AWS Batch Job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.batch.BatchSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_batch.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_batch.py :language: python :dedent: 4 :start-after: [START howto_sensor_batch] @@ -94,7 +94,7 @@ Wait on an AWS Batch compute environment status To wait on the status of an AWS Batch compute environment until it reaches a terminal status you can use :class:`~airflow.providers.amazon.aws.sensors.batch.BatchComputeEnvironmentSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_batch.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_batch.py :language: python :dedent: 4 :start-after: [START howto_sensor_batch_compute_environment] @@ -108,7 +108,7 @@ Wait on an AWS Batch job queue status To wait on the status of an AWS Batch job queue until it reaches a terminal status you can use :class:`~airflow.providers.amazon.aws.sensors.batch.BatchJobQueueSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_batch.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_batch.py :language: python :dedent: 4 :start-after: [START howto_sensor_batch_job_queue] diff --git a/docs/apache-airflow-providers-amazon/operators/bedrock.rst b/docs/apache-airflow-providers-amazon/operators/bedrock.rst index daf9301565171..6a2af21f2ff91 100644 --- a/docs/apache-airflow-providers-amazon/operators/bedrock.rst +++ b/docs/apache-airflow-providers-amazon/operators/bedrock.rst @@ -52,7 +52,7 @@ for details on the different formats, see For example, to invoke a Meta Llama model you would use: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock.py :language: python :dedent: 4 :start-after: [START howto_operator_invoke_llama_model] @@ -60,7 +60,7 @@ For example, to invoke a Meta Llama model you would use: To invoke an Amazon Titan model you would use: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock.py :language: python :dedent: 4 :start-after: [START howto_operator_invoke_titan_model] @@ -68,7 +68,7 @@ To invoke an Amazon Titan model you would use: To invoke a Claude V2 model using the Completions API you would use: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_operator_invoke_claude_model] @@ -90,7 +90,7 @@ and the training/validation data size. To monitor the state of the job, you can or the :class:`~airflow.providers.amazon.aws.triggers.BedrockCustomizeModelCompletedTrigger` Trigger. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock.py :language: python :dedent: 4 :start-after: [START howto_operator_customize_model] @@ -112,7 +112,7 @@ or the :class:`~airflow.providers.amazon.aws.triggers.BedrockProvisionModelThrou Trigger. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock.py :language: python :dedent: 4 :start-after: [START howto_operator_provision_throughput] @@ -129,7 +129,7 @@ To create an Amazon Bedrock Knowledge Base, you can use For more information on which models support embedding data into a vector store, see https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-supported.html -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_operator_bedrock_create_knowledge_base] @@ -142,7 +142,7 @@ Delete an Amazon Bedrock Knowledge Base Deleting a Knowledge Base is a simple boto API call and can be done in a TaskFlow task like the example below. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :start-after: [START howto_operator_bedrock_delete_knowledge_base] :end-before: [END howto_operator_bedrock_delete_knowledge_base] @@ -155,7 +155,7 @@ Create an Amazon Bedrock Data Source To create an Amazon Bedrock Data Source, you can use :class:`~airflow.providers.amazon.aws.operators.bedrock.BedrockCreateDataSourceOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_operator_bedrock_create_data_source] @@ -168,7 +168,7 @@ Delete an Amazon Bedrock Data Source Deleting a Data Source is a simple boto API call and can be done in a TaskFlow task like the example below. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :start-after: [START howto_operator_bedrock_delete_data_source] :end-before: [END howto_operator_bedrock_delete_data_source] @@ -181,7 +181,7 @@ Ingest data into an Amazon Bedrock Data Source To add data from an Amazon S3 bucket into an Amazon Bedrock Data Source, you can use :class:`~airflow.providers.amazon.aws.operators.bedrock.BedrockIngestDataOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_operator_bedrock_ingest_data] @@ -201,7 +201,7 @@ would like to pass the results through an LLM in order to generate a text respon For more information on which models support retrieving information from a knowledge base, see https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-supported.html -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_operator_bedrock_retrieve] @@ -223,7 +223,7 @@ NOTE: Support for "external sources" was added in boto 1.34.90 Example using an Amazon Bedrock Knowledge Base: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_operator_bedrock_knowledge_base_rag] @@ -231,7 +231,7 @@ Example using an Amazon Bedrock Knowledge Base: Example using a PDF file in an Amazon S3 Bucket: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_operator_bedrock_external_sources_rag] @@ -249,7 +249,7 @@ Wait for an Amazon Bedrock customize model job To wait on the state of an Amazon Bedrock customize model job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.bedrock.BedrockCustomizeModelCompletedSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock.py :language: python :dedent: 4 :start-after: [START howto_sensor_customize_model] @@ -264,7 +264,7 @@ To wait on the state of an Amazon Bedrock provision model throughput job until i terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.bedrock.BedrockProvisionModelThroughputCompletedSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock.py :language: python :dedent: 4 :start-after: [START howto_sensor_provision_throughput] @@ -278,7 +278,7 @@ Wait for an Amazon Bedrock Knowledge Base To wait on the state of an Amazon Bedrock Knowledge Base until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.bedrock.BedrockKnowledgeBaseActiveSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_sensor_bedrock_knowledge_base_active] @@ -292,7 +292,7 @@ Wait for an Amazon Bedrock ingestion job to finish To wait on the state of an Amazon Bedrock data ingestion job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.bedrock.BedrockIngestionJobSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_sensor_bedrock_ingest_data] diff --git a/docs/apache-airflow-providers-amazon/operators/cloudformation.rst b/docs/apache-airflow-providers-amazon/operators/cloudformation.rst index ff45efcdb645e..40ab46bcf732e 100644 --- a/docs/apache-airflow-providers-amazon/operators/cloudformation.rst +++ b/docs/apache-airflow-providers-amazon/operators/cloudformation.rst @@ -47,7 +47,7 @@ Create an AWS CloudFormation stack To create a new AWS CloudFormation stack use :class:`~airflow.providers.amazon.aws.operators.cloud_formation.CloudFormationCreateStackOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_cloudformation.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_cloudformation.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudformation_create_stack] @@ -61,7 +61,7 @@ Delete an AWS CloudFormation stack To delete an AWS CloudFormation stack you can use :class:`~airflow.providers.amazon.aws.operators.cloud_formation.CloudFormationDeleteStackOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_cloudformation.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_cloudformation.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudformation_delete_stack] @@ -78,7 +78,7 @@ Wait on an AWS CloudFormation stack creation state To wait on the state of an AWS CloudFormation stack creation until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.cloud_formation.CloudFormationCreateStackSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_cloudformation.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_cloudformation.py :language: python :dedent: 4 :start-after: [START howto_sensor_cloudformation_create_stack] @@ -92,7 +92,7 @@ Wait on an AWS CloudFormation stack deletion state To wait on the state of an AWS CloudFormation stack deletion until it reaches a terminal state you can use use :class:`~airflow.providers.amazon.aws.sensors.cloud_formation.CloudFormationDeleteStackSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_cloudformation.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_cloudformation.py :language: python :dedent: 4 :start-after: [START howto_sensor_cloudformation_delete_stack] diff --git a/docs/apache-airflow-providers-amazon/operators/comprehend.rst b/docs/apache-airflow-providers-amazon/operators/comprehend.rst index af6ed023fe05e..c273643651bbd 100644 --- a/docs/apache-airflow-providers-amazon/operators/comprehend.rst +++ b/docs/apache-airflow-providers-amazon/operators/comprehend.rst @@ -44,7 +44,7 @@ Create an Amazon Comprehend Start PII Entities Detection Job To create an Amazon Comprehend Start PII Entities Detection Job, you can use :class:`~airflow.providers.amazon.aws.operators.comprehend.ComprehendStartPiiEntitiesDetectionJobOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_comprehend.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_comprehend.py :language: python :dedent: 4 :start-after: [START howto_operator_start_pii_entities_detection_job] @@ -58,7 +58,7 @@ Create an Amazon Comprehend Document Classifier To create an Amazon Comprehend Document Classifier, you can use :class:`~airflow.providers.amazon.aws.operators.comprehend.ComprehendCreateDocumentClassifierOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_comprehend_document_classifier.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_comprehend_document_classifier.py :language: python :dedent: 4 :start-after: [START howto_operator_create_document_classifier] @@ -76,7 +76,7 @@ To wait on the state of an Amazon Comprehend Start PII Entities Detection Job un state you can use :class:`~airflow.providers.amazon.aws.sensors.comprehend.ComprehendStartPiiEntitiesDetectionJobCompletedSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_comprehend.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_comprehend.py :language: python :dedent: 4 :start-after: [START howto_sensor_start_pii_entities_detection_job] @@ -91,7 +91,7 @@ To wait on the state of an Amazon Comprehend Document Classifier until it reache state you can use :class:`~airflow.providers.amazon.aws.sensors.comprehend.ComprehendCreateDocumentClassifierCompletedSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_comprehend_document_classifier.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_comprehend_document_classifier.py :language: python :dedent: 4 :start-after: [START howto_sensor_create_document_classifier] diff --git a/docs/apache-airflow-providers-amazon/operators/datasync.rst b/docs/apache-airflow-providers-amazon/operators/datasync.rst index 16e65db42dbdc..26db89022b5de 100644 --- a/docs/apache-airflow-providers-amazon/operators/datasync.rst +++ b/docs/apache-airflow-providers-amazon/operators/datasync.rst @@ -64,7 +64,7 @@ Execute a task To execute a specific task, you can pass the ``task_arn`` to the operator. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_datasync.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_datasync.py :language: python :dedent: 4 :start-after: [START howto_operator_datasync_specific_task] @@ -78,7 +78,7 @@ If one task is found, this one will be executed. If more than one task is found, the operator will raise an Exception. To avoid this, you can set ``allow_random_task_choice`` to ``True`` to randomly choose from candidate tasks. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_datasync.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_datasync.py :language: python :dedent: 4 :start-after: [START howto_operator_datasync_search_task] @@ -97,7 +97,7 @@ existing Task was found. If these are left to their default value (None) then no Also, because ``delete_task_after_execution`` is set to ``True``, the task will be deleted from AWS DataSync after it completes successfully. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_datasync.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_datasync.py :language: python :dedent: 4 :start-after: [START howto_operator_datasync_create_task] diff --git a/docs/apache-airflow-providers-amazon/operators/dms.rst b/docs/apache-airflow-providers-amazon/operators/dms.rst index 2c30e3ca6ec88..b80fb9e2b3ac4 100644 --- a/docs/apache-airflow-providers-amazon/operators/dms.rst +++ b/docs/apache-airflow-providers-amazon/operators/dms.rst @@ -52,7 +52,7 @@ Create a replication task To create a replication task you can use :class:`~airflow.providers.amazon.aws.operators.dms.DmsCreateTaskOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dms.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dms.py :language: python :dedent: 4 :start-after: [START howto_operator_dms_create_task] @@ -66,7 +66,7 @@ Start a replication task To start a replication task you can use :class:`~airflow.providers.amazon.aws.operators.dms.DmsStartTaskOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dms.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dms.py :language: python :dedent: 4 :start-after: [START howto_operator_dms_start_task] @@ -80,7 +80,7 @@ Get details of replication tasks To retrieve the details for a list of replication tasks you can use :class:`~airflow.providers.amazon.aws.operators.dms.DmsDescribeTasksOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dms.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dms.py :language: python :dedent: 4 :start-after: [START howto_operator_dms_describe_tasks] @@ -94,7 +94,7 @@ Stop a replication task To stop a replication task you can use :class:`~airflow.providers.amazon.aws.operators.dms.DmsStopTaskOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dms.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dms.py :language: python :dedent: 4 :start-after: [START howto_operator_dms_stop_task] @@ -108,7 +108,7 @@ Delete a replication task To delete a replication task you can use :class:`~airflow.providers.amazon.aws.operators.dms.DmsDeleteTaskOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dms.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dms.py :language: python :dedent: 4 :start-after: [START howto_operator_dms_delete_task] @@ -125,7 +125,7 @@ Wait for a replication task to complete To check the state of a replication task until it is completed, you can use :class:`~airflow.providers.amazon.aws.sensors.dms.DmsTaskCompletedSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dms.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dms.py :language: python :dedent: 4 :start-after: [START howto_sensor_dms_task_completed] diff --git a/docs/apache-airflow-providers-amazon/operators/dynamodb.rst b/docs/apache-airflow-providers-amazon/operators/dynamodb.rst index d899f0de9223e..aab6626109d0e 100644 --- a/docs/apache-airflow-providers-amazon/operators/dynamodb.rst +++ b/docs/apache-airflow-providers-amazon/operators/dynamodb.rst @@ -51,7 +51,7 @@ Wait for a Single Attribute Value Match: This example shows how to use ``DynamoDBValueSensor`` to wait for a specific attribute/value pair in a DynamoDB item. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dynamodb.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dynamodb.py :language: python :start-after: [START howto_sensor_dynamodb_value] :dedent: 4 @@ -62,7 +62,7 @@ Wait for Any Value from a List of Attribute Values: In this example, the sensor waits for the DynamoDB item to have an attribute that matches any value from a provided list. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dynamodb.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dynamodb.py :language: python :start-after: [START howto_sensor_dynamodb_any_value] :dedent: 4 diff --git a/docs/apache-airflow-providers-amazon/operators/ec2.rst b/docs/apache-airflow-providers-amazon/operators/ec2.rst index e5462b32a18f4..4cfeb17d24651 100644 --- a/docs/apache-airflow-providers-amazon/operators/ec2.rst +++ b/docs/apache-airflow-providers-amazon/operators/ec2.rst @@ -38,7 +38,7 @@ Start an Amazon EC2 instance To start an Amazon EC2 instance you can use :class:`~airflow.providers.amazon.aws.operators.ec2.EC2StartInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ec2.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ec2.py :language: python :dedent: 4 :start-after: [START howto_operator_ec2_start_instance] @@ -52,7 +52,7 @@ Stop an Amazon EC2 instance To stop an Amazon EC2 instance you can use :class:`~airflow.providers.amazon.aws.operators.ec2.EC2StopInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ec2.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ec2.py :language: python :dedent: 4 :start-after: [START howto_operator_ec2_stop_instance] @@ -66,7 +66,7 @@ Create and start an Amazon EC2 instance To create and start an Amazon EC2 instance you can use :class:`~airflow.providers.amazon.aws.operators.ec2.EC2CreateInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ec2.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ec2.py :language: python :dedent: 4 :start-after: [START howto_operator_ec2_create_instance] @@ -80,7 +80,7 @@ Terminate an Amazon EC2 instance To terminate an Amazon EC2 instance you can use :class:`~airflow.providers.amazon.aws.operators.ec2.EC2TerminateInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ec2.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ec2.py :language: python :dedent: 4 :start-after: [START howto_operator_ec2_terminate_instance] @@ -94,7 +94,7 @@ Reboot an Amazon EC2 instance To reboot an Amazon EC2 instance you can use :class:`~airflow.providers.amazon.aws.operators.ec2.EC2RebootInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ec2.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ec2.py :language: python :dedent: 4 :start-after: [START howto_operator_ec2_reboot_instance] @@ -108,7 +108,7 @@ Hibernate an Amazon EC2 instance To hibernate an Amazon EC2 instance you can use :class:`~airflow.providers.amazon.aws.operators.ec2.EC2HibernateInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ec2.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ec2.py :language: python :dedent: 4 :start-after: [START howto_operator_ec2_hibernate_instance] @@ -125,7 +125,7 @@ Wait on an Amazon EC2 instance state To check the state of an Amazon EC2 instance and wait until it reaches the target state you can use :class:`~airflow.providers.amazon.aws.sensors.ec2.EC2InstanceStateSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ec2.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ec2.py :language: python :dedent: 4 :start-after: [START howto_sensor_ec2_instance_state] diff --git a/docs/apache-airflow-providers-amazon/operators/ecs.rst b/docs/apache-airflow-providers-amazon/operators/ecs.rst index 2c4b3c7e7f584..8cb02d539e7de 100644 --- a/docs/apache-airflow-providers-amazon/operators/ecs.rst +++ b/docs/apache-airflow-providers-amazon/operators/ecs.rst @@ -48,7 +48,7 @@ To create an Amazon ECS cluster you can use All optional parameters to be passed to the Create Cluster API should be passed in the 'create_cluster_kwargs' dict. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 4 :start-after: [START howto_operator_ecs_create_cluster] @@ -63,7 +63,7 @@ To delete an Amazon ECS cluster you can use :class:`~airflow.providers.amazon.aws.operators.ecs.EcsDeleteClusterOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 4 :start-after: [START howto_operator_ecs_delete_cluster] @@ -81,7 +81,7 @@ All optional parameters to be passed to the Register Task Definition API should passed in the 'register_task_kwargs' dict. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 4 :start-after: [START howto_operator_ecs_register_task_definition] @@ -96,7 +96,7 @@ To deregister a task definition you can use :class:`~airflow.providers.amazon.aws.operators.ecs.EcsDeregisterTaskDefinitionOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 4 :start-after: [START howto_operator_ecs_deregister_task_definition] @@ -125,14 +125,14 @@ The parameters you need to configure for this Operator will depend upon which `` * If you are using EC2 as the compute resources in your ECS Cluster, set the parameter to EC2. * If you have integrated external resources in your ECS Cluster, for example using ECS Anywhere, and want to run your containers on those external resources, set the parameter to EXTERNAL. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 4 :start-after: [START howto_operator_ecs_run_task] :end-before: [END howto_operator_ecs_run_task] -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs_fargate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs_fargate.py :language: python :dedent: 4 :start-after: [START howto_operator_ecs] @@ -145,7 +145,7 @@ To stream logs to AWS CloudWatch, you need to define the parameters below. Using the example above, we would add these additional parameters to enable logging to CloudWatch. You need to ensure that you have the appropriate level of permissions (see next section). -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 8 :start-after: [START howto_awslogs_ecs] @@ -228,7 +228,7 @@ the failure reason if a failed state is provided and that state is reached before the target state. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 4 :start-after: [START howto_sensor_ecs_cluster_state] @@ -248,7 +248,7 @@ to change that. Raises an AirflowException with the failure reason if the faile is reached before the target state. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs.py :language: python :dedent: 4 :start-after: [START howto_sensor_ecs_task_definition_state] @@ -267,7 +267,7 @@ both can be overridden with provided values. Raises an AirflowException with the failure reason if a failed state is provided and that state is reached before the target state. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ecs_fargate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ecs_fargate.py :language: python :dedent: 4 :start-after: [START howto_sensor_ecs_task_state] diff --git a/docs/apache-airflow-providers-amazon/operators/eks.rst b/docs/apache-airflow-providers-amazon/operators/eks.rst index 9f1cc9df61eec..aa774b9095957 100644 --- a/docs/apache-airflow-providers-amazon/operators/eks.rst +++ b/docs/apache-airflow-providers-amazon/operators/eks.rst @@ -46,7 +46,7 @@ Note: An AWS IAM role with the following permissions is required: ``eks.amazonaws.com`` must be added to the Trusted Relationships ``AmazonEKSClusterPolicy`` IAM Policy must be attached -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroups.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_create_cluster] @@ -65,7 +65,7 @@ Note: An AWS IAM role with the following permissions is required: ``AmazonEKSClusterPolicy`` IAM Policy must be attached ``AmazonEKSWorkerNodePolicy`` IAM Policy must be attached -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_create_cluster_with_nodegroup] @@ -85,7 +85,7 @@ Note: An AWS IAM role with the following permissions is required: ``AmazonEKSClusterPolicy`` IAM Policy must be attached ``AmazonEKSWorkerNodePolicy`` IAM Policy must be attached -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_create_cluster_with_fargate_profile] @@ -100,7 +100,7 @@ To delete an existing Amazon EKS Cluster you can use :class:`~airflow.providers.amazon.aws.operators.eks.EksDeleteClusterOperator`. You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroups.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_delete_cluster] @@ -110,7 +110,7 @@ Note: If the cluster has any attached resources, such as an Amazon EKS Nodegroup Fargate profile, the cluster can not be deleted. Using the ``force`` parameter will attempt to delete any attached resources first. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_force_delete_cluster] @@ -130,7 +130,7 @@ Note: An AWS IAM role with the following permissions is required: ``AmazonEC2ContainerRegistryReadOnly`` IAM Policy must be attached ``AmazonEKSWorkerNodePolicy`` IAM Policy must be attached -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroups.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_create_nodegroup] @@ -145,7 +145,7 @@ To delete an existing Amazon EKS managed node group you can use :class:`~airflow.providers.amazon.aws.operators.eks.EksDeleteNodegroupOperator`. You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroups.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_delete_nodegroup] @@ -164,7 +164,7 @@ Note: An AWS IAM role with the following permissions is required: ``AmazonEC2ContainerRegistryReadOnly`` IAM Policy must be attached ``AmazonEKSWorkerNodePolicy`` IAM Policy must be attached -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_create_fargate_profile] @@ -178,7 +178,7 @@ Delete an AWS Fargate Profile To delete an existing AWS Fargate Profile you can use :class:`~airflow.providers.amazon.aws.operators.eks.EksDeleteFargateProfileOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_delete_fargate_profile] @@ -194,7 +194,7 @@ To run a pod on an existing Amazon EKS Cluster, you can use Note: An Amazon EKS Cluster with underlying compute infrastructure is required. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroups.py :language: python :dedent: 4 :start-after: [START howto_operator_eks_pod_operator] @@ -211,7 +211,7 @@ Wait on an Amazon EKS cluster state To check the state of an Amazon EKS Cluster until it reaches the target state or another terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.eks.EksClusterStateSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroups.py :language: python :dedent: 4 :start-after: [START howto_sensor_eks_cluster] @@ -225,7 +225,7 @@ Wait on an Amazon EKS managed node group state To check the state of an Amazon EKS managed node group until it reaches the target state or another terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.eks.EksNodegroupStateSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_nodegroups.py :language: python :dedent: 4 :start-after: [START howto_sensor_eks_nodegroup] @@ -239,7 +239,7 @@ Wait on an AWS Fargate profile state To check the state of an AWS Fargate profile until it reaches the target state or another terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.eks.EksFargateProfileSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py :language: python :dedent: 4 :start-after: [START howto_sensor_eks_fargate] diff --git a/docs/apache-airflow-providers-amazon/operators/emr/emr.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr.rst index 8a2255ddbf4cc..5e32baa151c4f 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr/emr.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr.rst @@ -57,7 +57,7 @@ JobFlow configuration To create a job flow on EMR, you need to specify the configuration for the EMR cluster: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr.py :language: python :start-after: [START howto_operator_emr_steps_config] :end-before: [END howto_operator_emr_steps_config] @@ -80,7 +80,7 @@ Create the Job Flow In the following code we are creating a new job flow using the configuration as explained above. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_create_job_flow] @@ -98,7 +98,7 @@ Using ``deferrable`` mode will release worker slots and leads to efficient utili resources within Airflow cluster.However this mode will need the Airflow triggerer to be available in your deployment. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_add_steps] @@ -116,7 +116,7 @@ Using ``deferrable`` mode will release worker slots and leads to efficient utili resources within Airflow cluster.However this mode will need the Airflow triggerer to be available in your deployment. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_terminate_job_flow] @@ -130,7 +130,7 @@ Modify Amazon EMR container To modify an existing EMR container you can use :class:`~airflow.providers.amazon.aws.sensors.emr.EmrContainerSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_modify_cluster] @@ -144,7 +144,7 @@ Start an EMR notebook execution You can use :class:`~airflow.providers.amazon.aws.operators.emr.EmrStartNotebookExecutionOperator` to start a notebook execution on an existing notebook attached to a running cluster. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_notebook_execution.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_notebook_execution.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_start_notebook_execution] @@ -158,7 +158,7 @@ Stop an EMR notebook execution You can use :class:`~airflow.providers.amazon.aws.operators.emr.EmrStopNotebookExecutionOperator` to stop a running notebook execution. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_notebook_execution.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_notebook_execution.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_stop_notebook_execution] @@ -175,7 +175,7 @@ Wait on an EMR notebook execution state To monitor the state of an EMR notebook execution you can use :class:`~airflow.providers.amazon.aws.sensors.emr.EmrNotebookExecutionSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_notebook_execution.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_notebook_execution.py :language: python :dedent: 4 :start-after: [START howto_sensor_emr_notebook_execution] @@ -189,7 +189,7 @@ Wait on an Amazon EMR job flow state To monitor the state of an EMR job flow you can use :class:`~airflow.providers.amazon.aws.sensors.emr.EmrJobFlowSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr.py :language: python :dedent: 4 :start-after: [START howto_sensor_emr_job_flow] @@ -203,7 +203,7 @@ Wait on an Amazon EMR step state To monitor the state of an EMR job step you can use :class:`~airflow.providers.amazon.aws.sensors.emr.EmrStepSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr.py :language: python :dedent: 4 :start-after: [START howto_sensor_emr_step] diff --git a/docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst index dc31c8f984618..122b22fa04841 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst @@ -48,7 +48,7 @@ the eks cluster that you would like to use , and an eks namespace. Refer to the `EMR on EKS Development guide `__ for more details. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_eks.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_eks.py :language: python :start-after: [START howto_operator_emr_eks_create_cluster] :end-before: [END howto_operator_emr_eks_create_cluster] @@ -81,7 +81,7 @@ and ``monitoringConfiguration`` to send logs to the ``/aws/emr-eks-spark`` log g Refer to the `EMR on EKS guide `__ for more details on job configuration. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_eks.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_eks.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_eks_config] @@ -92,7 +92,7 @@ can store them in a connection or provide them in the DAG. Your AWS region shoul in the ``aws_default`` connection as ``{"region_name": "us-east-1"}`` or a custom connection name that gets passed to the operator with the ``aws_conn_id`` parameter. The operator returns the Job ID of the job run. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_eks.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_eks.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_container] @@ -110,7 +110,7 @@ Wait on an Amazon EMR virtual cluster job To wait on the status of an Amazon EMR virtual cluster job to reach a terminal state, you can use :class:`~airflow.providers.amazon.aws.sensors.emr.EmrContainerSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_eks.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_eks.py :language: python :dedent: 4 :start-after: [START howto_sensor_emr_container] diff --git a/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst index 9915763e19e09..28c7bb4720a09 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst @@ -43,7 +43,7 @@ create a new EMR Serverless Application. This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires the aiobotocore module to be installed. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_serverless.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_serverless_create_application] @@ -59,7 +59,7 @@ start an EMR Serverless Job. This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires the aiobotocore module to be installed. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_serverless.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_serverless_start_job] @@ -93,7 +93,7 @@ stop an EMR Serverless Application. This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires the aiobotocore module to be installed. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_serverless.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_serverless_stop_application] @@ -109,7 +109,7 @@ delete an EMR Serverless Application. This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires the aiobotocore module to be installed. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_serverless.py :language: python :dedent: 4 :start-after: [START howto_operator_emr_serverless_delete_application] @@ -126,7 +126,7 @@ Wait on an EMR Serverless Job state To monitor the state of an EMR Serverless Job you can use :class:`~airflow.providers.amazon.aws.sensors.emr.EmrServerlessJobSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_serverless.py :language: python :dedent: 4 :start-after: [START howto_sensor_emr_serverless_job] @@ -140,7 +140,7 @@ Wait on an EMR Serverless Application state To monitor the state of an EMR Serverless Application you can use :class:`~airflow.providers.amazon.aws.sensors.emr.EmrServerlessApplicationSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_emr_serverless.py :language: python :dedent: 4 :start-after: [START howto_sensor_emr_serverless_application] diff --git a/docs/apache-airflow-providers-amazon/operators/eventbridge.rst b/docs/apache-airflow-providers-amazon/operators/eventbridge.rst index 453e5af31070a..a39c884149255 100644 --- a/docs/apache-airflow-providers-amazon/operators/eventbridge.rst +++ b/docs/apache-airflow-providers-amazon/operators/eventbridge.rst @@ -48,7 +48,7 @@ Send events to Amazon EventBridge To send custom events to Amazon EventBridge, use :class:`~airflow.providers.amazon.aws.operators.eventbridge.EventBridgePutEventsOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eventbridge.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eventbridge.py :language: python :dedent: 4 :start-after: [START howto_operator_eventbridge_put_events] @@ -63,7 +63,7 @@ Create or update a rule on Amazon EventBridge To create or update a rule on EventBridge, use :class:`~airflow.providers.amazon.aws.operators.eventbridge.EventBridgePutRuleOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eventbridge.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eventbridge.py :language: python :dedent: 4 :start-after: [START howto_operator_eventbridge_put_rule] @@ -78,7 +78,7 @@ Enable a rule on Amazon EventBridge To enable an existing rule on EventBridge, use :class:`~airflow.providers.amazon.aws.operators.eventbridge.EventBridgeEnableRuleOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eventbridge.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eventbridge.py :language: python :dedent: 4 :start-after: [START howto_operator_eventbridge_enable_rule] @@ -93,7 +93,7 @@ Disable a rule on Amazon EventBridge To disable an existing rule on EventBridge, use :class:`~airflow.providers.amazon.aws.operators.eventbridge.EventBridgeDisableRuleOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eventbridge.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_eventbridge.py :language: python :dedent: 4 :start-after: [START howto_operator_eventbridge_disable_rule] diff --git a/docs/apache-airflow-providers-amazon/operators/glue.rst b/docs/apache-airflow-providers-amazon/operators/glue.rst index 18a9b887d8b40..88b4d3374dbf3 100644 --- a/docs/apache-airflow-providers-amazon/operators/glue.rst +++ b/docs/apache-airflow-providers-amazon/operators/glue.rst @@ -46,7 +46,7 @@ AWS Glue Crawlers allow you to easily extract data from various data sources. To create a new AWS Glue Crawler or run an existing one you can use :class:`~airflow.providers.amazon.aws.operators.glue_crawler.GlueCrawlerOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue.py :language: python :dedent: 4 :start-after: [START howto_operator_glue_crawler] @@ -64,7 +64,7 @@ Submit an AWS Glue job To submit a new AWS Glue job you can use :class:`~airflow.providers.amazon.aws.operators.glue.GlueJobOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue.py :language: python :dedent: 4 :start-after: [START howto_operator_glue] @@ -84,7 +84,7 @@ of your data so that you can make good business decisions. To create a new AWS Glue Data Quality ruleset or update an existing one you can use :class:`~airflow.providers.amazon.aws.operators.glue.GlueDataQualityOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue_data_quality.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue_data_quality.py :language: python :dedent: 4 :start-after: [START howto_operator_glue_data_quality_operator] @@ -98,7 +98,7 @@ Start a AWS Glue Data Quality Evaluation Run To start a AWS Glue Data Quality ruleset evaluation run you can use :class:`~airflow.providers.amazon.aws.operators.glue.GlueDataQualityRuleSetEvaluationRunOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue_data_quality.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue_data_quality.py :language: python :dedent: 4 :start-after: [START howto_operator_glue_data_quality_ruleset_evaluation_run_operator] @@ -112,7 +112,7 @@ Start a AWS Glue Data Quality Recommendation Run To start a AWS Glue Data Quality rule recommendation run you can use :class:`~airflow.providers.amazon.aws.operators.glue.GlueDataQualityRuleRecommendationRunOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue_data_quality_with_recommendation.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py :language: python :dedent: 4 :start-after: [START howto_operator_glue_data_quality_rule_recommendation_run] @@ -129,7 +129,7 @@ Wait on an AWS Glue crawler state To wait on the state of an AWS Glue crawler execution until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.glue_crawler.GlueCrawlerSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue.py :language: python :dedent: 4 :start-after: [START howto_sensor_glue_crawler] @@ -143,7 +143,7 @@ Wait on an AWS Glue job state To wait on the state of an AWS Glue Job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.glue.GlueJobSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue.py :language: python :dedent: 4 :start-after: [START howto_sensor_glue] @@ -157,7 +157,7 @@ Wait on an AWS Glue Data Quality Evaluation Run To wait on the state of an AWS Glue Data Quality RuleSet Evaluation Run until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.glue.GlueDataQualityRuleSetEvaluationRunSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue_data_quality.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue_data_quality.py :language: python :dedent: 4 :start-after: [START howto_sensor_glue_data_quality_ruleset_evaluation_run] @@ -171,7 +171,7 @@ Wait on an AWS Glue Data Quality Recommendation Run To wait on the state of an AWS Glue Data Quality recommendation run until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.glue.GlueDataQualityRuleRecommendationRunSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue_data_quality_with_recommendation.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py :language: python :dedent: 4 :start-after: [START howto_sensor_glue_data_quality_rule_recommendation_run] @@ -185,7 +185,7 @@ Wait on an AWS Glue Catalog Partition To wait for a partition to show up in AWS Glue Catalog until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.glue_catalog_partition.GlueCatalogPartitionSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue.py :language: python :dedent: 4 :start-after: [START howto_sensor_glue_catalog_partition] diff --git a/docs/apache-airflow-providers-amazon/operators/glue_databrew.rst b/docs/apache-airflow-providers-amazon/operators/glue_databrew.rst index be654335ea1d0..2286a5146a59d 100644 --- a/docs/apache-airflow-providers-amazon/operators/glue_databrew.rst +++ b/docs/apache-airflow-providers-amazon/operators/glue_databrew.rst @@ -46,7 +46,7 @@ Start an AWS Glue DataBrew job To submit a new AWS Glue DataBrew job you can use :class:`~airflow.providers.amazon.aws.operators.glue_databrew.GlueDataBrewStartJobOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glue_databrew.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glue_databrew.py :language: python :dedent: 4 :start-after: [START howto_operator_glue_databrew_start] diff --git a/docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst b/docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst index d7156feabb771..dc351fc1c3401 100644 --- a/docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst +++ b/docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst @@ -44,7 +44,7 @@ Create an Amazon Managed Service for Apache Flink Application To create an Amazon Managed Service for Apache Flink application, you can use :class:`~airflow.providers.amazon.aws.operators.kinesis_analytics.KinesisAnalyticsV2CreateApplicationOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_kinesis_analytics.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_kinesis_analytics.py :language: python :dedent: 4 :start-after: [START howto_operator_create_application] @@ -58,7 +58,7 @@ Start an Amazon Managed Service for Apache Flink Application To start an Amazon Managed Service for Apache Flink application, you can use :class:`~airflow.providers.amazon.aws.operators.kinesis_analytics.KinesisAnalyticsV2StartApplicationOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_kinesis_analytics.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_kinesis_analytics.py :language: python :dedent: 4 :start-after: [START howto_operator_start_application] @@ -72,7 +72,7 @@ Stop an Amazon Managed Service for Apache Flink Application To stop an Amazon Managed Service for Apache Flink application, you can use :class:`~airflow.providers.amazon.aws.operators.kinesis_analytics.KinesisAnalyticsV2StopApplicationOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_kinesis_analytics.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_kinesis_analytics.py :language: python :dedent: 4 :start-after: [START howto_operator_stop_application] @@ -89,7 +89,7 @@ Wait for an Amazon Managed Service for Apache Flink Application to start To wait on the state of an Amazon Managed Service for Apache Flink Application to start you can use :class:`~airflow.providers.amazon.aws.sensors.kinesis_analytics.KinesisAnalyticsV2StartApplicationCompletedSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_kinesis_analytics.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_kinesis_analytics.py :language: python :dedent: 4 :start-after: [START howto_sensor_start_application] @@ -103,7 +103,7 @@ Wait for an Amazon Managed Service for Apache Flink Application to stop To wait on the state of an Amazon Managed Service for Apache Flink Application to stop you can use :class:`~airflow.providers.amazon.aws.sensors.kinesis_analytics.KinesisAnalyticsV2StopApplicationCompletedSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_kinesis_analytics.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_kinesis_analytics.py :language: python :dedent: 4 :start-after: [START howto_sensor_stop_application] diff --git a/docs/apache-airflow-providers-amazon/operators/lambda.rst b/docs/apache-airflow-providers-amazon/operators/lambda.rst index ef2576a1f5413..3a9f1a0132451 100644 --- a/docs/apache-airflow-providers-amazon/operators/lambda.rst +++ b/docs/apache-airflow-providers-amazon/operators/lambda.rst @@ -48,7 +48,7 @@ To create an AWS lambda function you can use This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires the aiobotocore module to be installed. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_lambda.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_lambda.py :language: python :dedent: 4 :start-after: [START howto_operator_create_lambda_function] @@ -111,7 +111,7 @@ To invoke an AWS lambda function you can use The only way is `configuring destinations for asynchronous invocation `_ and sensing destination. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_lambda.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_lambda.py :language: python :dedent: 4 :start-after: [START howto_operator_invoke_lambda_function] @@ -128,7 +128,7 @@ Wait on an AWS Lambda function deployment state To check the deployment state of an AWS Lambda function until it reaches the target state or another terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.lambda_function.LambdaFunctionStateSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_lambda.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_lambda.py :language: python :dedent: 4 :start-after: [START howto_sensor_lambda_function_state] diff --git a/docs/apache-airflow-providers-amazon/operators/neptune.rst b/docs/apache-airflow-providers-amazon/operators/neptune.rst index 98c0d7dd57c4a..7b9204e1c0279 100644 --- a/docs/apache-airflow-providers-amazon/operators/neptune.rst +++ b/docs/apache-airflow-providers-amazon/operators/neptune.rst @@ -49,7 +49,7 @@ the aiobotocore module to be installed. .. note:: This operator only starts an existing Neptune database cluster, it does not create a cluster. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_neptune.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_neptune.py :language: python :dedent: 4 :start-after: [START howto_operator_start_neptune_cluster] @@ -65,7 +65,7 @@ To stop a running Neptune database cluster, you can use This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires the aiobotocore module to be installed. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_neptune.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_neptune.py :language: python :dedent: 4 :start-after: [START howto_operator_stop_neptune_cluster] diff --git a/docs/apache-airflow-providers-amazon/operators/opensearchserverless.rst b/docs/apache-airflow-providers-amazon/operators/opensearchserverless.rst index 4fb0b4db93631..4c6f368655812 100644 --- a/docs/apache-airflow-providers-amazon/operators/opensearchserverless.rst +++ b/docs/apache-airflow-providers-amazon/operators/opensearchserverless.rst @@ -46,7 +46,7 @@ Wait for an Amazon OpenSearch Serverless Collection to become active To wait on the state of an Amazon Bedrock customize model job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.bedrock.OpenSearchServerlessCollectionActiveSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py :language: python :dedent: 4 :start-after: [START howto_sensor_opensearch_collection_active] diff --git a/docs/apache-airflow-providers-amazon/operators/quicksight.rst b/docs/apache-airflow-providers-amazon/operators/quicksight.rst index 9cc0abe337e41..4ae412da6e3ae 100644 --- a/docs/apache-airflow-providers-amazon/operators/quicksight.rst +++ b/docs/apache-airflow-providers-amazon/operators/quicksight.rst @@ -46,7 +46,7 @@ Amazon QuickSight create ingestion The ``QuickSightCreateIngestionOperator`` creates and starts a new SPICE ingestion for a dataset. The operator also refreshes existing SPICE datasets. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_quicksight.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_quicksight.py :language: python :dedent: 4 :start-after: [START howto_operator_quicksight_create_ingestion] @@ -62,7 +62,7 @@ Amazon QuickSight ingestion sensor The ``QuickSightSensor`` waits for an Amazon QuickSight create ingestion until it reaches a terminal state. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_quicksight.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_quicksight.py :language: python :dedent: 4 :start-after: [START howto_sensor_quicksight] diff --git a/docs/apache-airflow-providers-amazon/operators/rds.rst b/docs/apache-airflow-providers-amazon/operators/rds.rst index e27bbc2d2f699..9b06c1a048305 100644 --- a/docs/apache-airflow-providers-amazon/operators/rds.rst +++ b/docs/apache-airflow-providers-amazon/operators/rds.rst @@ -41,7 +41,7 @@ To create a snapshot of an Amazon RDS database instance or cluster you can use :class:`~airflow.providers.amazon.aws.operators.rds.RDSCreateDBSnapshotOperator`. The source database instance must be in the ``available`` or ``storage-optimization`` state. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_snapshot.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_snapshot.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_create_db_snapshot] @@ -56,7 +56,7 @@ To copy a snapshot of an Amazon RDS database instance or cluster you can use :class:`~airflow.providers.amazon.aws.operators.rds.RDSCopyDBSnapshotOperator`. The source database snapshot must be in the ``available`` state. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_snapshot.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_snapshot.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_copy_snapshot] @@ -71,7 +71,7 @@ To delete a snapshot of an Amazon RDS database instance or cluster you can use :class:`~airflow.providers.amazon.aws.operators.rds.RDSDeleteDBSnapshotOperator`. The database snapshot must be in the ``available`` state to be deleted. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_snapshot.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_snapshot.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_delete_snapshot] @@ -86,7 +86,7 @@ To export an Amazon RDS snapshot to Amazon S3 you can use :class:`~airflow.providers.amazon.aws.operators.rds.RDSStartExportTaskOperator`. The provided IAM role must have access to the S3 bucket. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_export.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_export.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_start_export_task] @@ -101,7 +101,7 @@ To cancel an Amazon RDS export task to S3 you can use :class:`~airflow.providers.amazon.aws.operators.rds.RDSCancelExportTaskOperator`. Any data that has already been written to the S3 bucket isn't removed. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_export.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_export.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_cancel_export] @@ -118,7 +118,7 @@ This action requires an Amazon SNS topic Amazon Resource Name (ARN). Amazon RDS event notification is only available for not encrypted SNS topics. If you specify an encrypted SNS topic, event notifications are not sent for the topic. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_event.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_event.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_create_event_subscription] @@ -132,7 +132,7 @@ Unsubscribe to an Amazon RDS event notification To delete an Amazon RDS event subscription you can use :class:`~airflow.providers.amazon.aws.operators.rds.RDSDeleteEventSubscriptionOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_event.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_event.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_delete_event_subscription] @@ -147,7 +147,7 @@ To create a AWS DB instance you can use :class:`~airflow.providers.amazon.aws.operators.rds.RdsCreateDbInstanceOperator`. You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_instance.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_instance.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_create_db_instance] @@ -162,7 +162,7 @@ To delete a AWS DB instance you can use :class:`~airflow.providers.amazon.aws.operators.rds.RDSDeleteDbInstanceOperator`. You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_instance.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_instance.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_delete_db_instance] @@ -176,7 +176,7 @@ Start a database instance or cluster To start an Amazon RDS DB instance or cluster you can use :class:`~airflow.providers.amazon.aws.operators.rds.RdsStartDbOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_instance.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_instance.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_start_db] @@ -191,7 +191,7 @@ Stop a database instance or cluster To stop an Amazon RDS DB instance or cluster you can use :class:`~airflow.providers.amazon.aws.operators.rds.RdsStopDbOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_instance.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_instance.py :language: python :dedent: 4 :start-after: [START howto_operator_rds_stop_db] @@ -209,7 +209,7 @@ To wait for an Amazon RDS instance or cluster to reach a specific status you can :class:`~airflow.providers.amazon.aws.sensors.rds.RdsDbSensor`. By default, the sensor waits for a database instance to reach the ``available`` state. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_instance.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_instance.py :language: python :dedent: 4 :start-after: [START howto_sensor_rds_instance] @@ -225,7 +225,7 @@ To wait for an Amazon RDS snapshot with specific statuses you can use :class:`~airflow.providers.amazon.aws.sensors.rds.RdsSnapshotExistenceSensor`. By default, the sensor waits for the existence of a snapshot with status ``available``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_snapshot.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_snapshot.py :language: python :dedent: 4 :start-after: [START howto_sensor_rds_snapshot_existence] @@ -241,7 +241,7 @@ To wait a for an Amazon RDS snapshot export task with specific statuses you can :class:`~airflow.providers.amazon.aws.sensors.rds.RdsExportTaskExistenceSensor`. By default, the sensor waits for the existence of a snapshot with status ``available``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_rds_export.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_rds_export.py :language: python :dedent: 4 :start-after: [START howto_sensor_rds_export_task_existence] diff --git a/docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst index a50e56a760bbd..bc710b56eecc7 100644 --- a/docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst +++ b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst @@ -40,7 +40,7 @@ Create an Amazon Redshift cluster To create an Amazon Redshift Cluster with the specified parameters you can use :class:`~airflow.providers.amazon.aws.operators.redshift_cluster.RedshiftCreateClusterOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_cluster] @@ -56,7 +56,7 @@ To resume a 'paused' Amazon Redshift cluster you can use You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. This will ensure that the task is deferred from the Airflow worker slot and polling for the task status happens on the trigger. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_resume_cluster] @@ -71,7 +71,7 @@ To pause an ``available`` Amazon Redshift cluster you can use :class:`RedshiftPauseClusterOperator `. You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True`` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_pause_cluster] @@ -85,7 +85,7 @@ Create an Amazon Redshift cluster snapshot To create Amazon Redshift cluster snapshot you can use :class:`RedshiftCreateClusterSnapshotOperator ` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_create_cluster_snapshot] @@ -99,7 +99,7 @@ Delete an Amazon Redshift cluster snapshot To delete Amazon Redshift cluster snapshot you can use :class:`RedshiftDeleteClusterSnapshotOperator ` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_delete_cluster_snapshot] @@ -114,7 +114,7 @@ To delete an Amazon Redshift cluster you can use :class:`RedshiftDeleteClusterOperator `. You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True`` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_delete_cluster] @@ -131,7 +131,7 @@ Wait on an Amazon Redshift cluster state To check the state of an Amazon Redshift Cluster until it reaches the target state or another terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.redshift_cluster.RedshiftClusterSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_sensor_redshift_cluster] diff --git a/docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst index 2638e1732cd6c..762eced74fa68 100644 --- a/docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst +++ b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst @@ -48,7 +48,7 @@ statements against an Amazon Redshift cluster. This differs from ``RedshiftSQLOperator`` in that it allows users to query and retrieve data via the AWS API and avoid the necessity of a Postgres connection. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_data] @@ -60,7 +60,7 @@ Reuse a session when executing multiple statements Specify the ``session_keep_alive_seconds`` parameter on an upstream task. In a downstream task, get the session ID from the XCom and pass it to the ``session_id`` parameter. This is useful when you work with temporary tables. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift.py :language: python :dedent: 4 :start-after: [START howto_operator_redshift_data_session_reuse] diff --git a/docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst index 86444d664f3a0..5f4f28b38c56c 100644 --- a/docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst +++ b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst @@ -40,7 +40,7 @@ The generic ``SQLExecuteQueryOperator`` can be used to execute SQL queries again To execute a SQL query against an Amazon Redshift cluster without using a Redshift connection, please check ``RedshiftDataOperator``. -.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_execute_query.py +.. exampleinclude:: /../../providers/tests/system/common/sql/example_sql_execute_query.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_execute_query] diff --git a/docs/apache-airflow-providers-amazon/operators/s3/glacier.rst b/docs/apache-airflow-providers-amazon/operators/s3/glacier.rst index c85e7ac294c7c..ab5b6acbe704c 100644 --- a/docs/apache-airflow-providers-amazon/operators/s3/glacier.rst +++ b/docs/apache-airflow-providers-amazon/operators/s3/glacier.rst @@ -45,7 +45,7 @@ use :class:`~airflow.providers.amazon.aws.transfers.glacier_to_gcs.GlacierCreate This Operator returns a dictionary of information related to the initiated job such as *jobId*, which is required for subsequent tasks. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glacier_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glacier_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_glacier_create_job] @@ -59,7 +59,7 @@ Upload archive to an Amazon Glacier To add an archive to an Amazon S3 Glacier vault use :class:`~airflow.providers.amazon.aws.transfers.glacier_to_gcs.GlacierUploadArchiveOperator` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glacier_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glacier_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_glacier_upload_archive] @@ -76,7 +76,7 @@ Wait on an Amazon Glacier job state To wait on the status of an Amazon Glacier Job to reach a terminal state use :class:`~airflow.providers.amazon.aws.sensors.glacier.GlacierJobOperationSensor` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glacier_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glacier_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_sensor_glacier_job_operation] diff --git a/docs/apache-airflow-providers-amazon/operators/s3/s3.rst b/docs/apache-airflow-providers-amazon/operators/s3/s3.rst index 41e5c7149bb18..ff48c1d5d15b8 100644 --- a/docs/apache-airflow-providers-amazon/operators/s3/s3.rst +++ b/docs/apache-airflow-providers-amazon/operators/s3/s3.rst @@ -38,7 +38,7 @@ Create an Amazon S3 bucket To create an Amazon S3 bucket you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3CreateBucketOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_create_bucket] @@ -52,7 +52,7 @@ Delete an Amazon S3 bucket To delete an Amazon S3 bucket you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteBucketOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_delete_bucket] @@ -66,7 +66,7 @@ Set the tags for an Amazon S3 bucket To set the tags for an Amazon S3 bucket you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3PutBucketTaggingOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_put_bucket_tagging] @@ -80,7 +80,7 @@ Get the tag of an Amazon S3 bucket To get the tag set associated with an Amazon S3 bucket you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3GetBucketTaggingOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_get_bucket_tagging] @@ -94,7 +94,7 @@ Delete the tags of an Amazon S3 bucket To delete the tags of an Amazon S3 bucket you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteBucketTaggingOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_delete_bucket_tagging] @@ -108,7 +108,7 @@ Create an Amazon S3 object To create a new (or replace) Amazon S3 object you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3CreateObjectOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_create_object] @@ -123,7 +123,7 @@ To copy an Amazon S3 object from one bucket to another you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3CopyObjectOperator`. The Amazon S3 connection used here needs to have access to both source and destination bucket/key. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_copy_object] @@ -137,7 +137,7 @@ Delete Amazon S3 objects To delete one or multiple Amazon S3 objects you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteObjectsOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_delete_objects] @@ -153,7 +153,7 @@ To transform the data from one Amazon S3 object and save it to another object yo You can also apply an optional `Amazon S3 Select expression `_ to select the data you want to retrieve from ``source_s3_key`` using ``select_expression``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_file_transform] @@ -169,7 +169,7 @@ To list all Amazon S3 prefixes within an Amazon S3 bucket you can use See `here `__ for more information about Amazon S3 prefixes. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_list_prefixes] @@ -184,7 +184,7 @@ To list all Amazon S3 objects within an Amazon S3 bucket you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3ListOperator`. You can specify a ``prefix`` to filter the objects whose name begins with such prefix. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_operator_s3_list] @@ -208,7 +208,7 @@ Please keep in mind, especially when used to check a large volume of keys, that To check one file: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_single_key] @@ -216,7 +216,7 @@ To check one file: To check multiple files: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_multiple_keys] @@ -224,7 +224,7 @@ To check multiple files: To check a file with regular expression: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_regex] @@ -244,13 +244,13 @@ multiple files can match one key. The list of matched S3 object attributes conta [{"Size": int}] -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_function_definition] :end-before: [END howto_sensor_s3_key_function_definition] -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_function] @@ -262,7 +262,7 @@ the triggerer asynchronously. Note that this will need triggerer to be available To check one file: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_single_key_deferrable] @@ -270,7 +270,7 @@ To check one file: To check multiple files: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_multiple_keys_deferrable] @@ -278,7 +278,7 @@ To check multiple files: To check a file with regular expression: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_key_regex_deferrable] @@ -295,7 +295,7 @@ the inactivity period has passed with no increase in the number of objects you c Note, this sensor will not behave correctly in reschedule mode, as the state of the listed objects in the Amazon S3 bucket will be lost between rescheduled invocations. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3.py :language: python :dedent: 4 :start-after: [START howto_sensor_s3_keys_unchanged] diff --git a/docs/apache-airflow-providers-amazon/operators/sagemaker.rst b/docs/apache-airflow-providers-amazon/operators/sagemaker.rst index c77b689693b03..c2f433267c306 100644 --- a/docs/apache-airflow-providers-amazon/operators/sagemaker.rst +++ b/docs/apache-airflow-providers-amazon/operators/sagemaker.rst @@ -42,7 +42,7 @@ Create an Amazon SageMaker processing job To create an Amazon Sagemaker processing job to sanitize your dataset you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerProcessingOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_processing] @@ -56,7 +56,7 @@ Create an Amazon SageMaker training job To create an Amazon Sagemaker training job you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerTrainingOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_training] @@ -70,7 +70,7 @@ Create an Amazon SageMaker model To create an Amazon Sagemaker model you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerModelOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_model] @@ -84,7 +84,7 @@ Start a hyperparameter tuning job To start a hyperparameter tuning job for an Amazon Sagemaker model you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerTuningOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_tuning] @@ -98,7 +98,7 @@ Delete an Amazon SageMaker model To delete an Amazon Sagemaker model you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerDeleteModelOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_delete_model] @@ -112,7 +112,7 @@ Create an Amazon SageMaker transform job To create an Amazon Sagemaker transform job you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerTransformOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_transform] @@ -126,7 +126,7 @@ Create an Amazon SageMaker endpoint config job To create an Amazon Sagemaker endpoint config job you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerEndpointConfigOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_endpoint.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_endpoint.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_endpoint_config] @@ -140,7 +140,7 @@ Create an Amazon SageMaker endpoint job To create an Amazon Sagemaker endpoint you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerEndpointOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_endpoint.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_endpoint.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_endpoint] @@ -154,7 +154,7 @@ Start an Amazon SageMaker pipeline execution To trigger an execution run for an already-defined Amazon Sagemaker pipeline, you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerStartPipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_pipeline.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_start_pipeline] @@ -168,7 +168,7 @@ Stop an Amazon SageMaker pipeline execution To stop an Amazon Sagemaker pipeline execution that is currently running, you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerStopPipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_pipeline.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_stop_pipeline] @@ -186,7 +186,7 @@ It consists of an inference specification that defines the inference image to us A model package group is a collection of model packages. You can use this operator to add a new version and model package to the group for every DAG run. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_register] @@ -202,7 +202,7 @@ An AutoML experiment will take some input data in CSV and the column it should l and train models on it without needing human supervision. The output is placed in an S3 bucket, and automatically deployed if configured for it. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_auto_ml] @@ -216,7 +216,7 @@ Create an Experiment for later use To create a SageMaker experiment, you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerCreateExperimentOperator`. This creates an experiment so that it's ready to be associated with processing, training and transform jobs. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_experiment] @@ -230,7 +230,7 @@ Create a SageMaker Notebook Instance To create a SageMaker Notebook Instance , you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerCreateNotebookOperator`. This creates a SageMaker Notebook Instance ready to run Jupyter notebooks. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_notebook.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_notebook.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_notebook_create] @@ -244,7 +244,7 @@ Stop a SageMaker Notebook Instance To terminate SageMaker Notebook Instance , you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerStopNotebookOperator`. This terminates the ML compute instance and disconnects the ML storage volume. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_notebook.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_notebook.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_notebook_stop] @@ -258,7 +258,7 @@ Start a SageMaker Notebook Instance To launch a SageMaker Notebook Instance and re-attach an ML storage volume, you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerStartNotebookOperator`. This launches a new ML compute instance with the latest version of the libraries and attached your ML storage volume. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_notebook.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_notebook.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_notebook_start] @@ -273,7 +273,7 @@ Delete a SageMaker Notebook Instance To delete a SageMaker Notebook Instance, you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerDeleteNotebookOperator`. This terminates the instance and deletes the ML storage volume and network interface associated with the instance. The instance must be stopped before it can be deleted. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_notebook.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_notebook.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_notebook_delete] @@ -290,7 +290,7 @@ Wait on an Amazon SageMaker training job state To check the state of an Amazon Sagemaker training job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.sagemaker.SageMakerTrainingSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_sensor_sagemaker_training] @@ -304,7 +304,7 @@ Wait on an Amazon SageMaker transform job state To check the state of an Amazon Sagemaker transform job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerTransformOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_sensor_sagemaker_transform] @@ -318,7 +318,7 @@ Wait on an Amazon SageMaker tuning job state To check the state of an Amazon Sagemaker hyperparameter tuning job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.sagemaker.SageMakerTuningSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_sensor_sagemaker_tuning] @@ -332,7 +332,7 @@ Wait on an Amazon SageMaker endpoint state To check the state of an Amazon Sagemaker endpoint until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.sagemaker.SageMakerEndpointSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_endpoint.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_endpoint.py :language: python :dedent: 4 :start-after: [START howto_sensor_sagemaker_endpoint] @@ -346,7 +346,7 @@ Wait on an Amazon SageMaker pipeline execution state To check the state of an Amazon Sagemaker pipeline execution until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.sagemaker.SageMakerPipelineSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker_pipeline.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker_pipeline.py :language: python :dedent: 4 :start-after: [START howto_sensor_sagemaker_pipeline] @@ -360,7 +360,7 @@ Wait on an Amazon SageMaker AutoML experiment state To check the state of an Amazon Sagemaker AutoML job until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.sagemaker.SageMakerAutoMLSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sagemaker.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sagemaker.py :language: python :dedent: 4 :start-after: [START howto_operator_sagemaker_auto_ml] diff --git a/docs/apache-airflow-providers-amazon/operators/sns.rst b/docs/apache-airflow-providers-amazon/operators/sns.rst index e589e38f89b07..f4eb699e52801 100644 --- a/docs/apache-airflow-providers-amazon/operators/sns.rst +++ b/docs/apache-airflow-providers-amazon/operators/sns.rst @@ -48,7 +48,7 @@ Publish a message to an existing SNS topic To publish a message to an Amazon SNS Topic you can use :class:`~airflow.providers.amazon.aws.operators.sns.SnsPublishOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sns.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sns.py :language: python :dedent: 4 :start-after: [START howto_operator_sns_publish_operator] diff --git a/docs/apache-airflow-providers-amazon/operators/sqs.rst b/docs/apache-airflow-providers-amazon/operators/sqs.rst index 77fb38e2d8567..3eb0087079c38 100644 --- a/docs/apache-airflow-providers-amazon/operators/sqs.rst +++ b/docs/apache-airflow-providers-amazon/operators/sqs.rst @@ -50,7 +50,7 @@ To publish a message to an Amazon SQS queue you can use the In the following example, the task ``publish_to_queue`` publishes a message containing the task instance and the execution date to a queue with a default name of ``Airflow-Example-Queue``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sqs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sqs.py :language: python :dedent: 4 :start-after: [START howto_operator_sqs] @@ -68,7 +68,7 @@ To read messages from an Amazon SQS queue until exhausted use the :class:`~airflow.providers.amazon.aws.sensors.sqs.SqsSensor` This sensor can also be run in deferrable mode by setting ``deferrable`` param to ``True``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sqs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sqs.py :language: python :dedent: 4 :start-after: [START howto_sensor_sqs] diff --git a/docs/apache-airflow-providers-amazon/operators/step_functions.rst b/docs/apache-airflow-providers-amazon/operators/step_functions.rst index 5ab5d19e68290..a83eafa6ae7ad 100644 --- a/docs/apache-airflow-providers-amazon/operators/step_functions.rst +++ b/docs/apache-airflow-providers-amazon/operators/step_functions.rst @@ -45,7 +45,7 @@ To start a new AWS Step Functions state machine execution you can use :class:`~airflow.providers.amazon.aws.operators.step_function.StepFunctionStartExecutionOperator`. You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_step_functions.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_step_functions.py :language: python :dedent: 4 :start-after: [START howto_operator_step_function_start_execution] @@ -59,7 +59,7 @@ Get an AWS Step Functions execution output To fetch the output from an AWS Step Function state machine execution you can use :class:`~airflow.providers.amazon.aws.operators.step_function.StepFunctionGetExecutionOutputOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_step_functions.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_step_functions.py :language: python :dedent: 4 :start-after: [START howto_operator_step_function_get_execution_output] @@ -76,7 +76,7 @@ Wait on an AWS Step Functions state machine execution state To wait on the state of an AWS Step Function state machine execution until it reaches a terminal state you can use :class:`~airflow.providers.amazon.aws.sensors.step_function.StepFunctionExecutionSensor`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_step_functions.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_step_functions.py :language: python :dedent: 4 :start-after: [START howto_sensor_step_function_execution] diff --git a/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst index a3b9df5eb1ee5..cf68049f95165 100644 --- a/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst @@ -39,7 +39,7 @@ To copy data from an Azure Blob Storage container to an Amazon S3 bucket you can Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_azure_blob_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_azure_blob_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_azure_blob_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst index 74c4b78cef497..c8d18f43c3c2e 100644 --- a/docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst @@ -48,7 +48,7 @@ To get more information visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dynamodb_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dynamodb_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_dynamodb_to_s3] @@ -57,7 +57,7 @@ Example usage: To parallelize the replication, users can create multiple ``DynamoDBToS3Operator`` tasks using the ``TotalSegments`` parameter. For instance to replicate with parallelism of 2, create two tasks: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dynamodb_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dynamodb_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_dynamodb_to_s3_segmented] @@ -67,7 +67,7 @@ Users can also pass in ``point_in_time_export`` boolean param to ``DynamoDBToS3O Full export example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dynamodb_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dynamodb_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_dynamodb_to_s3_in_some_point_in_time_full_export] @@ -75,7 +75,7 @@ Full export example usage: Incremental export example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_dynamodb_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_dynamodb_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_dynamodb_to_s3_in_some_point_in_time_incremental_export] diff --git a/docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst index 0f1904208538c..13166a753232e 100644 --- a/docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst @@ -41,7 +41,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_ftp_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_ftp_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_ftp_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst index 0f44f477016b3..eb032d813c1c1 100644 --- a/docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst @@ -39,7 +39,7 @@ To copy data from a Google Cloud Storage bucket to an Amazon S3 bucket you can u Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_gcs_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_gcs_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_gcs_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst b/docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst index 33ffa8033e2f0..775fda6491c4f 100644 --- a/docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst +++ b/docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst @@ -38,7 +38,7 @@ Amazon S3 Glacier To GCS transfer operator To transfer data from an Amazon Glacier vault to Google Cloud Storage you can use :class:`~airflow.providers.amazon.aws.transfers.glacier_to_gcs.GlacierToGCSOperator` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_glacier_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_glacier_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_transfer_glacier_to_gcs] diff --git a/docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst index b5593f9232b1a..a1d2c8ea99bfb 100644 --- a/docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst @@ -38,7 +38,7 @@ Google Sheets to Amazon S3 transfer operator This example loads data from Google Sheets and save it to an Amazon S3 file. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_google_api_sheets_to_s3] @@ -57,7 +57,7 @@ It searches for up to 50 videos (due to pagination) in a given time range (``YOUTUBE_VIDEO_PUBLISHED_AFTER``, ``YOUTUBE_VIDEO_PUBLISHED_BEFORE``) on a YouTube channel (``YOUTUBE_CHANNEL_ID``) saves the response in Amazon S3 and also pushes the data to xcom. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_google_api_youtube_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_google_api_youtube_search_to_s3] @@ -66,7 +66,7 @@ saves the response in Amazon S3 and also pushes the data to xcom. It passes over the YouTube IDs to the next request which then gets the information (``YOUTUBE_VIDEO_FIELDS``) for the requested videos and saves them in Amazon S3 (``S3_BUCKET_NAME``). -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_google_api_youtube_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_google_api_youtube_list_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst b/docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst index 733527e1acf54..a13cf012b4b97 100644 --- a/docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst +++ b/docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst @@ -44,7 +44,7 @@ To get more information visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_hive_to_dynamodb.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_hive_to_dynamodb.py :language: python :dedent: 4 :start-after: [START howto_transfer_hive_to_dynamodb] diff --git a/docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst index d40e66f861b2b..d28c4508aae1e 100644 --- a/docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst @@ -41,7 +41,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_http_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_http_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_http_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst index 5787587eb0181..cb6f46d338eff 100644 --- a/docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst @@ -38,7 +38,7 @@ Imap Attachment To Amazon S3 transfer operator To save an email attachment via IMAP protocol from an email server to an Amazon S3 Bucket you can use :class:`~airflow.providers.amazon.aws.transfers.imap_attachment_to_s3.ImapAttachmentToS3Operator` -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_imap_attachment_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_imap_attachment_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst index 403db3ea9a826..d1ca167844464 100644 --- a/docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst @@ -42,7 +42,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_local_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_local_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_local_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst index 8ad30e5830710..a9d8a441e670e 100644 --- a/docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst @@ -43,7 +43,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_mongo_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_mongo_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_mongo_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst index 980d44d48b756..f2b32d35c52d4 100644 --- a/docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst @@ -42,7 +42,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift_s3_transfers.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift_s3_transfers.py :language: python :dedent: 4 :start-after: [START howto_transfer_redshift_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/s3_to_dynamodb.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_dynamodb.rst index 630863547d7a4..002d47728c24c 100644 --- a/docs/apache-airflow-providers-amazon/transfer/s3_to_dynamodb.rst +++ b/docs/apache-airflow-providers-amazon/transfer/s3_to_dynamodb.rst @@ -48,7 +48,7 @@ To get more information visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3_to_dynamodb.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3_to_dynamodb.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_dynamodb] @@ -57,7 +57,7 @@ Example usage: To load S3 data into an existing DynamoDB table use: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3_to_dynamodb.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3_to_dynamodb.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_dynamodb_existing_table] diff --git a/docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst index 3db4f321b2120..ecb33de0aa5f2 100644 --- a/docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst +++ b/docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst @@ -42,7 +42,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3_to_ftp.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3_to_ftp.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_ftp] diff --git a/docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst index 431973653e1a4..bcef5b272ac7a 100644 --- a/docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst +++ b/docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst @@ -42,7 +42,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift_s3_transfers.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift_s3_transfers.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_redshift] @@ -50,7 +50,7 @@ Example usage: Example of ingesting multiple keys: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_redshift_s3_transfers.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_redshift_s3_transfers.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_redshift_multiple_keys] diff --git a/docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst index 9353721fa5099..56391b634c769 100644 --- a/docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst +++ b/docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst @@ -43,7 +43,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3_to_sftp.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3_to_sftp.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_sftp] diff --git a/docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst index 590b625413cdd..90899d6a399d3 100644 --- a/docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst +++ b/docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst @@ -43,7 +43,7 @@ To get more information about this operator visit: Example usage with a parser for a csv file. This parser loads the file into memory and returns a list of rows: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3_to_sql.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3_to_sql.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_sql] @@ -52,7 +52,7 @@ file into memory and returns a list of rows: Example usage with a parser function that returns a generator. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3_to_sql.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_s3_to_sql.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_sql_generator] diff --git a/docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst index 7e08226d39217..b0a41c16699b0 100644 --- a/docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst @@ -38,7 +38,7 @@ Extract data from Salesforce to Amazon S3 transfer operator The following example demonstrates a use case of extracting account data from a Salesforce instance and upload to an Amazon S3 bucket. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_salesforce_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_salesforce_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_salesforce_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst index 0f8dfd79ed4b6..821bf211097b4 100644 --- a/docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst @@ -42,7 +42,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sftp_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sftp_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_sftp_to_s3] diff --git a/docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst index e01d415942934..5e088f3a12759 100644 --- a/docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst +++ b/docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst @@ -44,7 +44,7 @@ To get more information about this operator visit: Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sql_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sql_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_sql_to_s3] @@ -57,7 +57,7 @@ We can group the data in the table by passing the ``groupby_kwargs`` param. This Example usage: -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_sql_to_s3.py +.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_sql_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_sql_to_s3_with_groupby_param] diff --git a/docs/apache-airflow-providers-apache-beam/changelog.rst b/docs/apache-airflow-providers-apache-beam/changelog.rst index 4c423b292bd9e..7728c11ed6a62 100644 --- a/docs/apache-airflow-providers-apache-beam/changelog.rst +++ b/docs/apache-airflow-providers-apache-beam/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/beam/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/beam/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-beam/index.rst b/docs/apache-airflow-providers-apache-beam/index.rst index a8b7396b6706e..7b311c10fcf4a 100644 --- a/docs/apache-airflow-providers-apache-beam/index.rst +++ b/docs/apache-airflow-providers-apache-beam/index.rst @@ -40,7 +40,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/beam/index> + System Tests <_api/tests/system/apache/beam/index> .. toctree:: :hidden: @@ -48,7 +48,7 @@ :caption: Resources PyPI Repository - Example DAGs + Example DAGs .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-apache-beam/operators.rst b/docs/apache-airflow-providers-apache-beam/operators.rst index b536c514f7851..da4e696d2ba73 100644 --- a/docs/apache-airflow-providers-apache-beam/operators.rst +++ b/docs/apache-airflow-providers-apache-beam/operators.rst @@ -54,13 +54,13 @@ recommend avoiding unless the Dataflow job requires it. Python Pipelines with DirectRunner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_python.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_direct_runner_pipeline_local_file] :end-before: [END howto_operator_start_python_direct_runner_pipeline_local_file] -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_python.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_direct_runner_pipeline_gcs_file] @@ -71,13 +71,13 @@ possibility to free up the worker when it knows it has to wait, and hand off the As a result, while it is suspended (deferred), it is not taking up a worker slot and your cluster will have a lot less resources wasted on idle Operators or Sensors: -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python_async.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_python_async.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_direct_runner_pipeline_local_file_async] :end-before: [END howto_operator_start_python_direct_runner_pipeline_local_file_async] -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python_async.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_python_async.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_direct_runner_pipeline_gcs_file_async] @@ -86,13 +86,13 @@ lot less resources wasted on idle Operators or Sensors: Python Pipelines with DataflowRunner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_python.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_dataflow_runner_pipeline_gcs_file] :end-before: [END howto_operator_start_python_dataflow_runner_pipeline_gcs_file] -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python_dataflow.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_python_dataflow.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_dataflow_runner_pipeline_async_gcs_file] @@ -104,7 +104,7 @@ possibility to free up the worker when it knows it has to wait, and hand off the As a result, while it is suspended (deferred), it is not taking up a worker slot and your cluster will have a lot less resources wasted on idle Operators or Sensors: -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python_async.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_python_async.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_dataflow_runner_pipeline_gcs_file_async] @@ -126,7 +126,7 @@ has the ability to download or available on the local filesystem (provide the ab Java Pipelines with DirectRunner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_beam.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_beam.py :language: python :dedent: 4 :start-after: [START howto_operator_start_java_direct_runner_pipeline] @@ -135,7 +135,7 @@ Java Pipelines with DirectRunner Java Pipelines with DataflowRunner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_java_dataflow.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_java_dataflow.py :language: python :dedent: 4 :start-after: [START howto_operator_start_java_dataflow_runner_pipeline] @@ -159,13 +159,13 @@ init the module and install dependencies with ``go run init example.com/main`` a Go Pipelines with DirectRunner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_go.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_go.py :language: python :dedent: 4 :start-after: [START howto_operator_start_go_direct_runner_pipeline_local_file] :end-before: [END howto_operator_start_go_direct_runner_pipeline_local_file] -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_go.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_go.py :language: python :dedent: 4 :start-after: [START howto_operator_start_go_direct_runner_pipeline_gcs_file] @@ -174,13 +174,13 @@ Go Pipelines with DirectRunner Go Pipelines with DataflowRunner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_go.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_go.py :language: python :dedent: 4 :start-after: [START howto_operator_start_go_dataflow_runner_pipeline_gcs_file] :end-before: [END howto_operator_start_go_dataflow_runner_pipeline_gcs_file] -.. exampleinclude:: /../../tests/system/providers/apache/beam/example_go_dataflow.py +.. exampleinclude:: /../../providers/tests/system/apache/beam/example_go_dataflow.py :language: python :dedent: 4 :start-after: [START howto_operator_start_go_dataflow_runner_pipeline_async_gcs_file] diff --git a/docs/apache-airflow-providers-apache-cassandra/changelog.rst b/docs/apache-airflow-providers-apache-cassandra/changelog.rst index 3fce287c1c072..2119d2c002672 100644 --- a/docs/apache-airflow-providers-apache-cassandra/changelog.rst +++ b/docs/apache-airflow-providers-apache-cassandra/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/cassandra/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/cassandra/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-cassandra/index.rst b/docs/apache-airflow-providers-apache-cassandra/index.rst index 0ca1a4f138045..0a3e52999e40b 100644 --- a/docs/apache-airflow-providers-apache-cassandra/index.rst +++ b/docs/apache-airflow-providers-apache-cassandra/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/cassandra/index> + System Tests <_api/tests/system/apache/cassandra/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-cassandra/operators.rst b/docs/apache-airflow-providers-apache-cassandra/operators.rst index e0a1e10b559a7..24bc51ef9027d 100644 --- a/docs/apache-airflow-providers-apache-cassandra/operators.rst +++ b/docs/apache-airflow-providers-apache-cassandra/operators.rst @@ -50,7 +50,7 @@ Use the ``keys`` parameter to poke until the provided record is found. The exist Example use of these sensors ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. exampleinclude:: /../../tests/system/providers/apache/cassandra/example_cassandra_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/cassandra/example_cassandra_dag.py :language: python :start-after: [START howto_operator_cassandra_sensors] :end-before: [END howto_operator_cassandra_sensors] diff --git a/docs/apache-airflow-providers-apache-drill/changelog.rst b/docs/apache-airflow-providers-apache-drill/changelog.rst index f2795b435a72b..79971613d2f63 100644 --- a/docs/apache-airflow-providers-apache-drill/changelog.rst +++ b/docs/apache-airflow-providers-apache-drill/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/drill/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/drill/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-drill/index.rst b/docs/apache-airflow-providers-apache-drill/index.rst index cf05944534b7c..8ba9ef37a6672 100644 --- a/docs/apache-airflow-providers-apache-drill/index.rst +++ b/docs/apache-airflow-providers-apache-drill/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/drill/index> + System Tests <_api/tests/system/apache/drill/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-drill/operators.rst b/docs/apache-airflow-providers-apache-drill/operators.rst index 7b647c0a2efe4..47784b67fedc3 100644 --- a/docs/apache-airflow-providers-apache-drill/operators.rst +++ b/docs/apache-airflow-providers-apache-drill/operators.rst @@ -39,7 +39,7 @@ The ``sql`` parameter can be templated and be an external ``.sql`` file. Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/drill/example_drill_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/drill/example_drill_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_drill] diff --git a/docs/apache-airflow-providers-apache-druid/changelog.rst b/docs/apache-airflow-providers-apache-druid/changelog.rst index f1d8377ab560f..652948c8ee8c2 100644 --- a/docs/apache-airflow-providers-apache-druid/changelog.rst +++ b/docs/apache-airflow-providers-apache-druid/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/druid/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/druid/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-druid/index.rst b/docs/apache-airflow-providers-apache-druid/index.rst index a351b7a40e353..0b920743839c3 100644 --- a/docs/apache-airflow-providers-apache-druid/index.rst +++ b/docs/apache-airflow-providers-apache-druid/index.rst @@ -47,7 +47,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/druid/index> + System Tests <_api/tests/system/apache/druid/index> .. toctree:: @@ -57,7 +57,7 @@ PyPI Repository Installing from sources - Example DAGs + Example DAGs .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-apache-druid/operators.rst b/docs/apache-airflow-providers-apache-druid/operators.rst index 6930e7b4d3ae3..758c51c538538 100644 --- a/docs/apache-airflow-providers-apache-druid/operators.rst +++ b/docs/apache-airflow-providers-apache-druid/operators.rst @@ -38,7 +38,7 @@ For parameter definition take a look at :class:`~airflow.providers.apache.druid. Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/druid/example_druid_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/druid/example_druid_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_druid_submit] diff --git a/docs/apache-airflow-providers-apache-flink/changelog.rst b/docs/apache-airflow-providers-apache-flink/changelog.rst index c6c5d10cb7eb4..07ffea0939e6f 100644 --- a/docs/apache-airflow-providers-apache-flink/changelog.rst +++ b/docs/apache-airflow-providers-apache-flink/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/flink/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/flink/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-flink/index.rst b/docs/apache-airflow-providers-apache-flink/index.rst index 09ede32f377f5..a909ecfb74006 100644 --- a/docs/apache-airflow-providers-apache-flink/index.rst +++ b/docs/apache-airflow-providers-apache-flink/index.rst @@ -47,7 +47,7 @@ :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-hdfs/changelog.rst b/docs/apache-airflow-providers-apache-hdfs/changelog.rst index 5504c69287833..3c984d0e1129a 100644 --- a/docs/apache-airflow-providers-apache-hdfs/changelog.rst +++ b/docs/apache-airflow-providers-apache-hdfs/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/hdfs/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/hdfs/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-hive/changelog.rst b/docs/apache-airflow-providers-apache-hive/changelog.rst index 59bdc59e2d44f..838d1ce4de318 100644 --- a/docs/apache-airflow-providers-apache-hive/changelog.rst +++ b/docs/apache-airflow-providers-apache-hive/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/hive/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/hive/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-hive/index.rst b/docs/apache-airflow-providers-apache-hive/index.rst index f99065fba4aa9..7a78a85eff547 100644 --- a/docs/apache-airflow-providers-apache-hive/index.rst +++ b/docs/apache-airflow-providers-apache-hive/index.rst @@ -50,14 +50,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/hive/index> + System Tests <_api/tests/system/apache/hive/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources Macros diff --git a/docs/apache-airflow-providers-apache-hive/operators.rst b/docs/apache-airflow-providers-apache-hive/operators.rst index 7a92cba9f2dd1..d6d65a893a9dd 100644 --- a/docs/apache-airflow-providers-apache-hive/operators.rst +++ b/docs/apache-airflow-providers-apache-hive/operators.rst @@ -27,7 +27,7 @@ HiveOperator This operator executes hql code or hive script in a specific Hive database. -.. exampleinclude:: /../../tests/system/providers/apache/hive/example_twitter_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/hive/example_twitter_dag.py :language: python :dedent: 4 :start-after: [START create_hive] diff --git a/docs/apache-airflow-providers-apache-iceberg/changelog.rst b/docs/apache-airflow-providers-apache-iceberg/changelog.rst index daefe9612ee68..220a77a762b68 100644 --- a/docs/apache-airflow-providers-apache-iceberg/changelog.rst +++ b/docs/apache-airflow-providers-apache-iceberg/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/iceberg/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/iceberg/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-iceberg/index.rst b/docs/apache-airflow-providers-apache-iceberg/index.rst index da33da6e4dd39..6bcfd228a8a84 100644 --- a/docs/apache-airflow-providers-apache-iceberg/index.rst +++ b/docs/apache-airflow-providers-apache-iceberg/index.rst @@ -41,7 +41,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/iceberg/index> + System Tests <_api/tests/system/apache/iceberg/index> .. toctree:: @@ -49,7 +49,7 @@ :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources Python API <_api/airflow/providers/apache/iceberg/index> diff --git a/docs/apache-airflow-providers-apache-impala/changelog.rst b/docs/apache-airflow-providers-apache-impala/changelog.rst index 8f7e22d898972..ad7e0972ce927 100644 --- a/docs/apache-airflow-providers-apache-impala/changelog.rst +++ b/docs/apache-airflow-providers-apache-impala/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/impala/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/impala/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-kafka/changelog.rst b/docs/apache-airflow-providers-apache-kafka/changelog.rst index 258e2231535f1..62b89d1ca33e1 100644 --- a/docs/apache-airflow-providers-apache-kafka/changelog.rst +++ b/docs/apache-airflow-providers-apache-kafka/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/kafka/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/kafka/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-kafka/index.rst b/docs/apache-airflow-providers-apache-kafka/index.rst index 0879b57fb7f1b..9575d35490b52 100644 --- a/docs/apache-airflow-providers-apache-kafka/index.rst +++ b/docs/apache-airflow-providers-apache-kafka/index.rst @@ -53,7 +53,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/kafka/index> + System Tests <_api/tests/system/apache/kafka/index> .. toctree:: @@ -61,7 +61,7 @@ :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-kafka/operators/index.rst b/docs/apache-airflow-providers-apache-kafka/operators/index.rst index da5e99b4b7ec1..47ae3d63615f1 100644 --- a/docs/apache-airflow-providers-apache-kafka/operators/index.rst +++ b/docs/apache-airflow-providers-apache-kafka/operators/index.rst @@ -33,7 +33,7 @@ For parameter definitions take a look at :class:`~airflow.providers.apache.kafka Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/kafka/example_dag_hello_kafka.py +.. exampleinclude:: /../../providers/tests/system/apache/kafka/example_dag_hello_kafka.py :language: python :dedent: 4 :start-after: [START howto_operator_consume_from_topic] @@ -58,7 +58,7 @@ For parameter definitions take a look at :class:`~airflow.providers.apache.kafka Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/kafka/example_dag_hello_kafka.py +.. exampleinclude:: /../../providers/tests/system/apache/kafka/example_dag_hello_kafka.py :language: python :dedent: 4 :start-after: [START howto_operator_produce_to_topic] diff --git a/docs/apache-airflow-providers-apache-kafka/sensors.rst b/docs/apache-airflow-providers-apache-kafka/sensors.rst index 4014154640fea..02fd89e5ed642 100644 --- a/docs/apache-airflow-providers-apache-kafka/sensors.rst +++ b/docs/apache-airflow-providers-apache-kafka/sensors.rst @@ -35,7 +35,7 @@ Using the sensor """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/kafka/example_dag_hello_kafka.py +.. exampleinclude:: /../../providers/tests/system/apache/kafka/example_dag_hello_kafka.py :language: python :dedent: 4 :start-after: [START howto_sensor_await_message] @@ -62,7 +62,7 @@ For parameter definitions take a look at :class:`~airflow.providers.apache.kafka Using the sensor """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/kafka/example_dag_event_listener.py +.. exampleinclude:: /../../providers/tests/system/apache/kafka/example_dag_event_listener.py :language: python :dedent: 4 :start-after: [START howto_sensor_await_message_trigger_function] diff --git a/docs/apache-airflow-providers-apache-kylin/changelog.rst b/docs/apache-airflow-providers-apache-kylin/changelog.rst index 2fbe89478c990..af326254ace9b 100644 --- a/docs/apache-airflow-providers-apache-kylin/changelog.rst +++ b/docs/apache-airflow-providers-apache-kylin/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/kylin/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/kylin/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-kylin/index.rst b/docs/apache-airflow-providers-apache-kylin/index.rst index f38208c5bab48..b496932307d0f 100644 --- a/docs/apache-airflow-providers-apache-kylin/index.rst +++ b/docs/apache-airflow-providers-apache-kylin/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/kylin/index> + System Tests <_api/tests/system/apache/kylin/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-livy/changelog.rst b/docs/apache-airflow-providers-apache-livy/changelog.rst index 5b0b68391eb12..cda7f99ad754e 100644 --- a/docs/apache-airflow-providers-apache-livy/changelog.rst +++ b/docs/apache-airflow-providers-apache-livy/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/livy/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/livy/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-livy/index.rst b/docs/apache-airflow-providers-apache-livy/index.rst index f662f0efdbfda..818766c0dd3ae 100644 --- a/docs/apache-airflow-providers-apache-livy/index.rst +++ b/docs/apache-airflow-providers-apache-livy/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/livy/index> + System Tests <_api/tests/system/apache/livy/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-livy/operators.rst b/docs/apache-airflow-providers-apache-livy/operators.rst index 5ef0e0b71a591..851f524c2a840 100644 --- a/docs/apache-airflow-providers-apache-livy/operators.rst +++ b/docs/apache-airflow-providers-apache-livy/operators.rst @@ -29,7 +29,7 @@ LivyOperator This operator wraps the Apache Livy batch REST API, allowing to submit a Spark application to the underlying cluster. -.. exampleinclude:: /../../tests/system/providers/apache/livy/example_livy.py +.. exampleinclude:: /../../providers/tests/system/apache/livy/example_livy.py :language: python :start-after: [START create_livy] :end-before: [END create_livy] @@ -38,7 +38,7 @@ You can also run this operator in deferrable mode by setting the parameter ``def This will lead to efficient utilization of Airflow workers as polling for job status happens on the triggerer asynchronously. Note that this will need triggerer to be available on your Airflow deployment. -.. exampleinclude:: /../../tests/system/providers/apache/livy/example_livy.py +.. exampleinclude:: /../../providers/tests/system/apache/livy/example_livy.py :language: python :start-after: [START create_livy_deferrable] :end-before: [END create_livy_deferrable] diff --git a/docs/apache-airflow-providers-apache-pig/changelog.rst b/docs/apache-airflow-providers-apache-pig/changelog.rst index d1b5cc0aa478a..e6c7d50b406bc 100644 --- a/docs/apache-airflow-providers-apache-pig/changelog.rst +++ b/docs/apache-airflow-providers-apache-pig/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/pig/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/pig/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-pig/index.rst b/docs/apache-airflow-providers-apache-pig/index.rst index 672c07de9a8b0..efab88e3f964c 100644 --- a/docs/apache-airflow-providers-apache-pig/index.rst +++ b/docs/apache-airflow-providers-apache-pig/index.rst @@ -47,14 +47,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/pig/index> + System Tests <_api/tests/system/apache/pig/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-pig/operators.rst b/docs/apache-airflow-providers-apache-pig/operators.rst index 42be0eeac496a..601970051abc7 100644 --- a/docs/apache-airflow-providers-apache-pig/operators.rst +++ b/docs/apache-airflow-providers-apache-pig/operators.rst @@ -26,7 +26,7 @@ Pig programs are amenable to substantial parallelization, which in turns enables Use the :class:`~airflow.providers.apache.pig.operators.pig.PigOperator` to execute a pig script. -.. exampleinclude:: /../../tests/system/providers/apache/pig/example_pig.py +.. exampleinclude:: /../../providers/tests/system/apache/pig/example_pig.py :language: python :start-after: [START create_pig] :end-before: [END create_pig] diff --git a/docs/apache-airflow-providers-apache-pinot/changelog.rst b/docs/apache-airflow-providers-apache-pinot/changelog.rst index 82c9498bc3dd0..6ec0eddf207dc 100644 --- a/docs/apache-airflow-providers-apache-pinot/changelog.rst +++ b/docs/apache-airflow-providers-apache-pinot/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/pinot/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/pinot/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-pinot/index.rst b/docs/apache-airflow-providers-apache-pinot/index.rst index 32cf0f675af5c..4eb704a923416 100644 --- a/docs/apache-airflow-providers-apache-pinot/index.rst +++ b/docs/apache-airflow-providers-apache-pinot/index.rst @@ -40,7 +40,7 @@ :maxdepth: 1 :caption: References - Example DAGs + Example DAGs Python API <_api/airflow/providers/apache/pinot/index> PyPI Repository Installing from sources @@ -50,7 +50,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/pinot/index> + System Tests <_api/tests/system/apache/pinot/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-apache-pinot/operators.rst b/docs/apache-airflow-providers-apache-pinot/operators.rst index dba3b2a059590..0882d856ca1c3 100644 --- a/docs/apache-airflow-providers-apache-pinot/operators.rst +++ b/docs/apache-airflow-providers-apache-pinot/operators.rst @@ -40,7 +40,7 @@ Parameters For parameter definition, take a look at :class:`~airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook` -.. exampleinclude:: /../../tests/system/providers/apache/pinot/example_pinot_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/pinot/example_pinot_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_pinot_admin_hook] @@ -61,7 +61,7 @@ Parameters For parameter definition, take a look at :class:`~airflow.providers.apache.pinot.hooks.pinot.PinotDbApiHook` -.. exampleinclude:: /../../tests/system/providers/apache/pinot/example_pinot_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/pinot/example_pinot_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_pinot_dbapi_example] diff --git a/docs/apache-airflow-providers-apache-spark/changelog.rst b/docs/apache-airflow-providers-apache-spark/changelog.rst index a21049f9e8386..7714d7d18e435 100644 --- a/docs/apache-airflow-providers-apache-spark/changelog.rst +++ b/docs/apache-airflow-providers-apache-spark/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apache/spark/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apache/spark/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst b/docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst index 1755e079b9214..ba61f31e9d993 100644 --- a/docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst +++ b/docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst @@ -44,7 +44,7 @@ Example The following example shows how to use the ``@task.pyspark`` decorator. Note that the ``spark`` and ``sc`` objects are injected into the function. -.. exampleinclude:: /../../tests/system/providers/apache/spark/example_pyspark.py +.. exampleinclude:: /../../providers/tests/system/apache/spark/example_pyspark.py :language: python :dedent: 4 :start-after: [START task_pyspark] diff --git a/docs/apache-airflow-providers-apache-spark/index.rst b/docs/apache-airflow-providers-apache-spark/index.rst index 6e36428a46291..4d96f64dfc935 100644 --- a/docs/apache-airflow-providers-apache-spark/index.rst +++ b/docs/apache-airflow-providers-apache-spark/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/apache/spark/index> + System Tests <_api/tests/system/apache/spark/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-apache-spark/operators.rst b/docs/apache-airflow-providers-apache-spark/operators.rst index f6c20985f24cf..63e728b57332e 100644 --- a/docs/apache-airflow-providers-apache-spark/operators.rst +++ b/docs/apache-airflow-providers-apache-spark/operators.rst @@ -44,7 +44,7 @@ Using the operator Using ``cmd_type`` parameter, is possible to transfer data from Spark to a database (``spark_to_jdbc``) or from a database to Spark (``jdbc_to_spark``), which will write the table using the Spark command ``saveAsTable``. -.. exampleinclude:: /../../tests/system/providers/apache/spark/example_spark_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/spark/example_spark_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_spark_jdbc] @@ -69,7 +69,7 @@ For parameter definition take a look at :class:`~airflow.providers.apache.spark. Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/spark/example_spark_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/spark/example_spark_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_spark_sql] @@ -92,7 +92,7 @@ For parameter definition take a look at :class:`~airflow.providers.apache.spark. Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/apache/spark/example_spark_dag.py +.. exampleinclude:: /../../providers/tests/system/apache/spark/example_spark_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_spark_submit] diff --git a/docs/apache-airflow-providers-apprise/changelog.rst b/docs/apache-airflow-providers-apprise/changelog.rst index 3634dd238a43b..202bf0b8520cf 100644 --- a/docs/apache-airflow-providers-apprise/changelog.rst +++ b/docs/apache-airflow-providers-apprise/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/apprise/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/apprise/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-arangodb/changelog.rst b/docs/apache-airflow-providers-arangodb/changelog.rst index c3ddda82157eb..c4229fd8a9c80 100644 --- a/docs/apache-airflow-providers-arangodb/changelog.rst +++ b/docs/apache-airflow-providers-arangodb/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/arangodb/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/arangodb/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-arangodb/operators/index.rst b/docs/apache-airflow-providers-arangodb/operators/index.rst index 33b4e10064c48..186e000a984b5 100644 --- a/docs/apache-airflow-providers-arangodb/operators/index.rst +++ b/docs/apache-airflow-providers-arangodb/operators/index.rst @@ -32,7 +32,7 @@ Callable as you like. An example of Listing all Documents in **students** collection can be implemented as following: -.. exampleinclude:: /../../airflow/providers/arangodb/example_dags/example_arangodb.py +.. exampleinclude:: /../../providers/src/airflow/providers/arangodb/example_dags/example_arangodb.py :language: python :start-after: [START howto_aql_operator_arangodb] :end-before: [END howto_aql_operator_arangodb] @@ -40,7 +40,7 @@ An example of Listing all Documents in **students** collection can be implemente You can also provide file template (.sql) to load query, remember path is relative to **dags/** folder, if you want to provide any other path please provide **template_searchpath** while creating **DAG** object, -.. exampleinclude:: /../../airflow/providers/arangodb/example_dags/example_arangodb.py +.. exampleinclude:: /../../providers/src/airflow/providers/arangodb/example_dags/example_arangodb.py :language: python :start-after: [START howto_aql_operator_template_file_arangodb] :end-before: [END howto_aql_operator_template_file_arangodb] @@ -53,14 +53,14 @@ AQL query in `ArangoDB `__. An example for waiting a document in **students** collection with student name **judy** can be implemented as following: -.. exampleinclude:: /../../airflow/providers/arangodb/example_dags/example_arangodb.py +.. exampleinclude:: /../../providers/src/airflow/providers/arangodb/example_dags/example_arangodb.py :language: python :start-after: [START howto_aql_sensor_arangodb] :end-before: [END howto_aql_sensor_arangodb] Similar to **AQLOperator**, You can also provide file template to load query - -.. exampleinclude:: /../../airflow/providers/arangodb/example_dags/example_arangodb.py +.. exampleinclude:: /../../providers/src/airflow/providers/arangodb/example_dags/example_arangodb.py :language: python :start-after: [START howto_aql_sensor_template_file_arangodb] :end-before: [END howto_aql_sensor_template_file_arangodb] diff --git a/docs/apache-airflow-providers-asana/changelog.rst b/docs/apache-airflow-providers-asana/changelog.rst index 92a6d94d0f8d4..ae927055eb9f1 100644 --- a/docs/apache-airflow-providers-asana/changelog.rst +++ b/docs/apache-airflow-providers-asana/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/asana/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/asana/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-asana/index.rst b/docs/apache-airflow-providers-asana/index.rst index ce04f8b4ae871..edc88f080c3f5 100644 --- a/docs/apache-airflow-providers-asana/index.rst +++ b/docs/apache-airflow-providers-asana/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/asana/index> + System Tests <_api/tests/system/asana/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-asana/operators/asana.rst b/docs/apache-airflow-providers-asana/operators/asana.rst index aff9f9f51e3db..5c9cdd0def8ad 100644 --- a/docs/apache-airflow-providers-asana/operators/asana.rst +++ b/docs/apache-airflow-providers-asana/operators/asana.rst @@ -86,7 +86,7 @@ the Asana connection to use to connect to your account (``conn_id``). There are `task attributes you can overwrite `_ through the ``task_parameters``. -.. exampleinclude:: /../../tests/system/providers/asana/example_asana.py +.. exampleinclude:: /../../providers/tests/system/asana/example_asana.py :language: python :dedent: 4 :start-after: [START asana_example_dag] diff --git a/docs/apache-airflow-providers-atlassian-jira/changelog.rst b/docs/apache-airflow-providers-atlassian-jira/changelog.rst index 6e1c12bdf4035..5ea7dfff3dea2 100644 --- a/docs/apache-airflow-providers-atlassian-jira/changelog.rst +++ b/docs/apache-airflow-providers-atlassian-jira/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/atlassian/jira/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/atlassian/jira/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-celery/changelog.rst b/docs/apache-airflow-providers-celery/changelog.rst index b9abe5dbf53e3..c1b38edc7d933 100644 --- a/docs/apache-airflow-providers-celery/changelog.rst +++ b/docs/apache-airflow-providers-celery/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/celery/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/celery/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cloudant/changelog.rst b/docs/apache-airflow-providers-cloudant/changelog.rst index 854f5fda01e2f..d969e082c17b2 100644 --- a/docs/apache-airflow-providers-cloudant/changelog.rst +++ b/docs/apache-airflow-providers-cloudant/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/cloudant/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/cloudant/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst b/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst index 493abbcdb8487..6ad86cec6753c 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/cncf/kubernetes/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/cncf/kubernetes/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/docs/apache-airflow-providers-cncf-kubernetes/index.rst index 1db81c09a3f6d..e0da7d26e1b42 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst @@ -59,14 +59,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/cncf/kubernetes/index> + System Tests <_api/tests/system/cncf/kubernetes/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst index 57d0a26742213..a85a793712885 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst @@ -108,21 +108,21 @@ With these requirements in mind, here are some examples of basic ``pod_template_ Storing DAGs in the image: -.. literalinclude:: /../../airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +.. literalinclude:: /../../providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml :language: yaml :start-after: [START template_with_dags_in_image] :end-before: [END template_with_dags_in_image] Storing DAGs in a ``persistentVolume``: -.. literalinclude:: /../../airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +.. literalinclude:: /../../providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml :language: yaml :start-after: [START template_with_dags_in_volume] :end-before: [END template_with_dags_in_volume] Pulling DAGs from ``git``: -.. literalinclude:: /../../airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml +.. literalinclude:: /../../providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml :language: yaml :start-after: [START git_sync_template] :end-before: [END git_sync_template] diff --git a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst index 6d5b1414488af..2268a8655c987 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst @@ -102,7 +102,7 @@ Using this method will ensure correctness and type safety. While we have removed almost all Kubernetes convenience classes, we have kept the :class:`~airflow.providers.cncf.kubernetes.secret.Secret` class to simplify the process of generating secret volumes/env variables. -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes.py :language: python :start-after: [START howto_operator_k8s_cluster_resources] :end-before: [END howto_operator_k8s_cluster_resources] @@ -135,21 +135,21 @@ Create the Secret using ``kubectl``: Then use it in your pod like so: -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes.py :language: python :start-after: [START howto_operator_k8s_private_image] :end-before: [END howto_operator_k8s_private_image] Also for this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_async.py :language: python :start-after: [START howto_operator_k8s_private_image_async] :end-before: [END howto_operator_k8s_private_image_async] Example to fetch and display container log periodically -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_async.py :language: python :start-after: [START howto_operator_async_log] :end-before: [END howto_operator_async_log] @@ -168,7 +168,7 @@ alongside the Pod. The Pod must write the XCom value into this location at the ` See the following example on how this occurs: -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes.py :language: python :start-after: [START howto_operator_k8s_write_xcom] :end-before: [END howto_operator_k8s_write_xcom] @@ -177,7 +177,7 @@ See the following example on how this occurs: Also for this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_async.py :language: python :start-after: [START howto_operator_k8s_write_xcom_async] :end-before: [END howto_operator_k8s_write_xcom_async] @@ -621,7 +621,7 @@ request that dynamically launches this Job. Users can specify a kubeconfig file using the ``config_file`` parameter, otherwise the operator will default to ``~/.kube/config``. It also allows users to supply a template YAML file using the ``job_template_file`` parameter. -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_k8s_job] @@ -629,7 +629,7 @@ to ``~/.kube/config``. It also allows users to supply a template YAML file using The :class:`~airflow.providers.cncf.kubernetes.operators.job.KubernetesJobOperator` also supports deferrable mode: -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_k8s_job_deferrable] @@ -656,7 +656,7 @@ KubernetesDeleteJobOperator The :class:`~airflow.providers.cncf.kubernetes.operators.job.KubernetesDeleteJobOperator` allows you to delete Jobs on a Kubernetes cluster. -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_k8s_job] @@ -671,7 +671,7 @@ KubernetesPatchJobOperator The :class:`~airflow.providers.cncf.kubernetes.operators.job.KubernetesPatchJobOperator` allows you to update Jobs on a Kubernetes cluster. -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_update_job] diff --git a/docs/apache-airflow-providers-cohere/changelog.rst b/docs/apache-airflow-providers-cohere/changelog.rst index f4e96e1909fb8..aa7753b85cd65 100644 --- a/docs/apache-airflow-providers-cohere/changelog.rst +++ b/docs/apache-airflow-providers-cohere/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/cohere/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/cohere/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index 8909c2bd72dbf..3ae62208a98fe 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -51,7 +51,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/cohere/index> + System Tests <_api/tests/system/cohere/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-cohere/operators/embedding.rst b/docs/apache-airflow-providers-cohere/operators/embedding.rst index b765fe8e9d07c..18adf619ce6d7 100644 --- a/docs/apache-airflow-providers-cohere/operators/embedding.rst +++ b/docs/apache-airflow-providers-cohere/operators/embedding.rst @@ -33,7 +33,7 @@ connect to your account. Example Code: ------------- -.. exampleinclude:: /../../tests/system/providers/cohere/example_cohere_embedding_operator.py +.. exampleinclude:: /../../providers/tests/system/cohere/example_cohere_embedding_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_cohere_embedding] diff --git a/docs/apache-airflow-providers-common-compat/changelog.rst b/docs/apache-airflow-providers-common-compat/changelog.rst index 32438992010a2..074a237efcc16 100644 --- a/docs/apache-airflow-providers-common-compat/changelog.rst +++ b/docs/apache-airflow-providers-common-compat/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/common/compat/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/common/compat/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-common-io/changelog.rst b/docs/apache-airflow-providers-common-io/changelog.rst index 16dec7d266c36..8662eabd816e1 100644 --- a/docs/apache-airflow-providers-common-io/changelog.rst +++ b/docs/apache-airflow-providers-common-io/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/common/io/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/common/io/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-common-io/index.rst b/docs/apache-airflow-providers-common-io/index.rst index 606b6c5161bcd..ac0f1433d20cd 100644 --- a/docs/apache-airflow-providers-common-io/index.rst +++ b/docs/apache-airflow-providers-common-io/index.rst @@ -51,14 +51,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/common/io/index> + System Tests <_api/tests/system/common/io/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-common-io/operators.rst b/docs/apache-airflow-providers-common-io/operators.rst index 12b4a1c207ff0..fee5bb8eac5f1 100644 --- a/docs/apache-airflow-providers-common-io/operators.rst +++ b/docs/apache-airflow-providers-common-io/operators.rst @@ -40,7 +40,7 @@ Otherwise the data will be streamed from the source to the destination. The example below shows how to instantiate the FileTransferOperator task. -.. exampleinclude:: /../../tests/system/providers/common/io/example_file_transfer_local_to_s3.py +.. exampleinclude:: /../../providers/tests/system/common/io/example_file_transfer_local_to_s3.py :language: python :dedent: 4 :start-after: [START howto_transfer_local_to_s3] diff --git a/docs/apache-airflow-providers-common-sql/changelog.rst b/docs/apache-airflow-providers-common-sql/changelog.rst index b1f08fee1ab60..d071c9fc1a1e6 100644 --- a/docs/apache-airflow-providers-common-sql/changelog.rst +++ b/docs/apache-airflow-providers-common-sql/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/common/sql/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/common/sql/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-common-sql/index.rst b/docs/apache-airflow-providers-common-sql/index.rst index f764b57a8b680..2d554bf3da26e 100644 --- a/docs/apache-airflow-providers-common-sql/index.rst +++ b/docs/apache-airflow-providers-common-sql/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/common/sql/index> + System Tests <_api/tests/system/common/sql/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-common-sql/operators.rst b/docs/apache-airflow-providers-common-sql/operators.rst index bc725be418c03..e6e24eefb0017 100644 --- a/docs/apache-airflow-providers-common-sql/operators.rst +++ b/docs/apache-airflow-providers-common-sql/operators.rst @@ -38,7 +38,7 @@ different databases. Parameters of the operators are: The example below shows how to instantiate the SQLExecuteQueryOperator task. -.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_execute_query.py +.. exampleinclude:: /../../providers/tests/system/common/sql/example_sql_execute_query.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_execute_query] @@ -101,7 +101,7 @@ empty tables to return valid integers. The below example demonstrates how to instantiate the SQLColumnCheckOperator task. -.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_column_table_check.py +.. exampleinclude:: /../../providers/tests/system/common/sql/example_sql_column_table_check.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_column_check] @@ -141,7 +141,7 @@ checks. The below example demonstrates how to instantiate the SQLTableCheckOperator task. -.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_column_table_check.py +.. exampleinclude:: /../../providers/tests/system/common/sql/example_sql_column_table_check.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_table_check] diff --git a/docs/apache-airflow-providers-databricks/changelog.rst b/docs/apache-airflow-providers-databricks/changelog.rst index 87771d0f51998..8e4f5f126efd4 100644 --- a/docs/apache-airflow-providers-databricks/changelog.rst +++ b/docs/apache-airflow-providers-databricks/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/databricks/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/databricks/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-databricks/index.rst b/docs/apache-airflow-providers-databricks/index.rst index 5a124818ea4dc..32cd75c0ef2be 100644 --- a/docs/apache-airflow-providers-databricks/index.rst +++ b/docs/apache-airflow-providers-databricks/index.rst @@ -50,14 +50,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/databricks/index> + System Tests <_api/tests/system/databricks/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-databricks/operators/copy_into.rst b/docs/apache-airflow-providers-databricks/operators/copy_into.rst index 71a3fa9e89ad8..56eb20c662286 100644 --- a/docs/apache-airflow-providers-databricks/operators/copy_into.rst +++ b/docs/apache-airflow-providers-databricks/operators/copy_into.rst @@ -46,7 +46,7 @@ Importing CSV data An example usage of the DatabricksCopyIntoOperator to import CSV data into a table is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sql.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sql.py :language: python :start-after: [START howto_operator_databricks_copy_into] :end-before: [END howto_operator_databricks_copy_into] diff --git a/docs/apache-airflow-providers-databricks/operators/jobs_create.rst b/docs/apache-airflow-providers-databricks/operators/jobs_create.rst index 7e6765eba420a..621423f83f32d 100644 --- a/docs/apache-airflow-providers-databricks/operators/jobs_create.rst +++ b/docs/apache-airflow-providers-databricks/operators/jobs_create.rst @@ -66,7 +66,7 @@ Specifying parameters as JSON An example usage of the DatabricksCreateJobsOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_jobs_create_json] :end-before: [END howto_operator_databricks_jobs_create_json] @@ -76,7 +76,7 @@ Using named parameters You can also use named parameters to initialize the operator and run the job. -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_jobs_create_named] :end-before: [END howto_operator_databricks_jobs_create_named] @@ -87,7 +87,7 @@ Pairing with DatabricksRunNowOperator You can use the ``job_id`` that is returned by the DatabricksCreateJobsOperator in the return_value XCom as an argument to the DatabricksRunNowOperator to run the job. -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_run_now] :end-before: [END howto_operator_databricks_run_now] diff --git a/docs/apache-airflow-providers-databricks/operators/notebook.rst b/docs/apache-airflow-providers-databricks/operators/notebook.rst index b87d0d20e6f5a..bf7b04ca74caa 100644 --- a/docs/apache-airflow-providers-databricks/operators/notebook.rst +++ b/docs/apache-airflow-providers-databricks/operators/notebook.rst @@ -31,14 +31,14 @@ Examples Running a notebook in Databricks on a new cluster ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_notebook_new_cluster] :end-before: [END howto_operator_databricks_notebook_new_cluster] Running a notebook in Databricks on an existing cluster ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_notebook_existing_cluster] :end-before: [END howto_operator_databricks_notebook_existing_cluster] diff --git a/docs/apache-airflow-providers-databricks/operators/repos_create.rst b/docs/apache-airflow-providers-databricks/operators/repos_create.rst index 6611a51cd6c15..6b60eae27870b 100644 --- a/docs/apache-airflow-providers-databricks/operators/repos_create.rst +++ b/docs/apache-airflow-providers-databricks/operators/repos_create.rst @@ -63,7 +63,7 @@ Create a Databricks Repo An example usage of the DatabricksReposCreateOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_repos.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_repos.py :language: python :start-after: [START howto_operator_databricks_repo_create] :end-before: [END howto_operator_databricks_repo_create] diff --git a/docs/apache-airflow-providers-databricks/operators/repos_delete.rst b/docs/apache-airflow-providers-databricks/operators/repos_delete.rst index 74d4b62972a14..3186dd131dff7 100644 --- a/docs/apache-airflow-providers-databricks/operators/repos_delete.rst +++ b/docs/apache-airflow-providers-databricks/operators/repos_delete.rst @@ -55,7 +55,7 @@ Deleting Databricks Repo by specifying path An example usage of the DatabricksReposDeleteOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_repos.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_repos.py :language: python :start-after: [START howto_operator_databricks_repo_delete] :end-before: [END howto_operator_databricks_repo_delete] diff --git a/docs/apache-airflow-providers-databricks/operators/repos_update.rst b/docs/apache-airflow-providers-databricks/operators/repos_update.rst index 56af4edabbcf6..6893ee0107e86 100644 --- a/docs/apache-airflow-providers-databricks/operators/repos_update.rst +++ b/docs/apache-airflow-providers-databricks/operators/repos_update.rst @@ -60,7 +60,7 @@ Updating Databricks Repo by specifying path An example usage of the DatabricksReposUpdateOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_repos.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_repos.py :language: python :start-after: [START howto_operator_databricks_repo_update] :end-before: [END howto_operator_databricks_repo_update] diff --git a/docs/apache-airflow-providers-databricks/operators/sql.rst b/docs/apache-airflow-providers-databricks/operators/sql.rst index 55bbf64758562..33acacae1c6bd 100644 --- a/docs/apache-airflow-providers-databricks/operators/sql.rst +++ b/docs/apache-airflow-providers-databricks/operators/sql.rst @@ -49,7 +49,7 @@ Selecting data An example usage of the DatabricksSqlOperator to select data from a table is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sql.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sql.py :language: python :start-after: [START howto_operator_databricks_sql_select] :end-before: [END howto_operator_databricks_sql_select] @@ -59,7 +59,7 @@ Selecting data into a file An example usage of the DatabricksSqlOperator to select data from a table and store in a file is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sql.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sql.py :language: python :start-after: [START howto_operator_databricks_sql_select_file] :end-before: [END howto_operator_databricks_sql_select_file] @@ -69,7 +69,7 @@ Executing multiple statements An example usage of the DatabricksSqlOperator to perform multiple SQL statements is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sql.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sql.py :language: python :start-after: [START howto_operator_databricks_sql_multiple] :end-before: [END howto_operator_databricks_sql_multiple] @@ -80,7 +80,7 @@ Executing multiple statements from a file An example usage of the DatabricksSqlOperator to perform statements from a file is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sql.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sql.py :language: python :start-after: [START howto_operator_databricks_sql_multiple_file] :end-before: [END howto_operator_databricks_sql_multiple_file] @@ -107,7 +107,7 @@ Examples -------- Configuring Databricks connection to be used with the Sensor. -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sensors.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_databricks_connection_setup] @@ -115,7 +115,7 @@ Configuring Databricks connection to be used with the Sensor. Poking the specific table with the SQL statement: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sensors.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_databricks_sql] @@ -154,7 +154,7 @@ Examples -------- Configuring Databricks connection to be used with the Sensor. -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sensors.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_databricks_connection_setup] @@ -162,7 +162,7 @@ Configuring Databricks connection to be used with the Sensor. Poking the specific table for existence of data/partition: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_sensors.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_databricks_partition] diff --git a/docs/apache-airflow-providers-databricks/operators/submit_run.rst b/docs/apache-airflow-providers-databricks/operators/submit_run.rst index 706920458c64c..10548583cfa3f 100644 --- a/docs/apache-airflow-providers-databricks/operators/submit_run.rst +++ b/docs/apache-airflow-providers-databricks/operators/submit_run.rst @@ -113,7 +113,7 @@ Specifying parameters as JSON An example usage of the DatabricksSubmitRunOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_json] :end-before: [END howto_operator_databricks_json] @@ -123,7 +123,7 @@ Using named parameters You can also use named parameters to initialize the operator and run the job. -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_named] :end-before: [END howto_operator_databricks_named] diff --git a/docs/apache-airflow-providers-databricks/operators/task.rst b/docs/apache-airflow-providers-databricks/operators/task.rst index 476e72c494b9b..331481d915c4b 100644 --- a/docs/apache-airflow-providers-databricks/operators/task.rst +++ b/docs/apache-airflow-providers-databricks/operators/task.rst @@ -33,14 +33,14 @@ Examples Running a notebook in Databricks using DatabricksTaskOperator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_task_notebook] :end-before: [END howto_operator_databricks_task_notebook] Running a SQL query in Databricks using DatabricksTaskOperator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks.py :language: python :start-after: [START howto_operator_databricks_task_sql] :end-before: [END howto_operator_databricks_task_sql] diff --git a/docs/apache-airflow-providers-databricks/operators/workflow.rst b/docs/apache-airflow-providers-databricks/operators/workflow.rst index 6da38add6669e..b5c81050143a8 100644 --- a/docs/apache-airflow-providers-databricks/operators/workflow.rst +++ b/docs/apache-airflow-providers-databricks/operators/workflow.rst @@ -45,7 +45,7 @@ Examples Example of what a DAG looks like with a DatabricksWorkflowTaskGroup ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. exampleinclude:: /../../tests/system/providers/databricks/example_databricks_workflow.py +.. exampleinclude:: /../../providers/tests/system/databricks/example_databricks_workflow.py :language: python :start-after: [START howto_databricks_workflow_notebook] :end-before: [END howto_databricks_workflow_notebook] diff --git a/docs/apache-airflow-providers-datadog/changelog.rst b/docs/apache-airflow-providers-datadog/changelog.rst index e392f2d04075b..43f0ce3e63a01 100644 --- a/docs/apache-airflow-providers-datadog/changelog.rst +++ b/docs/apache-airflow-providers-datadog/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/datadog/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/datadog/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-dbt-cloud/changelog.rst b/docs/apache-airflow-providers-dbt-cloud/changelog.rst index f4b14c26b0034..be4203ad0c942 100644 --- a/docs/apache-airflow-providers-dbt-cloud/changelog.rst +++ b/docs/apache-airflow-providers-dbt-cloud/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/dbt/cloud/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/dbt/cloud/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-dbt-cloud/index.rst b/docs/apache-airflow-providers-dbt-cloud/index.rst index f9f214af8323f..82d72999eaee5 100644 --- a/docs/apache-airflow-providers-dbt-cloud/index.rst +++ b/docs/apache-airflow-providers-dbt-cloud/index.rst @@ -53,14 +53,14 @@ an Integrated Developer Environment (IDE). :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/dbt/cloud/index> + System Tests <_api/tests/system/dbt/cloud/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-dbt-cloud/operators.rst b/docs/apache-airflow-providers-dbt-cloud/operators.rst index af5b900d2340c..eaa285f6d4082 100644 --- a/docs/apache-airflow-providers-dbt-cloud/operators.rst +++ b/docs/apache-airflow-providers-dbt-cloud/operators.rst @@ -67,7 +67,7 @@ The below examples demonstrate how to instantiate DbtCloudRunJobOperator tasks w asynchronous waiting for run termination, respectively. To note, the ``account_id`` for the operators is referenced within the ``default_args`` of the example DAG. -.. exampleinclude:: /../../tests/system/providers/dbt/cloud/example_dbt_cloud.py +.. exampleinclude:: /../../providers/tests/system/dbt/cloud/example_dbt_cloud.py :language: python :dedent: 4 :start-after: [START howto_operator_dbt_cloud_run_job] @@ -76,7 +76,7 @@ referenced within the ``default_args`` of the example DAG. This next example also shows how to pass in custom runtime configuration (in this case for ``threads_override``) via the ``additional_run_config`` dictionary. -.. exampleinclude:: /../../tests/system/providers/dbt/cloud/example_dbt_cloud.py +.. exampleinclude:: /../../providers/tests/system/dbt/cloud/example_dbt_cloud.py :language: python :dedent: 4 :start-after: [START howto_operator_dbt_cloud_run_job_async] @@ -95,7 +95,7 @@ In the example below, the ``run_id`` value in the example below comes from the o DbtCloudRunJobOperator task by utilizing the ``.output`` property exposed for all operators. Also, to note, the ``account_id`` for the task is referenced within the ``default_args`` of the example DAG. -.. exampleinclude:: /../../tests/system/providers/dbt/cloud/example_dbt_cloud.py +.. exampleinclude:: /../../providers/tests/system/dbt/cloud/example_dbt_cloud.py :language: python :dedent: 4 :start-after: [START howto_operator_dbt_cloud_run_job_sensor] @@ -104,7 +104,7 @@ the ``account_id`` for the task is referenced within the ``default_args`` of the Also, you can poll for status of the job run asynchronously using ``deferrable`` mode. In this mode, worker slots are freed up while the sensor is running. -.. exampleinclude:: /../../tests/system/providers/dbt/cloud/example_dbt_cloud.py +.. exampleinclude:: /../../providers/tests/system/dbt/cloud/example_dbt_cloud.py :language: python :dedent: 4 :start-after: [START howto_operator_dbt_cloud_run_job_sensor_deferred] @@ -125,7 +125,7 @@ downloaded. For more information on dbt Cloud artifacts, reference `this documentation `__. -.. exampleinclude:: /../../tests/system/providers/dbt/cloud/example_dbt_cloud.py +.. exampleinclude:: /../../providers/tests/system/dbt/cloud/example_dbt_cloud.py :language: python :dedent: 4 :start-after: [START howto_operator_dbt_cloud_get_artifact] @@ -146,7 +146,7 @@ If a ``project_id`` is supplied, only jobs pertaining to this project id will be For more information on dbt Cloud list jobs, reference `this documentation `__. -.. exampleinclude:: /../../tests/system/providers/dbt/cloud/example_dbt_cloud.py +.. exampleinclude:: /../../providers/tests/system/dbt/cloud/example_dbt_cloud.py :language: python :dedent: 4 :start-after: [START howto_operator_dbt_cloud_list_jobs] diff --git a/docs/apache-airflow-providers-dingding/changelog.rst b/docs/apache-airflow-providers-dingding/changelog.rst index 43f13ec33466e..7bf7561b35dbf 100644 --- a/docs/apache-airflow-providers-dingding/changelog.rst +++ b/docs/apache-airflow-providers-dingding/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/dingding/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/dingding/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-dingding/index.rst b/docs/apache-airflow-providers-dingding/index.rst index be557b9bc5a1b..fd799c323c749 100644 --- a/docs/apache-airflow-providers-dingding/index.rst +++ b/docs/apache-airflow-providers-dingding/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/dingding/index> + System Tests <_api/tests/system/dingding/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-dingding/operators.rst b/docs/apache-airflow-providers-dingding/operators.rst index dc3ca1b5ffe9e..600c321b9f1f7 100644 --- a/docs/apache-airflow-providers-dingding/operators.rst +++ b/docs/apache-airflow-providers-dingding/operators.rst @@ -38,7 +38,7 @@ Basic Usage Use the :class:`~airflow.providers.dingding.operators.dingding.DingdingOperator` to send message through `DingTalk Custom Robot `__: -.. exampleinclude:: /../../tests/system/providers/dingding/example_dingding.py +.. exampleinclude:: /../../providers/tests/system/dingding/example_dingding.py :language: python :dedent: 4 :start-after: [START howto_operator_dingding] @@ -51,7 +51,7 @@ Remind users in message Use parameters ``at_mobiles`` and ``at_all`` to remind specific users when you send message, ``at_mobiles`` will be ignored When ``at_all`` is set to ``True``: -.. exampleinclude:: /../../tests/system/providers/dingding/example_dingding.py +.. exampleinclude:: /../../providers/tests/system/dingding/example_dingding.py :language: python :dedent: 4 :start-after: [START howto_operator_dingding_remind_users] @@ -66,7 +66,7 @@ can send rich text messages including link, markdown, actionCard and feedCard through `DingTalk Custom Robot `__. A rich text message can not remind specific users except by using markdown type message: -.. exampleinclude:: /../../tests/system/providers/dingding/example_dingding.py +.. exampleinclude:: /../../providers/tests/system/dingding/example_dingding.py :language: python :dedent: 4 :start-after: [START howto_operator_dingding_rich_text] @@ -80,7 +80,7 @@ Sending messages from a Task callback and then pass the function to ``sla_miss_callback``, ``on_success_callback``, ``on_failure_callback``, or ``on_retry_callback``. Here we use ``on_failure_callback`` as an example: -.. exampleinclude:: /../../tests/system/providers/dingding/example_dingding.py +.. exampleinclude:: /../../providers/tests/system/dingding/example_dingding.py :language: python :start-after: [START howto_operator_dingding_failure_callback] :end-before: [END howto_operator_dingding_failure_callback] diff --git a/docs/apache-airflow-providers-discord/changelog.rst b/docs/apache-airflow-providers-discord/changelog.rst index c5056231dabec..aec69ca9c5b1f 100644 --- a/docs/apache-airflow-providers-discord/changelog.rst +++ b/docs/apache-airflow-providers-discord/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/discord/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/discord/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-docker/changelog.rst b/docs/apache-airflow-providers-docker/changelog.rst index 46a39571be6e1..910d96f6cbe01 100644 --- a/docs/apache-airflow-providers-docker/changelog.rst +++ b/docs/apache-airflow-providers-docker/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/docker/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/docker/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-docker/decorators/docker.rst b/docs/apache-airflow-providers-docker/decorators/docker.rst index cfcd51860920b..df5b237bfa710 100644 --- a/docs/apache-airflow-providers-docker/decorators/docker.rst +++ b/docs/apache-airflow-providers-docker/decorators/docker.rst @@ -162,7 +162,7 @@ ulimits Usage Example ------------- -.. exampleinclude:: /../../tests/system/providers/docker/example_taskflow_api_docker_virtualenv.py +.. exampleinclude:: /../../providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py :language: python :start-after: [START transform_docker] :end-before: [END transform_docker] diff --git a/docs/apache-airflow-providers-docker/index.rst b/docs/apache-airflow-providers-docker/index.rst index 24593b8755889..4f5f263f263d1 100644 --- a/docs/apache-airflow-providers-docker/index.rst +++ b/docs/apache-airflow-providers-docker/index.rst @@ -42,14 +42,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/docker/index> + System Tests <_api/tests/system/docker/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-edge/changelog.rst b/docs/apache-airflow-providers-edge/changelog.rst index 4a87ccc753b07..46bd53ed4ccc6 100644 --- a/docs/apache-airflow-providers-edge/changelog.rst +++ b/docs/apache-airflow-providers-edge/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/edge/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/edge/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-elasticsearch/changelog.rst b/docs/apache-airflow-providers-elasticsearch/changelog.rst index 496bc18ce3663..840359e3b0a4e 100644 --- a/docs/apache-airflow-providers-elasticsearch/changelog.rst +++ b/docs/apache-airflow-providers-elasticsearch/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/elasticsearch/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/elasticsearch/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-elasticsearch/connections/elasticsearch.rst b/docs/apache-airflow-providers-elasticsearch/connections/elasticsearch.rst index c068abe1a3c77..8097b8bd61c4f 100644 --- a/docs/apache-airflow-providers-elasticsearch/connections/elasticsearch.rst +++ b/docs/apache-airflow-providers-elasticsearch/connections/elasticsearch.rst @@ -72,7 +72,7 @@ For example: export AIRFLOW_CONN_ELASTICSEARCH_DEFAULT='elasticsearch://elasticsearchlogin:elasticsearchpassword@elastic.co:80/http' -.. exampleinclude:: /../../tests/system/providers/elasticsearch/example_elasticsearch_query.py +.. exampleinclude:: /../../providers/tests/system/elasticsearch/example_elasticsearch_query.py :language: python :dedent: 4 :start-after: [START howto_elasticsearch_query] diff --git a/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_python_hook.rst b/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_python_hook.rst index ff1f925182ce0..537b4973b41cf 100644 --- a/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_python_hook.rst +++ b/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_python_hook.rst @@ -36,7 +36,7 @@ es_conn_args Usage Example --------------------- -.. exampleinclude:: /../../tests/system/providers/elasticsearch/example_elasticsearch_query.py +.. exampleinclude:: /../../providers/tests/system/elasticsearch/example_elasticsearch_query.py :language: python :start-after: [START howto_elasticsearch_python_hook] :end-before: [END howto_elasticsearch_python_hook] diff --git a/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_sql_hook.rst b/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_sql_hook.rst index 6021f4ac5dfeb..084d445cb0bad 100644 --- a/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_sql_hook.rst +++ b/docs/apache-airflow-providers-elasticsearch/hooks/elasticsearch_sql_hook.rst @@ -26,7 +26,7 @@ Elasticsearch Hook that interact with Elasticsearch through the elasticsearch-db Usage Example --------------------- -.. exampleinclude:: /../../tests/system/providers/elasticsearch/example_elasticsearch_query.py +.. exampleinclude:: /../../providers/tests/system/elasticsearch/example_elasticsearch_query.py :language: python :start-after: [START howto_elasticsearch_query] :end-before: [END howto_elasticsearch_query] diff --git a/docs/apache-airflow-providers-elasticsearch/index.rst b/docs/apache-airflow-providers-elasticsearch/index.rst index 727d852d7cf26..6aeca1e89e97d 100644 --- a/docs/apache-airflow-providers-elasticsearch/index.rst +++ b/docs/apache-airflow-providers-elasticsearch/index.rst @@ -51,7 +51,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/elasticsearch/index> + System Tests <_api/tests/system/elasticsearch/index> .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-exasol/changelog.rst b/docs/apache-airflow-providers-exasol/changelog.rst index 998d537e49b47..b1f41f01d8ee0 100644 --- a/docs/apache-airflow-providers-exasol/changelog.rst +++ b/docs/apache-airflow-providers-exasol/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/exasol/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/exasol/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-fab/auth-manager/access-control.rst b/docs/apache-airflow-providers-fab/auth-manager/access-control.rst index 5292c3f1f6296..914635942b7c1 100644 --- a/docs/apache-airflow-providers-fab/auth-manager/access-control.rst +++ b/docs/apache-airflow-providers-fab/auth-manager/access-control.rst @@ -46,7 +46,7 @@ Viewer ^^^^^^ ``Viewer`` users have limited read permissions: -.. exampleinclude:: /../../airflow/providers/fab/auth_manager/security_manager/override.py +.. exampleinclude:: /../../providers/src/airflow/providers/fab/auth_manager/security_manager/override.py :language: python :start-after: [START security_viewer_perms] :end-before: [END security_viewer_perms] @@ -55,7 +55,7 @@ User ^^^^ ``User`` users have ``Viewer`` permissions plus additional permissions: -.. exampleinclude:: /../../airflow/providers/fab/auth_manager/security_manager/override.py +.. exampleinclude:: /../../providers/src/airflow/providers/fab/auth_manager/security_manager/override.py :language: python :start-after: [START security_user_perms] :end-before: [END security_user_perms] @@ -64,7 +64,7 @@ Op ^^ ``Op`` users have ``User`` permissions plus additional permissions: -.. exampleinclude:: /../../airflow/providers/fab/auth_manager/security_manager/override.py +.. exampleinclude:: /../../providers/src/airflow/providers/fab/auth_manager/security_manager/override.py :language: python :start-after: [START security_op_perms] :end-before: [END security_op_perms] @@ -74,7 +74,7 @@ Admin ``Admin`` users have all possible permissions, including granting or revoking permissions from other users. ``Admin`` users have ``Op`` permission plus additional permissions: -.. exampleinclude:: /../../airflow/providers/fab/auth_manager/security_manager/override.py +.. exampleinclude:: /../../providers/src/airflow/providers/fab/auth_manager/security_manager/override.py :language: python :start-after: [START security_admin_perms] :end-before: [END security_admin_perms] diff --git a/docs/apache-airflow-providers-fab/changelog.rst b/docs/apache-airflow-providers-fab/changelog.rst index c6bdcaa11e75a..41fc2e14c26a5 100644 --- a/docs/apache-airflow-providers-fab/changelog.rst +++ b/docs/apache-airflow-providers-fab/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/fab/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/fab/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-facebook/changelog.rst b/docs/apache-airflow-providers-facebook/changelog.rst index e6717758dc284..e84fbb0ce3094 100644 --- a/docs/apache-airflow-providers-facebook/changelog.rst +++ b/docs/apache-airflow-providers-facebook/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/facebook/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/facebook/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-ftp/changelog.rst b/docs/apache-airflow-providers-ftp/changelog.rst index c9c69b3a7d2aa..36f8c3d14951e 100644 --- a/docs/apache-airflow-providers-ftp/changelog.rst +++ b/docs/apache-airflow-providers-ftp/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/ftp/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/ftp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-ftp/index.rst b/docs/apache-airflow-providers-ftp/index.rst index 269f712d8d257..25dfa5ef7d756 100644 --- a/docs/apache-airflow-providers-ftp/index.rst +++ b/docs/apache-airflow-providers-ftp/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/ftp/index> + System Tests <_api/tests/system/ftp/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources @@ -65,7 +65,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/ftp/index> + System Tests <_api/tests/system/ftp/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-ftp/operators/index.rst b/docs/apache-airflow-providers-ftp/operators/index.rst index 48933b9ed8a21..3e2f6592e1c35 100644 --- a/docs/apache-airflow-providers-ftp/operators/index.rst +++ b/docs/apache-airflow-providers-ftp/operators/index.rst @@ -33,7 +33,7 @@ For parameter definition take a look at :class:`~airflow.providers.ftp.operators The below example shows how to use the FTPFileTransmitOperator to transfer a locally stored file to a remote FTP Server: -.. exampleinclude:: /../../tests/system/providers/ftp/example_ftp.py +.. exampleinclude:: /../../providers/tests/system/ftp/example_ftp.py :language: python :dedent: 4 :start-after: [START howto_operator_ftp_put] @@ -41,7 +41,7 @@ The below example shows how to use the FTPFileTransmitOperator to transfer a loc The below example shows how to use the FTPFileTransmitOperator to pull a file from a remote FTP Server. -.. exampleinclude:: /../../tests/system/providers/ftp/example_ftp.py +.. exampleinclude:: /../../providers/tests/system/ftp/example_ftp.py :language: python :dedent: 4 :start-after: [START howto_operator_ftp_get] @@ -63,7 +63,7 @@ For parameter definition take a look at :class:`~airflow.providers.ftp.operators The below example shows how to use the FTPSFileTransmitOperator to transfer a locally stored file to a remote FTPS Server: -.. exampleinclude:: /../../tests/system/providers/ftp/example_ftp.py +.. exampleinclude:: /../../providers/tests/system/ftp/example_ftp.py :language: python :dedent: 4 :start-after: [START howto_operator_ftps_put] @@ -71,7 +71,7 @@ The below example shows how to use the FTPSFileTransmitOperator to transfer a lo The below example shows how to use the FTPSFileTransmitOperator to pull a file from a remote FTPS Server. -.. exampleinclude:: /../../tests/system/providers/ftp/example_ftp.py +.. exampleinclude:: /../../providers/tests/system/ftp/example_ftp.py :language: python :dedent: 4 :start-after: [START howto_operator_ftps_get] diff --git a/docs/apache-airflow-providers-github/changelog.rst b/docs/apache-airflow-providers-github/changelog.rst index 93bddd8d65fb4..231425db490d2 100644 --- a/docs/apache-airflow-providers-github/changelog.rst +++ b/docs/apache-airflow-providers-github/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/github/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/github/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-github/index.rst b/docs/apache-airflow-providers-github/index.rst index eb8fc3249f986..847dba31d1068 100644 --- a/docs/apache-airflow-providers-github/index.rst +++ b/docs/apache-airflow-providers-github/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/github/index> + System Tests <_api/tests/system/github/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-github/operators/index.rst b/docs/apache-airflow-providers-github/operators/index.rst index 1b2bf63894af1..448fb8dc3dc20 100644 --- a/docs/apache-airflow-providers-github/operators/index.rst +++ b/docs/apache-airflow-providers-github/operators/index.rst @@ -33,7 +33,7 @@ You can further process the result using An example of Listing all Repositories owned by a user, **client.get_user().get_repos()** can be implemented as following: -.. exampleinclude:: /../../tests/system/providers/github/example_github.py +.. exampleinclude:: /../../providers/tests/system/github/example_github.py :language: python :dedent: 4 :start-after: [START howto_operator_list_repos_github] @@ -43,7 +43,7 @@ An example of Listing all Repositories owned by a user, **client.get_user().get_ An example of Listing Tags in a Repository, **client.get_repo(full_name_or_id='apache/airflow').get_tags()** can be implemented as following: -.. exampleinclude:: /../../tests/system/providers/github/example_github.py +.. exampleinclude:: /../../providers/tests/system/github/example_github.py :language: python :dedent: 4 :start-after: [START howto_operator_list_tags_github] @@ -64,7 +64,7 @@ a Tag in `GitHub `__. An example for tag **v1.0**: -.. exampleinclude:: /../../tests/system/providers/github/example_github.py +.. exampleinclude:: /../../providers/tests/system/github/example_github.py :language: python :dedent: 4 :start-after: [START howto_tag_sensor_github] @@ -73,7 +73,7 @@ An example for tag **v1.0**: Similar Functionality can be achieved by directly using :class:`~from airflow.providers.github.sensors.github.GithubSensor`. -.. exampleinclude:: /../../tests/system/providers/github/example_github.py +.. exampleinclude:: /../../providers/tests/system/github/example_github.py :language: python :dedent: 4 :start-after: [START howto_sensor_github] diff --git a/docs/apache-airflow-providers-google/changelog.rst b/docs/apache-airflow-providers-google/changelog.rst index d18b3ef2017a9..f078efd1e3af5 100644 --- a/docs/apache-airflow-providers-google/changelog.rst +++ b/docs/apache-airflow-providers-google/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/google/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/google/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-google/example-dags.rst b/docs/apache-airflow-providers-google/example-dags.rst index 49f4fbf327f12..15245b948c6c1 100644 --- a/docs/apache-airflow-providers-google/example-dags.rst +++ b/docs/apache-airflow-providers-google/example-dags.rst @@ -19,10 +19,10 @@ Example DAGs ============ You can learn how to use Google integrations by analyzing the source code of the example DAGs: -* `Google Ads `__ +* `Google Ads `__ * `Google Cloud (legacy) `__ -* `Google Cloud `__ -* `Google Firebase `__ +* `Google Cloud `__ +* `Google Firebase `__ * `Google Marketing Platform `__ -* `Google Workplace `__ (formerly Google Suite) -* `Google LevelDB `__ +* `Google Workplace `__ (formerly Google Suite) +* `Google LevelDB `__ diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst index 05e05725aa17e..172b8d0931e7c 100644 --- a/docs/apache-airflow-providers-google/index.rst +++ b/docs/apache-airflow-providers-google/index.rst @@ -52,7 +52,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/google/index> + System Tests <_api/tests/system/google/index> .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-google/operators/ads.rst b/docs/apache-airflow-providers-google/operators/ads.rst index 8e228bc0fde93..af5fe5913a9d2 100644 --- a/docs/apache-airflow-providers-google/operators/ads.rst +++ b/docs/apache-airflow-providers-google/operators/ads.rst @@ -32,7 +32,7 @@ Google Ads to GCS To query the Google Ads API and generate a CSV report of the results use ``GoogleAdsToGcsOperator``. -.. exampleinclude:: /../../tests/system/providers/google/ads/example_ads.py +.. exampleinclude:: /../../providers/tests/system/google/ads/example_ads.py :language: python :dedent: 4 :start-after: [START howto_google_ads_to_gcs_operator] @@ -58,7 +58,7 @@ Upload Google Ads Accounts to GCS To upload Google Ads accounts to Google Cloud Storage bucket use the ``GoogleAdsListAccountsOperator``. -.. exampleinclude:: /../../tests/system/providers/google/ads/example_ads.py +.. exampleinclude:: /../../providers/tests/system/google/ads/example_ads.py :language: python :dedent: 4 :start-after: [START howto_ads_list_accounts_operator] diff --git a/docs/apache-airflow-providers-google/operators/cloud/automl.rst b/docs/apache-airflow-providers-google/operators/cloud/automl.rst index 8a92f49ac34f1..ebfe3ca501f44 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/automl.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/automl.rst @@ -46,7 +46,7 @@ All the functionality of legacy AutoML Natural Language, Vision, Video Intellige available on the Vertex AI platform. Please use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.CreateDatasetOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/automl/example_automl_dataset.py :language: python :dedent: 4 :start-after: [START howto_operator_automl_create_dataset] @@ -55,7 +55,7 @@ available on the Vertex AI platform. Please use After creating a dataset you can use it to import some data using :class:`~airflow.providers.google.cloud.operators.automl.AutoMLImportDataOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/automl/example_automl_dataset.py :language: python :dedent: 4 :start-after: [START howto_operator_automl_import_data] @@ -69,7 +69,7 @@ All the functionality of legacy AutoML Natural Language, Vision, Video Intellige available on the Vertex AI platform. Please use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.UpdateDatasetOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_update_dataset_operator] @@ -111,7 +111,7 @@ available on the Vertex AI platform. Please use You can find example on how to use VertexAI operators for AutoML Vision classification here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_vision_classification.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/automl/example_automl_vision_classification.py :language: python :dedent: 4 :start-after: [START howto_cloud_create_image_classification_training_job_operator] @@ -119,7 +119,7 @@ You can find example on how to use VertexAI operators for AutoML Vision classifi Example on how to use VertexAI operators for AutoML Video Intelligence classification you can find here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_video_classification.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/automl/example_automl_video_classification.py :language: python :dedent: 4 :start-after: [START howto_cloud_create_video_classification_training_job_operator] @@ -131,7 +131,7 @@ datasets. To create and import data to the dataset please use and :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.ImportDataOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_translation.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/automl/example_automl_translation.py :language: python :dedent: 4 :start-after: [START howto_operator_automl_create_model] @@ -145,7 +145,7 @@ and will be removed after 31.03.2024. Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.model_service.GetModelOperator` instead. You can find example on how to use VertexAI operators here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_get_model_operator] @@ -159,7 +159,7 @@ and will be removed after 31.03.2024. Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.DeployModelOperator` instead. You can find example on how to use VertexAI operators here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_deploy_model_operator] @@ -173,7 +173,7 @@ and will be removed after 31.03.2024. Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteModelOperator` instead. You can find example on how to use VertexAI operators here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_model_operator] @@ -190,13 +190,12 @@ To obtain predictions from Google Cloud AutoML model you can use :class:`~airflow.providers.google.cloud.operators.automl.AutoMLBatchPredictOperator`. In the first case the model must be deployed. -.. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_translation.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/automl/example_automl_translation.py :language: python :dedent: 4 :start-after: [START howto_operator_prediction] :end-before: [END howto_operator_prediction] - Th :class:`~airflow.providers.google.cloud.operators.automl.AutoMLBatchPredictOperator` deprecated for tables, video intelligence, vision and natural language is deprecated and will be removed after 31.03.2024. Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job.CreateBatchPredictionJobOperator`, @@ -206,19 +205,19 @@ video intelligence, vision and natural language is deprecated and will be remove instead. You can find examples on how to use VertexAI operators here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_batch_prediction_job_operator] :end-before: [END how_to_cloud_vertex_ai_create_batch_prediction_job_operator] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_batch_prediction_job_operator] :end-before: [END how_to_cloud_vertex_ai_list_batch_prediction_job_operator] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_batch_prediction_job_operator] @@ -239,7 +238,7 @@ and will be removed after 31.03.2024. Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.ListDatasetsOperator` instead. You can find example on how to use VertexAI operators here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_dataset_operator] @@ -253,7 +252,7 @@ and will be removed after 31.03.2024. Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.DeleteDatasetOperator` instead. You can find example on how to use VertexAI operators here: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_dataset_operator] diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst index 1635372a35d7a..5e2c6689c2365 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst @@ -43,7 +43,7 @@ Create dataset To create an empty dataset in a BigQuery database you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyDatasetOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_create_dataset] @@ -59,7 +59,7 @@ To get the details of an existing dataset you can use This operator returns a `Dataset Resource `__. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_get_dataset] @@ -73,7 +73,7 @@ List tables in dataset To retrieve the list of tables in a given dataset use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryGetDatasetTablesOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_get_dataset_tables] @@ -90,7 +90,7 @@ To update a table in BigQuery you can use The update method replaces the entire Table resource, whereas the patch method only replaces fields that are provided in the submitted Table resource. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_update_table] @@ -107,7 +107,7 @@ To update a dataset in BigQuery you can use The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_update_dataset] @@ -121,7 +121,7 @@ Delete dataset To delete an existing dataset from a BigQuery database you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteDatasetOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_delete_dataset] @@ -144,7 +144,7 @@ ways. You may either directly pass the schema fields in, or you may point the operator to a Google Cloud Storage object name. The object in Google Cloud Storage must be a JSON file with the schema fields in it. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_create_table] @@ -152,7 +152,7 @@ Storage must be a JSON file with the schema fields in it. You can use this operator to create a view on top of an existing table. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_create_view] @@ -161,7 +161,7 @@ You can use this operator to create a view on top of an existing table. You can also use this operator to create a materialized view that periodically cache results of a query for increased performance and efficiency. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_create_materialized_view] @@ -180,7 +180,7 @@ Similarly to :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyTableOperator` you can directly pass the schema fields in. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_create_external_table] @@ -188,7 +188,7 @@ you can directly pass the schema fields in. Or you may point the operator to a Google Cloud Storage object name where the schema is stored. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_create_table_schema_json] @@ -213,7 +213,7 @@ nesting will a nested list where elements would represent the column values for that row. ``True`` - A Python list of dictionaries, where each dictionary represents a row. In each dictionary, the keys are the column names and the values are the corresponding values for those columns. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 8 :start-after: [START howto_operator_bigquery_get_data] @@ -224,7 +224,7 @@ The below example shows how to use in async (deferrable) mode. Note that a deferrable task requires the Triggerer to be running on your Airflow deployment. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_get_data_async] @@ -241,7 +241,7 @@ To upsert a table you can use This operator either updates the existing table or creates a new, empty table in the given dataset. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_upsert_table] @@ -258,7 +258,7 @@ To update the schema of a table you can use This operator updates the schema field values supplied, while leaving the rest unchanged. This is useful for instance to set new field descriptions on an existing table schema. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_update_table_schema] @@ -272,7 +272,7 @@ Delete table To delete an existing table you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_delete_table] @@ -280,7 +280,7 @@ To delete an existing table you can use You can also use this operator to delete a view. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_delete_view] @@ -288,7 +288,7 @@ You can also use this operator to delete a view. You can also use this operator to delete a materialized view. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_delete_materialized_view] @@ -301,7 +301,7 @@ Execute BigQuery jobs Let's say you would like to execute the following query. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 0 :start-after: [START howto_operator_bigquery_query] @@ -311,7 +311,7 @@ To execute the SQL query in a specific BigQuery database you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryInsertJobOperator` with proper query job configuration that can be Jinja templated. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 8 :start-after: [START howto_operator_bigquery_insert_job] @@ -322,7 +322,7 @@ The below example shows how to use in async (deferrable) mode. Note that a deferrable task requires the Triggerer to be running on your Airflow deployment. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_insert_job_async] @@ -334,7 +334,7 @@ For more information on types of BigQuery job please check If you want to include some files in your configuration you can use ``include`` clause of Jinja template language as follow: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 8 :start-after: [START howto_operator_bigquery_select_job] @@ -350,7 +350,7 @@ then it will reattach to the existing job. Also for all this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_insert_job_async] @@ -371,7 +371,7 @@ This operator expects a sql query that will return a single row. Each value on that first row is evaluated using python ``bool`` casting. If any of the values return ``False`` the check is failed and errors out. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 8 :start-after: [START howto_operator_bigquery_check] @@ -379,7 +379,7 @@ return ``False`` the check is failed and errors out. Also you can use deferrable mode in this operator -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_check_async] @@ -397,7 +397,7 @@ These operators expects a sql query that will return a single row. Each value on that first row is evaluated against ``pass_value`` which can be either a string or numeric value. If numeric, you can also specify ``tolerance``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 8 :start-after: [START howto_operator_bigquery_value_check] @@ -405,7 +405,7 @@ or numeric value. If numeric, you can also specify ``tolerance``. Also you can use deferrable mode in this operator -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_value_check_async] @@ -421,7 +421,7 @@ tolerance of the ones from ``days_back`` before you can either use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckOperator` or :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckAsyncOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 8 :start-after: [START howto_operator_bigquery_interval_check] @@ -429,7 +429,7 @@ tolerance of the ones from ``days_back`` before you can either use Also you can use deferrable mode in this operator -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_interval_check_async] @@ -443,7 +443,7 @@ Check columns with predefined tests To check that columns pass user-configurable tests you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryColumnCheckOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_column_check] @@ -457,7 +457,7 @@ Check table level data quality To check that tables pass user-defined tests you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryTableCheckOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_table_check] @@ -475,7 +475,7 @@ use the ``{{ ds_nodash }}`` macro as the table name suffix. :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_bigquery_table] @@ -483,13 +483,13 @@ use the ``{{ ds_nodash }}`` macro as the table name suffix. Also you can use deferrable mode in this operator if you would like to free up the worker slots while the sensor is running. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_bigquery_table_defered] :end-before: [END howto_sensor_bigquery_table_defered] -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_async_bigquery_table] @@ -501,7 +501,7 @@ Check that a Table Partition exists To check that a table exists and has a partition you can use. :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTablePartitionExistenceSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_bigquery_table_partition] @@ -511,13 +511,13 @@ For DAY partitioned tables, the partition_id parameter is a string on the "%Y%m% Also you can use deferrable mode in this operator if you would like to free up the worker slots while the sensor is running. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_bigquery_table_partition_defered] :end-before: [END howto_sensor_bigquery_table_partition_defered] -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py :language: python :dedent: 4 :start-after: [START howto_sensor_bigquery_table_partition_async] diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst index 0d3663419baad..002716c8ef630 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst @@ -47,7 +47,7 @@ for example :class:`~airflow.providers.google.cloud.operators.bigquery_dts.BigQu scheduling option is present in passed configuration. If present then nothing is done, otherwise its value is set to ``True``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py :language: python :start-after: [START howto_bigquery_dts_create_args] :end-before: [END howto_bigquery_dts_create_args] @@ -55,7 +55,7 @@ set to ``True``. You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Basic usage of the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py :language: python :dedent: 4 :start-after: [START howto_bigquery_create_data_transfer] @@ -78,7 +78,7 @@ To delete DTS transfer configuration you can use Basic usage of the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py :language: python :dedent: 4 :start-after: [START howto_bigquery_delete_data_transfer] @@ -99,7 +99,7 @@ Start manual transfer runs to be executed now with schedule_time equal to curren Basic usage of the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py :language: python :dedent: 4 :start-after: [START howto_bigquery_start_transfer] @@ -112,7 +112,7 @@ parameters which allows you to dynamically determine values. To check if operation succeeded you can use :class:`~airflow.providers.google.cloud.sensors.bigquery_dts.BigQueryDataTransferServiceTransferRunSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py :language: python :dedent: 4 :start-after: [START howto_bigquery_dts_sensor] diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigtable.rst b/docs/apache-airflow-providers-google/operators/cloud/bigtable.rst index 870cfe50fce82..d77a689447244 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/bigtable.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/bigtable.rst @@ -42,7 +42,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigtable/example_bigtable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigtable/example_bigtable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_bigtable_instance_create] @@ -65,7 +65,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigtable/example_bigtable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigtable/example_bigtable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_bigtable_instance_update] @@ -85,7 +85,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigtable/example_bigtable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigtable/example_bigtable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_bigtable_instance_delete] @@ -105,7 +105,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigtable/example_bigtable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigtable/example_bigtable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_bigtable_cluster_update] @@ -129,7 +129,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigtable/example_bigtable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigtable/example_bigtable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_bigtable_table_create] @@ -157,7 +157,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigtable/example_bigtable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigtable/example_bigtable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_bigtable_table_delete] @@ -182,7 +182,7 @@ timeout hits and does not raise any exception. Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigtable/example_bigtable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigtable/example_bigtable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_bigtable_table_wait_for_replication] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_batch.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_batch.rst index 2254ead25b1a3..c4ae3db7de395 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_batch.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_batch.rst @@ -35,7 +35,7 @@ For more information about the Job object fields, visit `Google Cloud Batch Job A simple job configuration can look as follows: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py :language: python :dedent: 0 :start-after: [START howto_operator_batch_job_creation] @@ -44,7 +44,7 @@ A simple job configuration can look as follows: With this configuration we can submit the job: :class:`~airflow.providers.google.cloud.operators.cloud_batch.CloudBatchSubmitJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py :language: python :dedent: 4 :start-after: [START howto_operator_batch_submit_job] @@ -53,7 +53,7 @@ With this configuration we can submit the job: or you can define the same operator in the deferrable mode: :class:`~airflow.providers.google.cloud.operators.cloud_batch.CloudBatchSubmitJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py :language: python :dedent: 4 :start-after: [START howto_operator_batch_submit_job_deferrable_mode] @@ -68,7 +68,7 @@ To list the tasks of a certain job, you can use: :class:`~airflow.providers.google.cloud.operators.cloud_batch.CloudBatchListTasksOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py :language: python :dedent: 4 :start-after: [START howto_operator_batch_list_tasks] @@ -83,7 +83,7 @@ To list the jobs, you can use: :class:`~airflow.providers.google.cloud.operators.cloud_batch.CloudBatchListJobsOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py :language: python :dedent: 4 :start-after: [START howto_operator_batch_list_jobs] @@ -98,7 +98,7 @@ To delete a job you can use: :class:`~airflow.providers.google.cloud.operators.cloud_batch.CloudBatchDeleteJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_job] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_build.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_build.rst index f7d91154095fc..e5e8e601b030c 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_build.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_build.rst @@ -46,7 +46,7 @@ Using the operator Cancel a build in progress with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildCancelBuildOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_cancel_build] @@ -74,7 +74,7 @@ Build configuration In order to trigger a build, it is necessary to pass the build configuration. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 0 :start-after: [START howto_operator_gcp_create_build_from_storage_body] @@ -82,7 +82,7 @@ In order to trigger a build, it is necessary to pass the build configuration. In addition, a build can refer to source stored in `Google Cloud Source Repositories `__. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 0 :start-after: [START howto_operator_create_build_from_repo_body] @@ -96,7 +96,7 @@ Using the operator Trigger a build is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildCreateBuildOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_from_storage] @@ -104,7 +104,7 @@ Trigger a build is performed with the You can use deferrable mode for this action in order to run the operator asynchronously: -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_from_storage_async] @@ -115,7 +115,7 @@ You can use :ref:`Jinja templating ` with parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_from_storage_result] @@ -124,7 +124,7 @@ to be used by other operators. By default, after the build is created, it will wait for the build operation to complete. If there is no need to wait for complete, you can pass wait=False as example shown below. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_without_wait] @@ -132,7 +132,7 @@ you can pass wait=False as example shown below. You can use deferrable mode for this action in order to run the operator asynchronously: -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_without_wait_async] @@ -141,7 +141,7 @@ You can use deferrable mode for this action in order to run the operator asynchr In order to start a build on Cloud Build you can use a build configuration file. A build config file defines the fields that are needed for Cloud Build to perform your tasks. You can write the build config file using the YAML or the JSON syntax. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_create_build_from_yaml_body] @@ -149,7 +149,7 @@ that are needed for Cloud Build to perform your tasks. You can write the build c You can use deferrable mode for this action in order to run the operator asynchronously: -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_create_build_from_yaml_body_async] @@ -158,7 +158,7 @@ You can use deferrable mode for this action in order to run the operator asynchr In addition, a Cloud Build can refer to source stored in `Google Cloud Source Repositories `__. Once build has started, it ill build the code in source repositories. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 0 :start-after: [START howto_operator_create_build_from_repo] @@ -166,7 +166,7 @@ Once build has started, it ill build the code in source repositories. You can use deferrable mode for this action in order to run the operator asynchronously: -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_from_repo_async] @@ -189,7 +189,7 @@ Using the operator Creates a new Cloud Build trigger with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildCreateBuildTriggerOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_trigger] @@ -216,7 +216,7 @@ Using the operator Deletes a new Cloud Build trigger with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildDeleteBuildTriggerOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_build_trigger] @@ -243,7 +243,7 @@ Using the operator Returns information about a previously requested build with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildGetBuildOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_get_build] @@ -270,7 +270,7 @@ Using the operator Returns information about a Cloud Build trigger with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildGetBuildTriggerOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_get_build_trigger] @@ -297,7 +297,7 @@ Using the operator Lists all the existing Cloud Build triggers with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildListBuildTriggersOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_list_build_triggers] @@ -324,7 +324,7 @@ Using the operator Lists previously requested builds with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildListBuildsOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_list_builds] @@ -352,7 +352,7 @@ Using the operator Creates a new build based on the specified build with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildRetryBuildOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build.py :language: python :dedent: 4 :start-after: [START howto_operator_retry_build] @@ -379,7 +379,7 @@ Using the operator Runs a trigger at a particular source revision with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildRunBuildTriggerOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_run_build_trigger] @@ -406,7 +406,7 @@ Using the operator Updates a Cloud Build trigger with the :class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildUpdateBuildTriggerOperator` operator. -.. exampleinclude:: ../../../../tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py +.. exampleinclude:: ../../../../providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_create_build_trigger] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst index f8f00fbe6c54a..7b05bbc6baa0a 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst @@ -39,7 +39,7 @@ For more information about the available fields to pass when creating a environm A simple environment configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 0 :start-after: [START howto_operator_composer_simple_environment] @@ -48,7 +48,7 @@ A simple environment configuration can look as followed: With this configuration we can create the environment: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerCreateEnvironmentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_create_composer_environment] @@ -57,7 +57,7 @@ With this configuration we can create the environment: or you can define the same operator in the deferrable mode: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerCreateEnvironmentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_create_composer_environment_deferrable_mode] @@ -70,7 +70,7 @@ To get a environment you can use: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerGetEnvironmentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_get_composer_environment] @@ -83,7 +83,7 @@ To get a environment you can use: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerListEnvironmentsOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_list_composer_environments] @@ -98,7 +98,7 @@ For more information on updateMask and other parameters take a look at `Cloud Co An example of a new service config and the updateMask: -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 0 :start-after: [START howto_operator_composer_update_environment] @@ -107,7 +107,7 @@ An example of a new service config and the updateMask: To update a service you can use: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerUpdateEnvironmentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_update_composer_environment] @@ -116,7 +116,7 @@ To update a service you can use: or you can define the same operator in the deferrable mode: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerCreateEnvironmentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_update_composer_environment_deferrable_mode] @@ -129,7 +129,7 @@ To delete a service you can use: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerDeleteEnvironmentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_composer_environment] @@ -138,7 +138,7 @@ To delete a service you can use: or you can define the same operator in the deferrable mode: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerDeleteEnvironmentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_composer_environment_deferrable_mode] @@ -152,7 +152,7 @@ You can also list all supported Cloud Composer images: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerListImageVersionsOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_composer_image_list] @@ -164,7 +164,7 @@ Run Airflow CLI commands You can run Airflow CLI commands in your environments, use: :class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerRunAirflowCLICommandOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_run_airflow_cli_command] @@ -172,7 +172,7 @@ You can run Airflow CLI commands in your environments, use: or you can define the same operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_operator_run_airflow_cli_command_deferrable_mode] @@ -184,7 +184,7 @@ Check if a DAG run has completed You can use sensor that checks if a DAG run has completed in your environments, use: :class:`~airflow.providers.google.cloud.sensors.cloud_composer.CloudComposerDAGRunSensor` -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_sensor_dag_run] @@ -192,7 +192,7 @@ You can use sensor that checks if a DAG run has completed in your environments, or you can define the same sensor in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/composer/example_cloud_composer.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/composer/example_cloud_composer.py :language: python :dedent: 4 :start-after: [START howto_sensor_dag_run_deferrable_mode] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore.rst index 7e7c04ba582d2..9adeea88513a3 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore.rst @@ -41,7 +41,7 @@ presented as a compatible dictionary also. Here is an example of instance -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :start-after: [START howto_operator_instance] :end-before: [END howto_operator_instance] @@ -59,7 +59,7 @@ make a use of the service account listed under ``persistenceIamIdentity``. You can use :class:`~airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator` operator to set permissions. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_set_acl_permission] @@ -76,7 +76,7 @@ Create instance Create a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreCreateInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_create_instance] @@ -87,7 +87,7 @@ You can use :ref:`Jinja templating ` with parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_create_instance_result] @@ -102,7 +102,7 @@ Delete instance Delete a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreDeleteInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_instance] @@ -120,7 +120,7 @@ Export instance Delete a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreExportInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_export_instance] @@ -138,7 +138,7 @@ Failover instance Delete a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreFailoverInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_failover_instance] @@ -156,7 +156,7 @@ Get instance Delete a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreGetInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_get_instance] @@ -174,7 +174,7 @@ Import instance Delete a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreImportOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_import_instance] @@ -192,7 +192,7 @@ List instances List a instances is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreListInstancesOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_list_instances] @@ -203,7 +203,7 @@ You can use :ref:`Jinja templating ` with parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_list_instances_result] @@ -217,7 +217,7 @@ Update instance Update a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreUpdateInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_update_instance] @@ -236,7 +236,7 @@ Scale instance Scale a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreScaleInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py :language: python :dedent: 4 :start-after: [START howto_operator_scale_instance] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore_memcached.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore_memcached.rst index 2fe980be0215f..6f52020c283be 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore_memcached.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore_memcached.rst @@ -41,7 +41,7 @@ The object can be presented as a compatible dictionary also. Here is an example of instance -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py :language: python :start-after: [START howto_operator_memcached_instance] :end-before: [END howto_operator_memcached_instance] @@ -56,7 +56,7 @@ Create a instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreMemcachedCreateInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py :language: python :dedent: 4 :start-after: [START howto_operator_create_instance_memcached] @@ -72,7 +72,7 @@ Delete an instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreMemcachedDeleteInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_instance_memcached] @@ -88,7 +88,7 @@ Get an instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreMemcachedGetInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py :language: python :dedent: 4 :start-after: [START howto_operator_get_instance_memcached] @@ -104,7 +104,7 @@ List instances is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreMemcachedListInstancesOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py :language: python :dedent: 4 :start-after: [START howto_operator_list_instances_memcached] @@ -120,7 +120,7 @@ Updating an instance is performed with the :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreMemcachedUpdateInstanceOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py :language: python :dedent: 4 :start-after: [START howto_operator_update_instance_memcached] @@ -138,7 +138,7 @@ and :class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreMemcachedApplyParametersOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py :language: python :dedent: 4 :start-after: [START howto_operator_update_and_apply_parameters_memcached] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst index 95a3b3a676679..65a28ea4f42c0 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst @@ -35,7 +35,7 @@ For more information about the Job object fields, visit `Google Cloud Run Job de A simple job configuration can be created with a Job object: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 0 :start-after: [START howto_cloud_run_job_instance_creation] @@ -43,7 +43,7 @@ A simple job configuration can be created with a Job object: or with a Python dictionary: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 0 :start-after: [START howto_cloud_run_job_dict_creation] @@ -52,7 +52,7 @@ or with a Python dictionary: You can create a Cloud Run Job with any of these configurations : :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunCreateJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_create_job] @@ -69,7 +69,7 @@ For more information about the Service object fields, visit `Google Cloud Run Se A simple service configuration can look as follows: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py :language: python :dedent: 0 :start-after: [START howto_operator_cloud_run_service_creation] @@ -79,7 +79,7 @@ A simple service configuration can look as follows: With this configuration we can create the service: :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunCreateServiceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_create_service] @@ -94,7 +94,7 @@ Delete a service With this configuration we can delete the service: :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunDeleteServiceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_delete_service] @@ -110,7 +110,7 @@ To execute a job, you can use: :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunExecuteJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_execute_job] @@ -120,7 +120,7 @@ or you can define the same operator in the deferrable mode: :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunExecuteJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_execute_job_deferrable_mode] @@ -130,7 +130,7 @@ You can also specify overrides that allow you to give a new entrypoint command t :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunExecuteJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_execute_job_with_overrides] @@ -144,7 +144,7 @@ To update a job, you can use: :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunUpdateJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_update_job] @@ -161,7 +161,7 @@ To list the jobs, you can use: :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunListJobsOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_list_jobs] @@ -177,7 +177,7 @@ To delete a job you can use: :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunDeleteJobOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_delete_job] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst index 42a32712867d9..e1b572046d609 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst @@ -42,7 +42,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_db_create] @@ -50,7 +50,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Example request body: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :start-after: [START howto_operator_cloudsql_db_create_body] :end-before: [END howto_operator_cloudsql_db_create_body] @@ -58,7 +58,7 @@ Example request body: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_db_create_template_fields] @@ -87,7 +87,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_db_delete] @@ -96,7 +96,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_db_delete_template_fields] @@ -127,7 +127,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_db_patch] @@ -135,7 +135,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Example request body: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :start-after: [START howto_operator_cloudsql_db_patch_body] :end-before: [END howto_operator_cloudsql_db_patch_body] @@ -143,7 +143,7 @@ Example request body: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_db_patch_template_fields] @@ -174,7 +174,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_delete] @@ -183,7 +183,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_delete_template_fields] @@ -215,7 +215,7 @@ Arguments Example body defining the export operation: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :start-after: [START howto_operator_cloudsql_export_body] :end-before: [END howto_operator_cloudsql_export_body] @@ -226,7 +226,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_export] @@ -234,7 +234,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Also for all this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_export_async] @@ -243,7 +243,7 @@ Also for all this action you can use operator in the deferrable mode: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_export_template_fields] @@ -268,7 +268,7 @@ To grant the service account with the appropriate WRITE permissions for the GCS you can use the :class:`~airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator`, as shown in the example: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_export_gcs_permissions] @@ -308,7 +308,7 @@ Arguments Example body defining the import operation: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :start-after: [START howto_operator_cloudsql_import_body] :end-before: [END howto_operator_cloudsql_import_body] @@ -319,7 +319,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_import] @@ -328,7 +328,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_import_template_fields] @@ -353,7 +353,7 @@ To grant the service account with the appropriate READ permissions for the GCS o you can use the :class:`~airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator`, as shown in the example: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_import_gcs_permissions] @@ -379,7 +379,7 @@ Arguments Example body defining the instance with failover replica: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :start-after: [START howto_operator_cloudsql_create_body] :end-before: [END howto_operator_cloudsql_create_body] @@ -390,7 +390,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_create] @@ -399,7 +399,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_create_template_fields] @@ -430,7 +430,7 @@ Arguments Example body defining the instance: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :start-after: [START howto_operator_cloudsql_patch_body] :end-before: [END howto_operator_cloudsql_patch_body] @@ -441,7 +441,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_patch] @@ -450,7 +450,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_patch_template_fields] @@ -484,7 +484,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_cloudsql_clone] @@ -493,7 +493,7 @@ Cloud connection used. Both variants are shown: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_clone_template_fields] @@ -558,14 +558,14 @@ NFS-like volumes in the same path for all the workers. Example connection definitions for all non-SSL connectivity. Note that all the components of the connection URI should be URL-encoded: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py :language: python :start-after: [START howto_operator_cloudsql_query_connections] :end-before: [END howto_operator_cloudsql_query_connections] Similar connection definition for all SSL-enabled connectivity: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py :language: python :start-after: [START howto_operator_cloudsql_query_connections] :end-before: [END howto_operator_cloudsql_query_connections] @@ -574,12 +574,12 @@ It is also possible to configure a connection via environment variable (note tha matches the :envvar:`AIRFLOW_CONN_{CONN_ID}` postfix uppercase if you are using a standard AIRFLOW notation for defining connection via environment variables): -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py :language: python :start-after: [START howto_operator_cloudsql_query_connections_env] :end-before: [END howto_operator_cloudsql_query_connections_env] -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py :language: python :start-after: [START howto_operator_cloudsql_query_connections_env] :end-before: [END howto_operator_cloudsql_query_connections_env] @@ -593,7 +593,7 @@ or the connection configured via environment variable (note that the connection :envvar:`AIRFLOW_CONN_{CONN_ID}` postfix uppercase if you are using a standard AIRFLOW notation for defining connection via environment variables): -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py :language: python :start-after: [START howto_operator_cloudsql_query_operators] :end-before: [END howto_operator_cloudsql_query_operators] @@ -603,7 +603,7 @@ will be overridden. One of the ways to do so is specifying paths to each certifi Note that these files will be copied into a temporary location with minimal required permissions for security reasons. -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py :language: python :start-after: [START howto_operator_cloudsql_query_operators_ssl] :end-before: [END howto_operator_cloudsql_query_operators_ssl] @@ -614,7 +614,7 @@ format is: {"sslcert": "", "sslkey": "", "sslrootcert": ""} -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py :language: python :start-after: [START howto_operator_cloudsql_query_operators_ssl_secret_id] :end-before: [END howto_operator_cloudsql_query_operators_ssl_secret_id] @@ -622,7 +622,7 @@ format is: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_sql.py :language: python :dedent: 4 :start-after: [START gcp_sql_query_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst index df3365631f1ce..4779ce0dfcd91 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst @@ -62,17 +62,17 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py :language: python :start-after: [START howto_operator_gcp_transfer_create_job_body_gcp] :end-before: [END howto_operator_gcp_transfer_create_job_body_gcp] -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :start-after: [START howto_operator_gcp_transfer_create_job_body_aws] :end-before: [END howto_operator_gcp_transfer_create_job_body_aws] -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_create_job] @@ -81,7 +81,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_job_create_template_fields] @@ -107,7 +107,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_delete_job] @@ -116,7 +116,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_job_delete_template_fields] @@ -142,7 +142,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_run_job] @@ -151,7 +151,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_job_run_template_fields] @@ -177,12 +177,12 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py :language: python :start-after: [START howto_operator_gcp_transfer_update_job_body] :end-before: [END howto_operator_gcp_transfer_update_job_body] -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_update_job] @@ -191,7 +191,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_job_update_template_fields] @@ -216,7 +216,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_cancel_operation] @@ -225,7 +225,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_operation_cancel_template_fields] @@ -252,7 +252,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_get_operation] @@ -261,7 +261,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_operation_get_template_fields] @@ -287,7 +287,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_list_operations] @@ -296,7 +296,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_operations_list_template_fields] @@ -321,7 +321,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_pause_operation] @@ -330,7 +330,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_operation_pause_template_fields] @@ -355,7 +355,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_resume_operation] @@ -364,7 +364,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_operation_resume_template_fields] @@ -390,7 +390,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_transfer_wait_operation] @@ -399,7 +399,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_job_sensor_template_fields] @@ -418,7 +418,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_transfer_gcs_to_gcs] @@ -427,7 +427,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py :language: python :dedent: 4 :start-after: [START gcp_transfer_gcs_to_gcs_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/compute.rst b/docs/apache-airflow-providers-google/operators/cloud/compute.rst index 1628db48e1364..df10dbc479c5e 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/compute.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/compute.rst @@ -39,7 +39,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_insert] @@ -48,7 +48,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection id used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_insert_no_project_id] @@ -58,7 +58,7 @@ from the Google Cloud connection id used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_insert_fields] @@ -84,7 +84,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_insert_from_template] @@ -93,7 +93,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection id used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_insert_from_template_no_project_id] @@ -103,7 +103,7 @@ from the Google Cloud connection id used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_insert_from_template_fields] @@ -130,7 +130,7 @@ Using the operator You can create the operator without project id - project id will be retrieved from the Google Cloud connection id used. The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_delete_no_project_id] @@ -140,7 +140,7 @@ from the Google Cloud connection id used. The code to create the operator: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_delete_template_fields] @@ -166,7 +166,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_start] @@ -175,7 +175,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection id used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_start_no_project_id] @@ -185,7 +185,7 @@ from the Google Cloud connection id used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_start_template_fields] @@ -212,7 +212,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_stop] @@ -221,7 +221,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_stop_no_project_id] @@ -230,7 +230,7 @@ from the Google Cloud connection used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_stop_template_fields] @@ -262,7 +262,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_set_machine_type] @@ -271,7 +271,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_set_machine_type_no_project_id] @@ -280,7 +280,7 @@ from the Google Cloud connection used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_set_machine_type_template_fields] @@ -310,7 +310,7 @@ The code to create the operator: You can create the operator without project id - project id will be retrieved from the Google Cloud connection used. The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_delete_old_template_no_project_id] @@ -319,7 +319,7 @@ from the Google Cloud connection used. The code to create the operator: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_template_delete_fields] @@ -346,7 +346,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_igm_insert_template] @@ -355,7 +355,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_igm_insert_template_no_project_id] @@ -364,7 +364,7 @@ from the Google Cloud connection used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_template_insert_fields] @@ -392,7 +392,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_igm_copy_template] @@ -401,7 +401,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_igm_copy_template_no_project_id] @@ -410,7 +410,7 @@ from the Google Cloud connection used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_instance_template_copy_operator_template_fields] @@ -441,7 +441,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :start-after: [START howto_operator_gce_insert_igm] :end-before: [END howto_operator_gce_insert_igm] @@ -449,7 +449,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_insert_igm_no_project_id] @@ -459,7 +459,7 @@ from the Google Cloud connection used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_igm_insert_fields] @@ -491,7 +491,7 @@ Using the operator You can create the operator without project id - project id will be retrieved from the Google Cloud connection used. The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_delete_igm_no_project_id] @@ -501,7 +501,7 @@ from the Google Cloud connection used. The code to create the operator: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_igm_delete_fields] @@ -532,7 +532,7 @@ Using the operator The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_igm_update_template] @@ -541,7 +541,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_igm.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_igm.py :language: python :dedent: 4 :start-after: [START howto_operator_gce_igm_update_template_no_project_id] @@ -551,7 +551,7 @@ from the Google Cloud connection used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/compute.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/compute.py :language: python :dedent: 4 :start-after: [START gce_igm_update_template_operator_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/compute_ssh.rst b/docs/apache-airflow-providers-google/operators/cloud/compute_ssh.rst index 8e50f01566aff..b4e5c10bbf186 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/compute_ssh.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/compute_ssh.rst @@ -45,7 +45,7 @@ Please note that the target instance must allow tcp traffic on port 22. Below is the code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_ssh.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_ssh.py :language: python :dedent: 4 :start-after: [START howto_execute_command_on_remote_1] @@ -54,7 +54,7 @@ Below is the code to create the operator: You can also create the hook without project id - project id will be retrieved from the Google credentials used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/compute/example_compute_ssh.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/compute/example_compute_ssh.py :language: python :dedent: 4 :start-after: [START howto_execute_command_on_remote_2] diff --git a/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst b/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst index 76ac1abbd1d06..2147e9f1ab9ab 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst @@ -37,7 +37,7 @@ Create Stored Info-Type To create a custom info-type you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateStoredInfoTypeOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_info_types.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_create_info_type] @@ -70,7 +70,7 @@ Update Stored Info-Type To update a info-type you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPUpdateStoredInfoTypeOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_info_types.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_update_info_type] @@ -85,7 +85,7 @@ Deleting Stored Info-Type To delete a info-type you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteStoredInfoTypeOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_info_types.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_delete_info_type] @@ -112,7 +112,7 @@ Creating Template To create a inspection template you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateInspectTemplateOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_inspect_template.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_create_inspect_template] @@ -139,7 +139,7 @@ Using Template To find potentially sensitive info using the inspection template we just created, we can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPInspectContentOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_inspect_template.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_use_inspect_template] @@ -161,7 +161,7 @@ Deleting Template To delete the template you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteInspectTemplateOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_inspect_template.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_delete_inspect_template] @@ -236,7 +236,7 @@ Creating Job Trigger To create a job trigger you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateJobTriggerOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job_trigger.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_create_job_trigger] @@ -261,7 +261,7 @@ Updating Job Trigger To update a job trigger you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPUpdateJobTriggerOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job_trigger.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_update_job_trigger] @@ -275,7 +275,7 @@ Deleting Job Trigger To delete a job trigger you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteJobTriggerOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job_trigger.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py :language: python :dedent: 4 :start-after: [START howto_operator_dlp_delete_job_trigger] @@ -297,7 +297,7 @@ Configuration information defines how you want the sensitive data de-identified. This config can either be saved and persisted in de-identification templates or defined in a :class:`~google.cloud.dlp_v2.types.DeidentifyConfig` object: -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py :language: python :start-after: [START dlp_deidentify_config_example] :end-before: [END dlp_deidentify_config_example] @@ -305,7 +305,7 @@ This config can either be saved and persisted in de-identification templates or To de-identify potentially sensitive information from a content item, you can use :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeidentifyContentOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py :language: python :dedent: 4 :start-after: [START _howto_operator_dlp_deidentify_content] diff --git a/docs/apache-airflow-providers-google/operators/cloud/datacatalog.rst b/docs/apache-airflow-providers-google/operators/cloud/datacatalog.rst index eaad69735e45d..10383b62ba25d 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/datacatalog.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/datacatalog.rst @@ -59,7 +59,7 @@ operators. The ``CloudDataCatalogGetEntryOperator`` use Project ID, Entry Group ID, Entry ID to get the entry. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_get_entry] @@ -71,7 +71,7 @@ parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_get_entry_result] @@ -79,7 +79,7 @@ The result is saved to :ref:`XCom `, which allows it to be used b The ``CloudDataCatalogLookupEntryOperator`` use the resource name to get the entry. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_lookup_entry_linked_resource] @@ -91,7 +91,7 @@ parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_lookup_entry_result] @@ -105,7 +105,7 @@ Creating an entry The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateEntryOperator` operator create the entry. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_entry_gcs] @@ -119,7 +119,7 @@ The result is saved to :ref:`XCom `, which allows it to be used b The newly created entry ID can be read with the ``entry_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_entry_gcs_result] @@ -133,7 +133,7 @@ Updating an entry The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateEntryOperator` operator update the entry. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_update_entry] @@ -151,7 +151,7 @@ Deleting a entry The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteEntryOperator` operator delete the entry. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_delete_entry] @@ -180,7 +180,7 @@ Creating an entry group The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateEntryGroupOperator` operator create the entry group. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_entry_group] @@ -194,7 +194,7 @@ The result is saved to :ref:`XCom `, which allows it to be used b The newly created entry group ID can be read with the ``entry_group_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_entry_group_result] @@ -208,7 +208,7 @@ Getting an entry group The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogGetEntryGroupOperator` operator get the entry group. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_get_entry_group] @@ -220,7 +220,7 @@ parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_get_entry_group_result] @@ -234,7 +234,7 @@ Deleting an entry group The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteEntryGroupOperator` operator delete the entry group. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_entries.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_entries.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_delete_entry_group] @@ -263,7 +263,7 @@ Creating a tag template The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateTagTemplateOperator` operator get the tag template. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_tag_template] @@ -277,7 +277,7 @@ The result is saved to :ref:`XCom `, which allows it to be used b The newly created tag template ID can be read with the ``tag_template_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_tag_template_result] @@ -291,7 +291,7 @@ Deleting a tag template The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagTemplateOperator` operator delete the tag template. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_delete_tag_template] @@ -310,7 +310,7 @@ Getting a tag template The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogGetTagTemplateOperator` operator get the tag template. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_get_tag_template] @@ -322,7 +322,7 @@ parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_get_tag_template_result] @@ -336,7 +336,7 @@ Updating a tag template The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateTagTemplateOperator` operator update the tag template. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_update_tag_template] @@ -365,7 +365,7 @@ Creating a tag on an entry The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateTagOperator` operator get the tag template. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tags.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tags.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_tag] @@ -379,7 +379,7 @@ The result is saved to :ref:`XCom `, which allows it to be used b The newly created tag ID can be read with the ``tag_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tags.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tags.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_tag_result] @@ -393,7 +393,7 @@ Updating a tag The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateTagOperator` operator update the tag template. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tags.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tags.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_update_tag] @@ -411,7 +411,7 @@ Deleting a tag The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagOperator` operator delete the tag template. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tags.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tags.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_delete_tag] @@ -429,7 +429,7 @@ Listing tags on an entry The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogListTagsOperator` operator get list of the tags on the entry. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tags.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tags.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_list_tags] @@ -441,7 +441,7 @@ parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tags.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tags.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_list_tags_result] @@ -467,7 +467,7 @@ Creating a field The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateTagTemplateFieldOperator` operator get the tag template field. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_tag_template_field] @@ -481,7 +481,7 @@ The result is saved to :ref:`XCom `, which allows it to be used b The newly created field ID can be read with the ``tag_template_field_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_create_tag_template_field_result] @@ -495,7 +495,7 @@ Renaming a field The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogRenameTagTemplateFieldOperator` operator rename the tag template field. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_rename_tag_template_field] @@ -513,7 +513,7 @@ Updating a field The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateTagTemplateFieldOperator` operator get the tag template field. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_update_tag_template_field] @@ -532,7 +532,7 @@ Deleting a field The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagTemplateFieldOperator` operator delete the tag template field. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_delete_tag_template_field] @@ -553,7 +553,7 @@ operator searches Data Catalog for multiple resources like entries, tags that ma The ``query`` parameters should defined using `search syntax `__. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_search_catalog] @@ -565,7 +565,7 @@ parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py +.. exampleinclude:: /../../providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_datacatalog_search_catalog_result] diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataflow.rst b/docs/apache-airflow-providers-google/operators/cloud/dataflow.rst index a9eb98ea9a509..21895e4e6e679 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/dataflow.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/dataflow.rst @@ -78,7 +78,7 @@ This will create a new pipeline that will be visible on Dataflow Pipelines UI. Here is an example of how you can create a Dataflow Pipeline by running DataflowCreatePipelineOperator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_pipeline.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_create_dataflow_pipeline] @@ -87,7 +87,7 @@ Here is an example of how you can create a Dataflow Pipeline by running Dataflow To run a newly created pipeline you can use :class:`~airflow.providers.google.cloud.operators.dataflow.DataflowRunPipelineOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_pipeline.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_run_dataflow_pipeline] @@ -119,7 +119,7 @@ has the ability to download or available on the local filesystem (provide the ab Here is an example of creating and running a pipeline in Java with jar stored on GCS: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_java.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py :language: python :dedent: 4 :start-after: [START howto_operator_start_java_job_jar_on_gcs] @@ -127,7 +127,7 @@ Here is an example of creating and running a pipeline in Java with jar stored on Here is an example of creating and running a pipeline in Java with jar stored on GCS in deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_java.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py :language: python :dedent: 4 :start-after: [START howto_operator_start_java_job_jar_on_gcs_deferrable] @@ -135,7 +135,7 @@ Here is an example of creating and running a pipeline in Java with jar stored on Here is an example of creating and running a pipeline in Java with jar stored on local file system: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_java.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py :language: python :dedent: 4 :start-after: [START howto_operator_start_java_job_local_jar] @@ -162,7 +162,7 @@ The ``py_system_site_packages`` argument specifies whether or not all the Python will be accessible within virtual environment (if ``py_requirements`` argument is specified), recommend avoiding unless the Dataflow job requires it. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_python.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_job] @@ -197,7 +197,7 @@ or Python file) and how it is written. In order for the Dataflow job to execute pipeline objects are not being waited upon (not calling ``waitUntilFinish`` or ``wait_until_finish`` on the ``PipelineResult`` in your application code). -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py :language: python :dedent: 4 :start-after: [START howto_operator_start_python_job_async] @@ -221,7 +221,7 @@ Streaming execution To execute a streaming Dataflow job, ensure the streaming option is set (for Python) or read from an unbounded data source, such as Pub/Sub, in your pipeline (for Java). -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_streaming_python.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py :language: python :dedent: 4 :start-after: [START howto_operator_start_streaming_python_job] @@ -249,7 +249,7 @@ See the `official documentation for Dataflow templates Here is an example of running a Dataflow job using a Classic Template with :class:`~airflow.providers.google.cloud.operators.dataflow.DataflowTemplatedJobStartOperator`: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_template.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_template.py :language: python :dedent: 4 :start-after: [START howto_operator_start_template_job] @@ -257,7 +257,7 @@ Here is an example of running a Dataflow job using a Classic Template with Also for this action you can use the operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_template.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_template.py :language: python :dedent: 4 :start-after: [START howto_operator_start_template_job_deferrable] @@ -269,7 +269,7 @@ See the `list of Google-provided templates that can be used with this operator Here is an example of running a Dataflow job using a Flex Template with :class:`~airflow.providers.google.cloud.operators.dataflow.DataflowStartFlexTemplateOperator`: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_template.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_template.py :language: python :dedent: 4 :start-after: [START howto_operator_start_flex_template_job] @@ -277,7 +277,7 @@ Here is an example of running a Dataflow job using a Flex Template with Also for this action you can use the operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_template.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_template.py :language: python :dedent: 4 :start-after: [START howto_operator_start_flex_template_job_deferrable] @@ -293,7 +293,7 @@ extensions for running Dataflow streaming jobs. Here is an example of running Dataflow SQL job with :class:`~airflow.providers.google.cloud.operators.dataflow.DataflowStartSqlJobOperator`: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py :language: python :dedent: 4 :start-after: [START howto_operator_start_sql_job] @@ -317,7 +317,7 @@ This API can be used to define both streaming and batch pipelines. Here is an example of running Dataflow YAML job with :class:`~airflow.providers.google.cloud.operators.dataflow.DataflowStartYamlJobOperator`: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_yaml.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py :language: python :dedent: 4 :start-after: [START howto_operator_dataflow_start_yaml_job] @@ -325,7 +325,7 @@ Here is an example of running Dataflow YAML job with This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_yaml.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py :language: python :dedent: 4 :start-after: [START howto_operator_dataflow_start_yaml_job_def] @@ -347,7 +347,7 @@ To stop one or more Dataflow pipelines you can use Streaming pipelines are drained by default, setting ``drain_pipeline`` to ``False`` will cancel them instead. Provide ``job_id`` to stop a specific job, or ``job_name_prefix`` to stop all jobs with provided name prefix. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_python.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py :language: python :dedent: 4 :start-after: [START howto_operator_stop_dataflow_job] @@ -364,7 +364,7 @@ To delete a Dataflow pipeline you can use :class:`~airflow.providers.google.cloud.operators.dataflow.DataflowDeletePipelineOperator`. Here is an example how you can use this operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_pipeline.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_dataflow_pipeline] @@ -382,7 +382,7 @@ When job is triggered asynchronously sensors may be used to run checks for speci :class:`~airflow.providers.google.cloud.sensors.dataflow.DataflowJobStatusSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_status] @@ -390,7 +390,7 @@ When job is triggered asynchronously sensors may be used to run checks for speci This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_sensors_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_status_deferrable] @@ -398,7 +398,7 @@ This operator can be run in deferrable mode by passing ``deferrable=True`` as a :class:`~airflow.providers.google.cloud.sensors.dataflow.DataflowJobMetricsSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_metric] @@ -406,7 +406,7 @@ This operator can be run in deferrable mode by passing ``deferrable=True`` as a This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_sensors_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_metric_deferrable] @@ -414,7 +414,7 @@ This operator can be run in deferrable mode by passing ``deferrable=True`` as a :class:`~airflow.providers.google.cloud.sensors.dataflow.DataflowJobMessagesSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_message] @@ -422,7 +422,7 @@ This operator can be run in deferrable mode by passing ``deferrable=True`` as a This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_sensors_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_message_deferrable] @@ -430,7 +430,7 @@ This operator can be run in deferrable mode by passing ``deferrable=True`` as a :class:`~airflow.providers.google.cloud.sensors.dataflow.DataflowJobAutoScalingEventsSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_autoscaling_event] @@ -438,7 +438,7 @@ This operator can be run in deferrable mode by passing ``deferrable=True`` as a This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataflow/example_dataflow_sensors_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py :language: python :dedent: 4 :start-after: [START howto_sensor_wait_for_job_autoscaling_event_deferrable] diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataform.rst b/docs/apache-airflow-providers-google/operators/cloud/dataform.rst index 97d9d28e223e0..09d8a6e6b8f93 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/dataform.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/dataform.rst @@ -40,7 +40,7 @@ Create repository for tracking your code in Dataform service. Example of usage c :class:`~airflow.providers.google.cloud.operators.dataform.DataformCreateRepositoryOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_operator_create_repository] @@ -52,7 +52,7 @@ Create workspace for storing your code in Dataform service. Example of usage can :class:`~airflow.providers.google.cloud.operators.dataform.DataformCreateWorkspaceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_operator_create_workspace] @@ -65,7 +65,7 @@ A simple configuration to create Compilation Result can look as followed: :class:`~airflow.providers.google.cloud.operators.dataform.DataformCreateCompilationResultOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_operator_create_compilation_result] @@ -78,7 +78,7 @@ To get a Compilation Result you can use: :class:`~airflow.providers.google.cloud.operators.dataform.DataformGetCompilationResultOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_get_compilation_result] @@ -95,13 +95,13 @@ We have possibility to run this operation in the sync mode and async, for async a sensor: :class:`~airflow.providers.google.cloud.operators.dataform.DataformWorkflowInvocationStateSensor` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_create_workflow_invocation] :end-before: [END howto_operator_create_workflow_invocation] -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_create_workflow_invocation_async] @@ -114,7 +114,7 @@ To get a Workflow Invocation you can use: :class:`~airflow.providers.google.cloud.operators.dataform.DataformGetWorkflowInvocationOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_get_workflow_invocation] @@ -127,7 +127,7 @@ To query Workflow Invocation Actions you can use: :class:`~airflow.providers.google.cloud.operators.dataform.DataformQueryWorkflowInvocationActionsOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_query_workflow_invocation_actions] @@ -140,7 +140,7 @@ To cancel a Workflow Invocation you can use: :class:`~airflow.providers.google.cloud.sensors.dataform.DataformCancelWorkflowInvocationOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_cancel_workflow_invocation] @@ -152,7 +152,7 @@ Deletes repository. Example of usage can be seen below: :class:`~airflow.providers.google.cloud.operators.dataform.DataformDeleteRepositoryOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_operator_delete_workspace] @@ -164,7 +164,7 @@ Deletes workspace. Example of usage can be seen below: :class:`~airflow.providers.google.cloud.operators.dataform.DataformDeleteRepositoryOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_operator_delete_repository] @@ -176,7 +176,7 @@ Removes file. Example of usage can be seen below: :class:`~airflow.providers.google.cloud.operators.dataform.DataformRemoveFileOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_operator_remove_file] @@ -188,7 +188,7 @@ Removes directory. Example of usage can be seen below: :class:`~airflow.providers.google.cloud.operators.dataform.DataformRemoveDirectoryOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_operator_remove_directory] @@ -200,7 +200,7 @@ Creates default projects structure for provided workspace. Before it can be done :class:`~airflow.providers.google.cloud.utils.dataform.make_initialization_workspace_flow` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 0 :start-after: [START howto_initialize_workspace] @@ -212,7 +212,7 @@ Writes file with given content to specified workspace. :class:`~airflow.providers.google.cloud.operators.dataform.DataformWriteFileOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_write_file] @@ -224,7 +224,7 @@ Make directory with given path in specified workspace. :class:`~airflow.providers.google.cloud.operators.dataform.DataformMakeDirectoryOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_make_directory] @@ -236,7 +236,7 @@ Installs npm packages for specified workspace :class:`~airflow.providers.google.cloud.operators.dataform.DataformInstallNpmPackagesOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataform/example_dataform.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataform/example_dataform.py :language: python :dedent: 4 :start-after: [START howto_operator_install_npm_packages] diff --git a/docs/apache-airflow-providers-google/operators/cloud/datafusion.rst b/docs/apache-airflow-providers-google/operators/cloud/datafusion.rst index 0a728cdb10f56..207479d5de96c 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/datafusion.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/datafusion.rst @@ -40,7 +40,7 @@ Restart DataFusion Instance To restart Data Fusion instance use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionRestartInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_restart_instance_operator] @@ -59,7 +59,7 @@ Delete DataFusion Instance To delete Data Fusion instance use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionDeleteInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_delete_instance_operator] @@ -79,7 +79,7 @@ Create DataFusion Instance To create Data Fusion instance use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionCreateInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_create_instance_operator] @@ -99,7 +99,7 @@ Update DataFusion Instance To update Data Fusion instance use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionUpdateInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_update_instance_operator] @@ -118,7 +118,7 @@ Get DataFusion Instance To retrieve Data Fusion instance use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionGetInstanceOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_get_instance_operator] @@ -138,7 +138,7 @@ Create a DataFusion pipeline To create Data Fusion pipeline use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionCreatePipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_create_pipeline] @@ -157,7 +157,7 @@ Start a DataFusion pipeline To start Data Fusion pipeline using synchronous mode: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionStartPipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_start_pipeline] @@ -166,7 +166,7 @@ To start Data Fusion pipeline using synchronous mode: To start Data Fusion pipeline using asynchronous mode: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionStartPipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_start_pipeline_async] @@ -179,7 +179,7 @@ It is not possible to use both asynchronous and deferrable parameters at the sam Please, check the example of using deferrable mode: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionStartPipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_start_pipeline_def] @@ -198,7 +198,7 @@ Stop a DataFusion pipeline To stop Data Fusion pipeline use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionStopPipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_stop_pipeline] @@ -217,7 +217,7 @@ Delete a DataFusion pipeline To delete Data Fusion pipeline use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionDeletePipelineOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_delete_pipeline] @@ -237,7 +237,7 @@ List DataFusion pipelines To list Data Fusion pipelines use: :class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionListPipelinesOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_list_pipelines] @@ -255,7 +255,7 @@ When start pipeline is triggered asynchronously sensors may be used to run check :class:`~airflow.providers.google.cloud.sensors.datafusion.CloudDataFusionPipelineStateSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/datafusion/example_datafusion.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datafusion/example_datafusion.py :language: python :dedent: 4 :start-after: [START howto_cloud_data_fusion_start_pipeline_sensor] diff --git a/docs/apache-airflow-providers-google/operators/cloud/datapipeline.rst b/docs/apache-airflow-providers-google/operators/cloud/datapipeline.rst index 10186138edb25..1123c48d6d792 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/datapipeline.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/datapipeline.rst @@ -51,7 +51,7 @@ The project id and location will be used to build the parent name needed to crea Here is an example of how you can create a Data Pipelines instance by running the above parameters with CreateDataPipelineOperator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/datapipelines/example_datapipeline.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datapipelines/example_datapipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_create_data_pipeline] @@ -79,7 +79,7 @@ The Project ID and Location will be used to build the parent name, which is wher You can run a Data Pipelines instance by running the above parameters with RunDataPipelineOperator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/datapipelines/example_datapipeline.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datapipelines/example_datapipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_run_data_pipeline] diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst b/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst index 1b8cc56dcf51d..788024ce0bbea 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst @@ -31,7 +31,7 @@ For more information about the available fields to pass when creating a task, vi A simple task configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 0 :start-after: [START howto_dataplex_configuration] @@ -40,13 +40,13 @@ A simple task configuration can look as followed: With this configuration we can create the task both synchronously & asynchronously: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateTaskOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_create_task_operator] :end-before: [END howto_dataplex_create_task_operator] -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_async_create_task_operator] @@ -59,7 +59,7 @@ To delete a task you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteTaskOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_delete_task_operator] @@ -72,7 +72,7 @@ To list tasks you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexListTasksOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_list_tasks_operator] @@ -85,7 +85,7 @@ To get a task you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetTaskOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_get_task_operator] @@ -98,7 +98,7 @@ To wait for a task created asynchronously you can use: :class:`~airflow.providers.google.cloud.sensors.dataplex.DataplexTaskStateSensor` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_task_state_sensor] @@ -113,7 +113,7 @@ For more information about the available fields to pass when creating a lake, vi A simple task configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 0 :start-after: [START howto_dataplex_lake_configuration] @@ -123,7 +123,7 @@ With this configuration we can create the lake: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateLakeOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_create_lake_operator] @@ -136,7 +136,7 @@ To delete a lake you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteLakeOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 :start-after: [START howto_dataplex_delete_lake_operator] @@ -150,7 +150,7 @@ For more information about the available fields to pass when creating a Data Qua A simple Data Quality scan configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 0 :start-after: [START howto_dataplex_data_quality_configuration] @@ -160,7 +160,7 @@ With this configuration we can create or update the Data Quality scan: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateOrUpdateDataQualityScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_create_data_quality_operator] @@ -173,7 +173,7 @@ To get a Data Quality scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataQualityScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_get_data_quality_operator] @@ -188,7 +188,7 @@ To delete a Data Quality scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteDataQualityScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_delete_data_quality_operator] @@ -201,7 +201,7 @@ You can run Dataplex Data Quality scan in asynchronous modes to later check its :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexRunDataQualityScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_run_data_quality_operator] @@ -211,7 +211,7 @@ To check that running Dataplex Data Quality scan succeeded you can use: :class:`~airflow.providers.google.cloud.sensors.dataplex.DataplexDataQualityJobStatusSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_data_scan_job_state_sensor] @@ -219,7 +219,7 @@ To check that running Dataplex Data Quality scan succeeded you can use: Also for this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_run_data_quality_def_operator] @@ -232,7 +232,7 @@ To get a Data Quality scan job you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataQualityScanResultOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_get_data_quality_job_operator] @@ -240,7 +240,7 @@ To get a Data Quality scan job you can use: Also for this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_get_data_quality_job_def_operator] @@ -255,7 +255,7 @@ For more information about the available fields to pass when creating a zone, vi A simple zone configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 0 :start-after: [START howto_dataplex_zone_configuration] @@ -265,7 +265,7 @@ With this configuration we can create a zone: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateZoneOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_create_zone_operator] @@ -278,7 +278,7 @@ To delete a zone you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteZoneOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_delete_zone_operator] @@ -293,7 +293,7 @@ For more information about the available fields to pass when creating a asset, v A simple asset configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 0 :start-after: [START howto_dataplex_asset_configuration] @@ -303,7 +303,7 @@ With this configuration we can create the asset: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateAssetOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_create_asset_operator] @@ -316,7 +316,7 @@ To delete a asset you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteAssetOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_delete_asset_operator] @@ -330,7 +330,7 @@ For more information about the available fields to pass when creating a Data Pro A simple Data Profile scan configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 0 :start-after: [START howto_dataplex_data_profile_configuration] @@ -340,7 +340,7 @@ With this configuration we can create or update the Data Profile scan: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateOrUpdateDataProfileScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_create_data_profile_operator] @@ -353,7 +353,7 @@ To get a Data Profile scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataProfileScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_get_data_profile_operator] @@ -368,7 +368,7 @@ To delete a Data Profile scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteDataProfileScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_delete_data_profile_operator] @@ -381,7 +381,7 @@ You can run Dataplex Data Profile scan in asynchronous modes to later check its :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexRunDataProfileScanOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_run_data_profile_operator] @@ -391,7 +391,7 @@ To check that running Dataplex Data Profile scan succeeded you can use: :class:`~airflow.providers.google.cloud.sensors.dataplex.DataplexDataProfileJobStatusSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_data_scan_job_state_sensor] @@ -399,7 +399,7 @@ To check that running Dataplex Data Profile scan succeeded you can use: Also for this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_run_data_profile_def_operator] @@ -412,7 +412,7 @@ To get a Data Profile scan job you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataProfileScanResultOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_get_data_profile_job_operator] diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataprep.rst b/docs/apache-airflow-providers-google/operators/cloud/dataprep.rst index 661443f26470b..8e7ad129fa0aa 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/dataprep.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/dataprep.rst @@ -59,7 +59,7 @@ To get information about jobs within a Cloud Dataprep job use: Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataprep/example_dataprep.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataprep/example_dataprep.py :language: python :dedent: 4 :start-after: [START how_to_dataprep_run_job_group_operator] @@ -77,7 +77,7 @@ To get information about jobs within a Cloud Dataprep job use: Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataprep/example_dataprep.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataprep/example_dataprep.py :language: python :dedent: 4 :start-after: [START how_to_dataprep_get_jobs_for_job_group_operator] @@ -96,7 +96,7 @@ To get information about jobs within a Cloud Dataprep job use: Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataprep/example_dataprep.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataprep/example_dataprep.py :language: python :dedent: 4 :start-after: [START how_to_dataprep_get_job_group_operator] @@ -112,7 +112,7 @@ To get information about jobs within a Cloud Dataprep job use: Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataprep/example_dataprep.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataprep/example_dataprep.py :language: python :dedent: 4 :start-after: [START how_to_dataprep_copy_flow_operator] @@ -130,7 +130,7 @@ To get information about jobs within a Cloud Dataprep job use: Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataprep/example_dataprep.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataprep/example_dataprep.py :language: python :dedent: 4 :start-after: [START how_to_dataprep_dataprep_run_flow_operator] @@ -148,7 +148,7 @@ To get information about jobs within a Cloud Dataprep job use: Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataprep/example_dataprep.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataprep/example_dataprep.py :language: python :dedent: 4 :start-after: [START how_to_dataprep_delete_flow_operator] @@ -167,7 +167,7 @@ To get information about jobs within a Cloud Dataprep job use: Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataprep/example_dataprep.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataprep/example_dataprep.py :language: python :dedent: 4 :start-after: [START how_to_dataprep_job_group_finished_sensor] diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst b/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst index 1f7bac8566caf..f8489dd6795e3 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst @@ -56,7 +56,7 @@ For more information about the available fields to pass when creating a cluster, A cluster configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_create_cluster] @@ -65,7 +65,7 @@ A cluster configuration can look as followed: With this configuration we can create the cluster: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_cluster_operator] @@ -84,7 +84,7 @@ This can be beneficial for running Dataproc workloads on GKE while optimizing co To create Dataproc cluster in Google Kubernetes Engine you could pass cluster configuration: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_create_cluster_in_gke_config] @@ -93,7 +93,7 @@ To create Dataproc cluster in Google Kubernetes Engine you could pass cluster co With this configuration we can create the cluster: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_cluster_operator_in_gke] @@ -105,7 +105,7 @@ Note that default image might not support the chosen optional component. If this is your case, please specify correct ``image_version`` that you can find in the `documentation. `__ -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_create_cluster] @@ -118,7 +118,7 @@ If this is your case, please specify correct ``image_version`` that you can find `documentation. `__ -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_create_cluster] @@ -126,7 +126,7 @@ If this is your case, please specify correct ``image_version`` that you can find You can use deferrable mode for this action in order to run the operator asynchronously: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_cluster_operator_async] @@ -139,7 +139,7 @@ this could be easily done using **make()** of :class:`~airflow.providers.google.cloud.operators.dataproc.ClusterGenerator` You can generate and use config as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_create_cluster_generate_cluster_config] @@ -156,7 +156,7 @@ For more information about the available fields to pass when diagnosing a cluste To diagnose a Dataproc cluster use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocDiagnoseClusterOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_diagnose_cluster] @@ -164,7 +164,7 @@ To diagnose a Dataproc cluster use: You can also use deferrable mode in order to run the operator asynchronously: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_diagnose_cluster_deferrable] @@ -178,7 +178,7 @@ For more information on updateMask and other parameters take a look at `Dataproc An example of a new cluster config and the updateMask: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_updatemask_cluster_operator] @@ -187,7 +187,7 @@ An example of a new cluster config and the updateMask: To update a cluster you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocUpdateClusterOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_update_cluster_operator] @@ -195,7 +195,7 @@ To update a cluster you can use: You can use deferrable mode for this action in order to run the operator asynchronously: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_update_cluster_operator_async] @@ -207,7 +207,7 @@ Starting a cluster To start a cluster you can use the :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocStartClusterOperator`: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_start_cluster_operator] @@ -219,7 +219,7 @@ Stopping a cluster To stop a cluster you can use the :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocStopClusterOperator`: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_stop_cluster_operator] @@ -231,7 +231,7 @@ Deleting a cluster To delete a cluster you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_delete_cluster_operator] @@ -239,7 +239,7 @@ To delete a cluster you can use: You can use deferrable mode for this action in order to run the operator asynchronously: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_delete_cluster_operator_async] @@ -258,7 +258,7 @@ file system. You can specify a file:/// path to refer to a local file on a clust The job configuration can be submitted by using: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_submit_job_to_cluster_operator] @@ -273,7 +273,7 @@ There are more arguments to provide in the jobs than the examples show. For the Example of the configuration for a PySpark Job: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_pyspark_config] @@ -281,7 +281,7 @@ Example of the configuration for a PySpark Job: Example of the configuration for a SparkSQl Job: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_sparksql_config] @@ -289,7 +289,7 @@ Example of the configuration for a SparkSQl Job: Example of the configuration for a Spark Job: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_spark_config] @@ -297,7 +297,7 @@ Example of the configuration for a Spark Job: Example of the configuration for a Spark Job running in `deferrable mode `__: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_spark_deferrable_config] @@ -305,7 +305,7 @@ Example of the configuration for a Spark Job running in `deferrable mode `__: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_cluster_for_persistent_history_server] @@ -434,7 +434,7 @@ with specific parameters. Documentation how create cluster you can find After Cluster was created you should add it to the Batch configuration. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_batch_operator_with_persistent_history_server] @@ -443,7 +443,7 @@ After Cluster was created you should add it to the Batch configuration. To check if operation succeeded you can use :class:`~airflow.providers.google.cloud.sensors.dataproc.DataprocBatchSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_batch_async_sensor] @@ -451,7 +451,7 @@ To check if operation succeeded you can use Also for all this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_batch_operator_async] @@ -463,7 +463,7 @@ Get a Batch To get a batch you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocGetBatchOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_get_batch_operator] @@ -475,7 +475,7 @@ List a Batch To get a list of exists batches you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocListBatchesOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_list_batches_operator] @@ -487,7 +487,7 @@ Delete a Batch To delete a batch you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocDeleteBatchOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_delete_batch_operator] @@ -499,7 +499,7 @@ Cancel a Batch Operation To cancel a operation you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCancelOperationOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_cancel_operation_operator] diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataproc_metastore.rst b/docs/apache-airflow-providers-google/operators/cloud/dataproc_metastore.rst index 6bf5bc9cf5408..7f04f9db81a16 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/dataproc_metastore.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/dataproc_metastore.rst @@ -33,7 +33,7 @@ For more information about the available fields to pass when creating a service, A simple service configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_metastore_create_service] @@ -42,7 +42,7 @@ A simple service configuration can look as followed: With this configuration we can create the service: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreCreateServiceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_create_service_operator] @@ -55,7 +55,7 @@ To get a service you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreGetServiceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_get_service_operator] @@ -69,7 +69,7 @@ For more information on updateMask and other parameters take a look at `Dataproc An example of a new service config and the updateMask: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_metastore_update_service] @@ -78,7 +78,7 @@ An example of a new service config and the updateMask: To update a service you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreUpdateServiceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_update_service_operator] @@ -91,7 +91,7 @@ To delete a service you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreDeleteServiceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_delete_service_operator] @@ -104,7 +104,7 @@ To export metadata you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreExportMetadataOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_export_metadata_operator] @@ -117,7 +117,7 @@ To restore a service you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreRestoreServiceOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_restore_service_operator] @@ -131,7 +131,7 @@ For more information about the available fields to pass when creating a metadata A simple metadata import configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_metastore_create_metadata_import] @@ -140,7 +140,7 @@ A simple metadata import configuration can look as followed: To create a metadata import you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreCreateMetadataImportOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_create_metadata_import_operator] @@ -154,7 +154,7 @@ For more information about the available fields to pass when creating a backup, A simple backup configuration can look as followed: -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py :language: python :dedent: 0 :start-after: [START how_to_cloud_dataproc_metastore_create_backup] @@ -163,7 +163,7 @@ A simple backup configuration can look as followed: With this configuration we can create the backup: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreCreateBackupOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_create_backup_operator] @@ -176,7 +176,7 @@ To delete a backup you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreDeleteBackupOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_delete_backup_operator] @@ -189,7 +189,7 @@ To list backups you can use: :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreListBackupsOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_list_backups_operator] @@ -201,7 +201,7 @@ Check Hive partitions existence To check that Hive partitions have been created in the Metastore for a given table you can use: :class:`~airflow.providers.google.cloud.sensors.dataproc_metastore.MetastoreHivePartitionSensor` -.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_metastore_hive_partition_sensor] diff --git a/docs/apache-airflow-providers-google/operators/cloud/datastore.rst b/docs/apache-airflow-providers-google/operators/cloud/datastore.rst index 8218e526a192c..05b441543fbc1 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/datastore.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/datastore.rst @@ -38,7 +38,7 @@ Export Entities To export entities from Google Cloud Datastore to Cloud Storage use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreExportEntitiesOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 4 :start-after: [START how_to_export_task] @@ -52,7 +52,7 @@ Import Entities To import entities from Cloud Storage to Google Cloud Datastore use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreImportEntitiesOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 4 :start-after: [START how_to_import_task] @@ -66,7 +66,7 @@ Allocate Ids To allocate IDs for incomplete keys use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreAllocateIdsOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 4 :start-after: [START how_to_allocate_ids] @@ -74,7 +74,7 @@ To allocate IDs for incomplete keys use An example of a partial keys required by the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 0 :start-after: [START how_to_keys_def] @@ -88,7 +88,7 @@ Begin transaction To begin a new transaction use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreBeginTransactionOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 4 :start-after: [START how_to_begin_transaction] @@ -96,7 +96,7 @@ To begin a new transaction use An example of a transaction options required by the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 0 :start-after: [START how_to_transaction_def] @@ -110,7 +110,7 @@ Commit transaction To commit a transaction, optionally creating, deleting or modifying some entities use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreCommitOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 4 :start-after: [START how_to_commit_task] @@ -118,7 +118,7 @@ use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreCo An example of a commit information required by the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 0 :start-after: [START how_to_commit_def] @@ -132,7 +132,7 @@ Run query To run a query for entities use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreRunQueryOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_query.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_query.py :language: python :dedent: 4 :start-after: [START how_to_run_query] @@ -140,7 +140,7 @@ To run a query for entities use An example of a query required by the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_query.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_query.py :language: python :dedent: 0 :start-after: [START how_to_query_def] @@ -154,7 +154,7 @@ Roll back transaction To roll back a transaction use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreRollbackOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_rollback.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_rollback.py :language: python :dedent: 4 :start-after: [START how_to_rollback_transaction] @@ -168,7 +168,7 @@ Get operation state To get the current state of a long-running operation use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreGetOperationOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 4 :start-after: [START get_operation_state] @@ -182,7 +182,7 @@ Delete operation To delete an operation use :class:`~airflow.providers.google.cloud.operators.datastore.CloudDatastoreDeleteOperationOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/datastore/example_datastore_commit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/datastore/example_datastore_commit.py :language: python :dedent: 4 :start-after: [START delete_operation] diff --git a/docs/apache-airflow-providers-google/operators/cloud/functions.rst b/docs/apache-airflow-providers-google/operators/cloud/functions.rst index d8ac6bbdd6d0c..3124e8e356f74 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/functions.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/functions.rst @@ -38,7 +38,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_functions/example_functions.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_functions/example_functions.py :language: python :dedent: 4 :start-after: [START howto_operator_gcf_delete] @@ -47,7 +47,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/functions.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/functions.py :language: python :dedent: 4 :start-after: [START gcf_function_delete_template_fields] @@ -77,7 +77,7 @@ Arguments When a DAG is created, the default_args dictionary can be used to pass arguments common with other tasks: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_functions/example_functions.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_functions/example_functions.py :language: python :start-after: [START howto_operator_gcf_default_args] :end-before: [END howto_operator_gcf_default_args] @@ -101,19 +101,19 @@ Using the operator Depending on the combination of parameters, the Function's source code can be obtained from different sources: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_functions/example_functions.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_functions/example_functions.py :language: python :start-after: [START howto_operator_gcf_deploy_body] :end-before: [END howto_operator_gcf_deploy_body] -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_functions/example_functions.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_functions/example_functions.py :language: python :start-after: [START howto_operator_gcf_deploy_variants] :end-before: [END howto_operator_gcf_deploy_variants] The code to create the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_functions/example_functions.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_functions/example_functions.py :language: python :dedent: 4 :start-after: [START howto_operator_gcf_deploy] @@ -122,7 +122,7 @@ The code to create the operator: You can also create the operator without project id - project id will be retrieved from the Google Cloud connection used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_functions/example_functions.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/cloud_functions/example_functions.py :language: python :dedent: 4 :start-after: [START howto_operator_gcf_deploy_no_project_id] @@ -131,7 +131,7 @@ from the Google Cloud connection used: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/functions.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/functions.py :language: python :dedent: 4 :start-after: [START gcf_function_deploy_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/gcs.rst b/docs/apache-airflow-providers-google/operators/cloud/gcs.rst index c4be59533a0a6..82d38fda87b80 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/gcs.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/gcs.rst @@ -46,7 +46,7 @@ The time span is defined by the time span's start and end timestamps. If a DAG does not have a *next* DAG instance scheduled, the time span end infinite, meaning the operator processes all files older than ``data_interval_start``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_transform_timespan.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_timespan_file_transform_operator_Task] @@ -66,7 +66,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_acl.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_acl.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_bucket_create_acl_entry_task] @@ -75,7 +75,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/gcs.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/gcs.py :language: python :dedent: 4 :start-after: [START gcs_bucket_create_acl_template_fields] @@ -100,7 +100,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_acl.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_acl.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_object_create_acl_entry_task] @@ -109,7 +109,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/gcs.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/gcs.py :language: python :dedent: 4 :start-after: [START gcs_object_create_acl_template_fields] @@ -131,7 +131,7 @@ Deleting Bucket allows you to remove bucket object from the Google Cloud Storage It is performed through the :class:`~airflow.providers.google.cloud.operators.gcs.GCSDeleteBucketOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_upload_download.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_delete_bucket] @@ -160,7 +160,7 @@ GCSObjectExistenceSensor Use the :class:`~airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceSensor` to wait (poll) for the existence of a file in Google Cloud Storage. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_object_exists_task] @@ -168,7 +168,7 @@ Use the :class:`~airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceSe Also you can use deferrable mode in this operator if you would like to free up the worker slots while the sensor is running. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_object_exists_task_defered] @@ -181,7 +181,7 @@ GCSObjectExistenceAsyncSensor :class:`~airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceAsyncSensor` is deprecated and will be removed in a future release. Please use :class:`~airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceSensor` and use the deferrable mode in that operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_object_exists_task_async] @@ -195,7 +195,7 @@ GCSObjectsWithPrefixExistenceSensor Use the :class:`~airflow.providers.google.cloud.sensors.gcs.GCSObjectsWithPrefixExistenceSensor` to wait (poll) for the existence of a file with a specified prefix in Google Cloud Storage. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_object_with_prefix_exists_task] @@ -205,7 +205,7 @@ You can set the ``deferrable`` param to True if you want this sensor to run asyn efficient utilization of resources in your Airflow deployment. However the triggerer component needs to be enabled for this functionality to work. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_object_with_prefix_exists_task_async] @@ -220,7 +220,7 @@ GCSUploadSessionCompleteSensor Use the :class:`~airflow.providers.google.cloud.sensors.gcs.GCSUploadSessionCompleteSensor` to check for a change in the number of files with a specified prefix in Google Cloud Storage. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_gcs_upload_session_complete_task] @@ -229,7 +229,7 @@ Use the :class:`~airflow.providers.google.cloud.sensors.gcs.GCSUploadSessionComp You can set the parameter ``deferrable`` to True if you want the worker slots to be freed up while sensor is running. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_gcs_upload_session_async_task] @@ -242,7 +242,7 @@ GCSObjectUpdateSensor Use the :class:`~airflow.providers.google.cloud.sensors.gcs.GCSObjectUpdateSensor` to check if an object is updated in Google Cloud Storage. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_object_update_exists_task] @@ -252,7 +252,7 @@ You can set the ``deferrable`` param to True if you want this sensor to run asyn utilization of resources in your Airflow deployment. However the triggerer component needs to be enabled for this functionality to work. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_object_update_exists_task_async] diff --git a/docs/apache-airflow-providers-google/operators/cloud/index.rst b/docs/apache-airflow-providers-google/operators/cloud/index.rst index f974c0eb81a4f..3daed7a11e9bf 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/index.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/index.rst @@ -29,4 +29,4 @@ Google Cloud Operators .. note:: You can learn how to use Google Cloud integrations by analyzing the - `source code `_ of the particular example DAGs. + `source code `_ of the particular example DAGs. diff --git a/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst b/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst index d71ebf87e929f..9eaf268824975 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst @@ -43,7 +43,7 @@ Create GKE cluster Here is an example of a cluster definition: -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py :language: python :start-after: [START howto_operator_gcp_gke_create_cluster_definition] :end-before: [END howto_operator_gcp_gke_create_cluster_definition] @@ -53,7 +53,7 @@ A dict object like this, or a definition, is required when creating a cluster with :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKECreateClusterOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_create_cluster] @@ -64,7 +64,7 @@ possibility to free up the worker when it knows it has to wait, and hand off the As a result, while it is suspended (deferred), it is not taking up a worker slot and your cluster will have a lot less resources wasted on idle Operators or Sensors: -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_create_cluster_async] @@ -84,7 +84,7 @@ To install and use Kueue on your cluster with the help of :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartKueueInsideClusterOperator` as shown in this example: -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py :language: python :start-after: [START howto_operator_gke_install_kueue] :end-before: [END howto_operator_gke_install_kueue] @@ -99,7 +99,7 @@ To delete a cluster, use :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEDeleteClusterOperator`. This would also delete all the nodes allocated to the cluster. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_delete_cluster] @@ -110,7 +110,7 @@ possibility to free up the worker when it knows it has to wait, and hand off the As a result, while it is suspended (deferred), it is not taking up a worker slot and your cluster will have a lot less resources wasted on idle Operators or Sensors: -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_delete_cluster_async] @@ -167,7 +167,7 @@ is the path ``/airflow/xcom``. To provide values to the XCom, ensure your Pod wr ``return.json`` in the sidecar. The contents of this can then be used downstream in your DAG. Here is an example of it being used: -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_start_pod_xcom] @@ -175,7 +175,7 @@ Here is an example of it being used: And then use it in other operators: -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_xcom_result] @@ -186,7 +186,7 @@ possibility to free up the worker when it knows it has to wait, and hand off the As a result, while it is suspended (deferred), it is not taking up a worker slot and your cluster will have a lot less resources wasted on idle Operators or Sensors: -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_start_pod_xcom_async] @@ -207,7 +207,7 @@ There are two operators available in order to run a job on a GKE cluster: There is no need to manage the ``kube_config`` file, as it will be generated automatically. All Kubernetes parameters (except ``config_file``) are also valid for the ``GKEStartJobOperator``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_start_job] @@ -216,7 +216,7 @@ All Kubernetes parameters (except ``config_file``) are also valid for the ``GKES ``GKEStartJobOperator`` also supports deferrable mode. Note that it makes sense only if the ``wait_until_job_complete`` parameter is set ``True``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_start_job_def] @@ -224,7 +224,7 @@ parameter is set ``True``. For run Job on a GKE cluster with Kueue enabled use ``GKEStartKueueJobOperator``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py :language: python :dedent: 4 :start-after: [START howto_operator_kueue_start_job] @@ -245,7 +245,7 @@ There are two operators available in order to delete a job on a GKE cluster: There is no need to manage the ``kube_config`` file, as it will be generated automatically. All Kubernetes parameters (except ``config_file``) are also valid for the ``GKEDeleteJobOperator``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_delete_job] @@ -260,7 +260,7 @@ Retrieve information about Job by given name You can use :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEDescribeJobOperator` to retrieve detailed description of existing Job by providing its name and namespace. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_describe_job] @@ -276,7 +276,7 @@ You can use :class:`~airflow.providers.google.cloud.operators.kubernetes_engine. list of existing Jobs. If ``namespace`` parameter is provided, output will include Jobs across given namespace. If ``namespace`` parameter is not specified, the information across all the namespaces will be outputted. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_list_jobs] @@ -291,7 +291,7 @@ Create a resource in a GKE cluster You can use :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKECreateCustomResourceOperator` to create resource in the specified Google Kubernetes Engine cluster. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_create_resource] @@ -306,7 +306,7 @@ Delete a resource in a GKE cluster You can use :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEDeleteCustomResourceOperator` to delete resource in the specified Google Kubernetes Engine cluster. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_delete_resource] @@ -321,7 +321,7 @@ Suspend a Job on a GKE cluster You can use :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKESuspendJobOperator` to suspend Job in the specified Google Kubernetes Engine cluster. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_suspend_job] @@ -336,7 +336,7 @@ Resume a Job on a GKE cluster You can use :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEResumeJobOperator` to resume Job in the specified Google Kubernetes Engine cluster. -.. exampleinclude:: /../../tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py :language: python :dedent: 4 :start-after: [START howto_operator_gke_resume_job] diff --git a/docs/apache-airflow-providers-google/operators/cloud/life_sciences.rst b/docs/apache-airflow-providers-google/operators/cloud/life_sciences.rst index 49f9b6c4ea9c2..6e7676e910508 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/life_sciences.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/life_sciences.rst @@ -34,7 +34,7 @@ Pipeline Configuration In order to run the pipeline, it is necessary to configure the request body. Here is an example of the pipeline configuration with a single action. -.. exampleinclude:: /../../tests/system/providers/google/cloud/life_sciences/example_life_sciences.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/life_sciences/example_life_sciences.py :language: python :dedent: 0 :start-after: [START howto_configure_simple_action_pipeline] @@ -42,7 +42,7 @@ Here is an example of the pipeline configuration with a single action. The pipeline can also be configured with multiple action. -.. exampleinclude:: /../../tests/system/providers/google/cloud/life_sciences/example_life_sciences.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/life_sciences/example_life_sciences.py :language: python :dedent: 0 :start-after: [START howto_configure_multiple_action_pipeline] @@ -59,7 +59,7 @@ Use the :class:`~airflow.providers.google.cloud.operators.life_sciences.LifeSciencesRunPipelineOperator` to execute pipelines. -.. exampleinclude:: /../../tests/system/providers/google/cloud/life_sciences/example_life_sciences.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/life_sciences/example_life_sciences.py :language: python :dedent: 0 :start-after: [START howto_run_pipeline] diff --git a/docs/apache-airflow-providers-google/operators/cloud/looker.rst b/docs/apache-airflow-providers-google/operators/cloud/looker.rst index f89d23c1c4c29..c8d3c2ad069ac 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/looker.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/looker.rst @@ -49,7 +49,7 @@ To submit a PDT materialization job to Looker you need to provide a model and vi The job configuration can be submitted in synchronous (blocking) mode by using: :class:`~airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator`. -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_looker.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_looker.py :language: python :dedent: 4 :start-after: [START how_to_cloud_looker_start_pdt_build_operator] @@ -60,7 +60,7 @@ Alternatively, the job configuration can be submitted in asynchronous mode by us :class:`~airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator` and :class:`~airflow.providers.google.cloud.sensors.looker.LookerCheckPdtBuildSensor`. -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_looker.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_looker.py :language: python :dedent: 4 :start-after: [START cloud_looker_async_start_pdt_sensor] diff --git a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst index f64705e1c267c..0848e1a28741a 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst @@ -49,7 +49,7 @@ This operator is deprecated. Please, use :class:`~airflow.providers.google.cloud.operators.vertex_ai.custom_job.CreateCustomPythonPackageTrainingJobOperator` instead. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_create_custom_python_training_job_v1] @@ -69,7 +69,7 @@ of any types. For example, you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.custom_job.CreateCustomPythonPackageTrainingJobOperator`. The result of running this operator will be ready-to-use model saved in Model Registry. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_create_custom_python_training_job_v1] @@ -87,7 +87,7 @@ This operator is deprecated. Please, use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.GetModelOperator` instead. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_get_model] @@ -98,7 +98,7 @@ fields to dynamically determine their values. The result are saved to :ref:`XCom allowing them to be used by other operators. In this case, the :class:`~airflow.providers.standard.operators.bash.BashOperator` is used to print the model information. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_print_model] @@ -120,7 +120,7 @@ instead. In this case, the new version of specific model could be created by spe ``parent_model`` parameter when running Training Job. This will ensure that new version of model will be trained except of creating new model. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_create_custom_python_training_job_v1] @@ -129,7 +129,7 @@ of creating new model. The :class:`~airflow.providers.google.cloud.operators.vertex_ai.custom_job.CreateCustomPythonPackageTrainingJobOperator` can also be used to create more versions with varying parameters. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_create_custom_python_training_job_v2] @@ -151,7 +151,7 @@ in format ``projects/{project}/locations/{location}/models/{model_id}@{version_i ``projects/{project}/locations/{location}/models/{model_id}@{version_alias}``. By default, the first model version created will be marked as default. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_default_version] @@ -166,7 +166,7 @@ This operator is deprecated. Please, use instead. You can pass the name of the desired model in ``model_id`` parameter. If the model ID is passed with version aliases, the operator will output all the versions available for this model. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_list_versions] @@ -185,7 +185,7 @@ This operator is deprecated. Please, use :class:`~airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job.CreateBatchPredictionJobOperator` instead. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_start_batch_prediction] @@ -204,7 +204,7 @@ This operator is deprecated. Please, use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteModelVersionOperator` instead. The default version could not be deleted on the model. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_delete_version] @@ -218,7 +218,7 @@ This operator is deprecated. Please, use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteModelOperator` instead. -.. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/ml_engine/example_mlengine.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_mlengine_delete_model] diff --git a/docs/apache-airflow-providers-google/operators/cloud/natural_language.rst b/docs/apache-airflow-providers-google/operators/cloud/natural_language.rst index 7b54e812c2dd5..0ba46ccf76b03 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/natural_language.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/natural_language.rst @@ -44,14 +44,14 @@ representing text. Here is an example of document with text provided as a string: -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :start-after: [START howto_operator_gcp_natural_language_document_text] :end-before: [END howto_operator_gcp_natural_language_document_text] In addition to supplying string, a document can refer to content stored in Google Cloud Storage. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :start-after: [START howto_operator_gcp_natural_language_document_gcs] :end-before: [END howto_operator_gcp_natural_language_document_gcs] @@ -66,7 +66,7 @@ public figures, landmarks, etc.), and returns information about those entities. Entity analysis is performed with the :class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageAnalyzeEntitiesOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_entities] @@ -77,7 +77,7 @@ You can use :ref:`Jinja templating ` with parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_entities_result] @@ -94,7 +94,7 @@ as positive, negative, or neutral. Sentiment analysis is performed through the :class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageAnalyzeEntitySentimentOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment] @@ -105,7 +105,7 @@ You can use :ref:`Jinja templating ` with parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment_result] @@ -123,7 +123,7 @@ through the :class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageAnalyzeSentimentOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_sentiment] @@ -134,7 +134,7 @@ You can use :ref:`Jinja templating ` with parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_sentiment_result] @@ -151,7 +151,7 @@ content in a document, use the :class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageClassifyTextOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_classify_text] @@ -162,7 +162,7 @@ You can use :ref:`Jinja templating ` with parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom `, which allows it to be used by other operators. -.. exampleinclude:: /../../tests/system/providers/google/cloud/natural_language/example_natural_language.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/natural_language/example_natural_language.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_natural_language_analyze_classify_text_result] diff --git a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst index 8fb497a14f01e..74091ccc1eec4 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst @@ -41,7 +41,7 @@ Creating a PubSub topic The PubSub topic is a named resource to which messages are sent by publishers. The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubCreateTopicOperator` operator creates a topic. -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_create_topic] :end-before: [END howto_operator_gcp_pubsub_create_topic] @@ -56,7 +56,7 @@ A ``Subscription`` is a named resource representing the stream of messages from to be delivered to the subscribing application. The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubCreateSubscriptionOperator` operator creates the subscription. -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_create_subscription] :end-before: [END howto_operator_gcp_pubsub_create_subscription] @@ -70,7 +70,7 @@ Publishing PubSub messages A ``Message`` is a combination of data and (optional) attributes that a publisher sends to a topic and is eventually delivered to subscribers. The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubPublishMessageOperator` operator would publish messages. -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_publish] :end-before: [END howto_operator_gcp_pubsub_publish] @@ -83,32 +83,32 @@ Pulling messages from a PubSub subscription The :class:`~airflow.providers.google.cloud.sensors.pubsub.PubSubPullSensor` sensor pulls messages from a PubSub subscription and pass them through XCom. -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_pull_message_with_sensor] :end-before: [END howto_operator_gcp_pubsub_pull_message_with_sensor] Also for this action you can use sensor in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub_deferrable.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_pubsub_pull_message_with_async_sensor] :end-before: [END howto_operator_gcp_pubsub_pull_message_with_async_sensor] -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_pull_message_with_operator] :end-before: [END howto_operator_gcp_pubsub_pull_message_with_operator] To pull messages from XCom use the :class:`~airflow.providers.standard.operators.bash.BashOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_pull_messages_result_cmd] :end-before: [END howto_operator_gcp_pubsub_pull_messages_result_cmd] -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_pull_messages_result] :end-before: [END howto_operator_gcp_pubsub_pull_messages_result] @@ -121,7 +121,7 @@ Deleting a PubSub subscription The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubDeleteSubscriptionOperator` operator deletes the subscription. -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_unsubscribe] :end-before: [END howto_operator_gcp_pubsub_unsubscribe] @@ -134,7 +134,7 @@ Deleting a PubSub topic The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubDeleteTopicOperator` operator deletes topic. -.. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/pubsub/example_pubsub.py :language: python :start-after: [START howto_operator_gcp_pubsub_delete_topic] :end-before: [END howto_operator_gcp_pubsub_delete_topic] diff --git a/docs/apache-airflow-providers-google/operators/cloud/spanner.rst b/docs/apache-airflow-providers-google/operators/cloud/spanner.rst index b505fbe187f6c..fbb79bbf68eb3 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/spanner.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/spanner.rst @@ -41,7 +41,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/spanner/example_spanner.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/spanner/example_spanner.py :language: python :dedent: 4 :start-after: [START howto_operator_spanner_deploy] @@ -50,7 +50,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/spanner.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/spanner.py :language: python :dedent: 4 :start-after: [START gcp_spanner_deploy_template_fields] @@ -80,7 +80,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/spanner/example_spanner.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/spanner/example_spanner.py :language: python :dedent: 4 :start-after: [START howto_operator_spanner_database_delete] @@ -89,7 +89,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/spanner.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/spanner.py :language: python :dedent: 4 :start-after: [START gcp_spanner_delete_template_fields] @@ -120,7 +120,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/spanner/example_spanner.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/spanner/example_spanner.py :language: python :dedent: 4 :start-after: [START howto_operator_spanner_database_deploy] @@ -129,7 +129,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/spanner.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/spanner.py :language: python :dedent: 4 :start-after: [START gcp_spanner_database_deploy_template_fields] @@ -164,13 +164,13 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/spanner/example_spanner.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/spanner/example_spanner.py :language: python :dedent: 4 :start-after: [START howto_operator_spanner_database_update] :end-before: [END howto_operator_spanner_database_update] -.. exampleinclude:: /../../tests/system/providers/google/cloud/spanner/example_spanner.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/spanner/example_spanner.py :language: python :dedent: 4 :start-after: [START howto_operator_spanner_database_update_idempotent] @@ -179,7 +179,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/spanner.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/spanner.py :language: python :dedent: 4 :start-after: [START gcp_spanner_database_update_template_fields] @@ -207,7 +207,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/spanner/example_spanner.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/spanner/example_spanner.py :language: python :dedent: 4 :start-after: [START howto_operator_spanner_query] @@ -216,7 +216,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/spanner.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/spanner.py :language: python :dedent: 4 :start-after: [START gcp_spanner_query_template_fields] @@ -246,7 +246,7 @@ Using the operator You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: -.. exampleinclude:: /../../tests/system/providers/google/cloud/spanner/example_spanner.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/spanner/example_spanner.py :language: python :dedent: 4 :start-after: [START howto_operator_spanner_delete] @@ -255,7 +255,7 @@ it will be retrieved from the Google Cloud connection used. Both variants are sh Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/spanner.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/spanner.py :language: python :dedent: 4 :start-after: [START gcp_spanner_delete_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/speech_to_text.rst b/docs/apache-airflow-providers-google/operators/cloud/speech_to_text.rst index 006c21657c098..3114a359de96b 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/speech_to_text.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/speech_to_text.rst @@ -42,14 +42,14 @@ google.cloud.speech_v1.types module for more information, see: https://googleapis.github.io/google-cloud-python/latest/speech/gapic/v1/api.html#google.cloud.speech_v1.SpeechClient.recognize -.. exampleinclude:: /../../tests/system/providers/google/cloud/speech_to_text/example_speech_to_text.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py :language: python :start-after: [START howto_operator_text_to_speech_api_arguments] :end-before: [END howto_operator_text_to_speech_api_arguments] filename is a simple string argument: -.. exampleinclude:: /../../tests/system/providers/google/cloud/speech_to_text/example_speech_to_text.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py :language: python :start-after: [START howto_operator_speech_to_text_api_arguments] :end-before: [END howto_operator_speech_to_text_api_arguments] @@ -57,7 +57,7 @@ filename is a simple string argument: Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/speech_to_text/example_speech_to_text.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py :language: python :dedent: 4 :start-after: [START howto_operator_speech_to_text_recognize] @@ -66,7 +66,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/speech_to_text.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/speech_to_text.py :language: python :dedent: 4 :start-after: [START gcp_speech_to_text_synthesize_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/stackdriver.rst b/docs/apache-airflow-providers-google/operators/cloud/stackdriver.rst index ecad902d4e525..04b732f6ce9c8 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/stackdriver.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/stackdriver.rst @@ -40,7 +40,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_list_alert_policy] @@ -60,7 +60,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_enable_alert_policy] @@ -80,7 +80,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_disable_alert_policy] @@ -101,7 +101,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_upsert_alert_policy] @@ -120,7 +120,7 @@ Using the operator The name of the alert to be deleted should be given in the format projects//alertPolicies/ -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_delete_alert_policy] @@ -140,7 +140,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_list_notification_channel] @@ -160,7 +160,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_enable_notification_channel] @@ -180,7 +180,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_disable_notification_channel] @@ -201,7 +201,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_disable_notification_channel] @@ -220,7 +220,7 @@ Using the operator You can use this operator with or without project id to fetch all the alert policies. If project id is missing it will be retrieved from Google Cloud connection used. -.. exampleinclude:: /../../tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/stackdriver/example_stackdriver.py :language: python :dedent: 4 :start-after: [START howto_operator_gcp_stackdriver_delete_notification_channel] diff --git a/docs/apache-airflow-providers-google/operators/cloud/tasks.rst b/docs/apache-airflow-providers-google/operators/cloud/tasks.rst index af9f03bc33b77..c67a0b77d9715 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/tasks.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/tasks.rst @@ -41,7 +41,7 @@ Create queue To create new Queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueCreateOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START create_queue] @@ -55,7 +55,7 @@ Delete queue To delete Queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueDeleteOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START delete_queue] @@ -70,7 +70,7 @@ Resume queue To resume Queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueResumeOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START resume_queue] @@ -84,7 +84,7 @@ Pause queue To pause Queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueuePauseOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START pause_queue] @@ -98,7 +98,7 @@ Purge queue To purge Queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueuePurgeOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START purge_queue] @@ -112,7 +112,7 @@ Get queue To get Queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueGetOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START get_queue] @@ -126,7 +126,7 @@ Update queue To update Queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueUpdateOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START update_queue] @@ -140,7 +140,7 @@ List queues To list all Queues use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueuesListOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_queue.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_queue.py :language: python :dedent: 4 :start-after: [START list_queue] @@ -158,7 +158,7 @@ Create task To create new Task in a particular queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskCreateOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_tasks.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_tasks.py :language: python :dedent: 4 :start-after: [START create_task] @@ -172,7 +172,7 @@ Get task To get the Tasks in a particular queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskGetOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_tasks.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_tasks.py :language: python :dedent: 4 :start-after: [START tasks_get] @@ -186,7 +186,7 @@ Run task To run the Task in a particular queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskRunOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_tasks.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_tasks.py :language: python :dedent: 4 :start-after: [START run_task] @@ -200,7 +200,7 @@ List tasks To list all Tasks in a particular queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTasksListOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_tasks.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_tasks.py :language: python :dedent: 4 :start-after: [START list_tasks] @@ -214,7 +214,7 @@ Delete task To delete the Task from particular queue use :class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskDeleteOperator` -.. exampleinclude:: /../../tests/system/providers/google/cloud/tasks/example_tasks.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/tasks/example_tasks.py :language: python :dedent: 4 :start-after: [START create_task] diff --git a/docs/apache-airflow-providers-google/operators/cloud/text_to_speech.rst b/docs/apache-airflow-providers-google/operators/cloud/text_to_speech.rst index d200e902b3f94..78bf9e5eaed6e 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/text_to_speech.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/text_to_speech.rst @@ -42,14 +42,14 @@ The ``input``, ``voice`` and ``audio_config`` arguments need to be dicts or obje for more information, see: https://googleapis.github.io/google-cloud-python/latest/texttospeech/gapic/v1/api.html#google.cloud.texttospeech_v1.TextToSpeechClient.synthesize_speech -.. exampleinclude:: /../../tests/system/providers/google/cloud/text_to_speech/example_text_to_speech.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py :language: python :start-after: [START howto_operator_text_to_speech_api_arguments] :end-before: [END howto_operator_text_to_speech_api_arguments] The ``filename`` argument is a simple string argument: -.. exampleinclude:: /../../tests/system/providers/google/cloud/text_to_speech/example_text_to_speech.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py :language: python :start-after: [START howto_operator_text_to_speech_gcp_filename] :end-before: [END howto_operator_text_to_speech_gcp_filename] @@ -57,7 +57,7 @@ The ``filename`` argument is a simple string argument: Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/text_to_speech/example_text_to_speech.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py :language: python :dedent: 4 :start-after: [START howto_operator_text_to_speech_synthesize] @@ -66,7 +66,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/text_to_speech.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/text_to_speech.py :language: python :dedent: 4 :start-after: [START gcp_text_to_speech_synthesize_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/translate.rst b/docs/apache-airflow-providers-google/operators/cloud/translate.rst index 518d7d28f72be..579236cb0883c 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/translate.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/translate.rst @@ -40,7 +40,7 @@ Using the operator Basic usage of the operator: -.. exampleinclude:: /../../tests/system/providers/google/cloud/translate/example_translate.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/translate/example_translate.py :language: python :dedent: 4 :start-after: [START howto_operator_translate_text] @@ -49,7 +49,7 @@ Basic usage of the operator: The result of translation is available as dictionary or array of dictionaries accessible via the usual XCom mechanisms of Airflow: -.. exampleinclude:: /../../tests/system/providers/google/cloud/translate/example_translate.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/translate/example_translate.py :language: python :dedent: 4 :start-after: [START howto_operator_translate_access] @@ -59,7 +59,7 @@ XCom mechanisms of Airflow: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/translate.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/translate.py :language: python :dedent: 4 :start-after: [START translate_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/translate_speech.rst b/docs/apache-airflow-providers-google/operators/cloud/translate_speech.rst index c02f45c252167..d1043d0e51cf3 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/translate_speech.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/translate_speech.rst @@ -43,7 +43,7 @@ for more information, see: https://googleapis.github.io/google-cloud-python/late Arguments for translation need to be specified. -.. exampleinclude:: /../../tests/system/providers/google/cloud/translate_speech/example_translate_speech.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/translate_speech/example_translate_speech.py :language: python :start-after: [START howto_operator_translate_speech_arguments] :end-before: [END howto_operator_translate_speech_arguments] @@ -52,7 +52,7 @@ Arguments for translation need to be specified. Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/translate_speech/example_translate_speech.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/translate_speech/example_translate_speech.py :language: python :dedent: 4 :start-after: [START howto_operator_translate_speech] @@ -61,7 +61,7 @@ Using the operator Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/translate_speech.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/translate_speech.py :language: python :dedent: 4 :start-after: [START translate_speech_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst b/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst index 8fb76cd80fdee..b6bc94620ff2d 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst @@ -33,7 +33,7 @@ To create a Google VertexAI dataset you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.CreateDatasetOperator`. The operator returns dataset id in :ref:`XCom ` under ``dataset_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_dataset_operator] @@ -42,7 +42,7 @@ The operator returns dataset id in :ref:`XCom ` under ``dataset_i After creating a dataset you can use it to import some data using :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.ImportDataOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_import_data_operator] @@ -51,7 +51,7 @@ After creating a dataset you can use it to import some data using To export dataset you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.ExportDataOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_export_data_operator] @@ -60,7 +60,7 @@ To export dataset you can use To delete dataset you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.DeleteDatasetOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_dataset_operator] @@ -69,7 +69,7 @@ To delete dataset you can use To get dataset you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.GetDatasetOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_get_dataset_operator] @@ -78,7 +78,7 @@ To get dataset you can use To get a dataset list you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.ListDatasetsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_dataset_operator] @@ -87,7 +87,7 @@ To get a dataset list you can use To update dataset you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.dataset.UpdateDatasetOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_update_dataset_operator] @@ -115,7 +115,7 @@ create image you can find by this link: https://cloud.google.com/vertex-ai/docs/ After that you should put link to the image in ``container_uri`` parameter. Also you can type executing command for container which will be created from this image in ``command`` parameter. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_container_training_job_operator] @@ -124,7 +124,7 @@ for container which will be created from this image in ``command`` parameter. The :class:`~airflow.providers.google.cloud.operators.vertex_ai.custom_job.CreateCustomContainerTrainingJobOperator` also provides the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_container_training_job_operator_deferrable] @@ -138,7 +138,7 @@ create you can find by this link: https://cloud.google.com/vertex-ai/docs/traini Next you should put link to the package in ``python_package_gcs_uri`` parameter, also ``python_module_name`` parameter should has the name of script which will run your training task. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_python_package_training_job_operator] @@ -147,7 +147,7 @@ parameter should has the name of script which will run your training task. The :class:`~airflow.providers.google.cloud.operators.vertex_ai.custom_job.CreateCustomPythonPackageTrainingJobOperator` also provides the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_python_package_training_job_operator_deferrable] @@ -158,7 +158,7 @@ How to run a Custom Training Job To create and run a Custom Training Job you should put the path to your local training script inside the ``script_path`` parameter. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_training_job_operator] @@ -166,7 +166,7 @@ To create and run a Custom Training Job you should put the path to your local tr The same operation can be performed in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_training_job_operator_deferrable] @@ -176,7 +176,7 @@ Additionally, you can create a new version of an existing Custom Training Job. I Model with another version, instead of creating a new Model in the Model Registry. This can be done by specifying the ``parent_model`` parameter when running a Custom Training Job. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_training_job_v2_operator] @@ -184,7 +184,7 @@ This can be done by specifying the ``parent_model`` parameter when running a Cus The same operation can be performed in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_custom_training_job_v2_deferrable_operator] @@ -194,7 +194,7 @@ The same operation can be performed in the deferrable mode: You can get a list of Training Jobs using :class:`~airflow.providers.google.cloud.operators.vertex_ai.custom_job.ListCustomTrainingJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_custom_training_job_operator] @@ -203,7 +203,7 @@ You can get a list of Training Jobs using If you wish to delete a Custom Training Job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.custom_job.DeleteCustomTrainingJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_custom_training_job_operator] @@ -227,7 +227,7 @@ How to run AutoML Forecasting Training Job Before start running this Job you must prepare and create ``TimeSeries`` dataset. After that you should put dataset id to ``dataset_id`` parameter in operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_auto_ml_forecasting_training_job_operator] @@ -239,7 +239,7 @@ How to run AutoML Image Training Job Before start running this Job you must prepare and create ``Image`` dataset. After that you should put dataset id to ``dataset_id`` parameter in operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_auto_ml_image_training_job_operator] @@ -251,7 +251,7 @@ How to run AutoML Tabular Training Job Before start running this Job you must prepare and create ``Tabular`` dataset. After that you should put dataset id to ``dataset_id`` parameter in operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_auto_ml_tabular_training_job_operator] @@ -274,7 +274,7 @@ How to run AutoML Video Training Job Before start running this Job you must prepare and create ``Video`` dataset. After that you should put dataset id to ``dataset_id`` parameter in operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_auto_ml_video_training_job_operator] @@ -284,7 +284,7 @@ Additionally, you can create new version of existing AutoML Video Training Job. version of existing Model instead of new Model created in Model Registry. This can be done by specifying ``parent_model`` parameter when running AutoML Video Training Job. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_auto_ml_video_training_job_v2_operator] @@ -293,7 +293,7 @@ version of existing Model instead of new Model created in Model Registry. This c You can get a list of AutoML Training Jobs using :class:`~airflow.providers.google.cloud.operators.vertex_ai.auto_ml.ListAutoMLTrainingJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_auto_ml_training_job_operator] @@ -302,7 +302,7 @@ You can get a list of AutoML Training Jobs using If you wish to delete a Auto ML Training Job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.auto_ml.DeleteAutoMLTrainingJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_auto_ml_training_job_operator] @@ -315,7 +315,7 @@ To create a Google VertexAI Batch Prediction Job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job.CreateBatchPredictionJobOperator`. The operator returns batch prediction job id in :ref:`XCom ` under ``batch_prediction_job_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_batch_prediction_job_operator] @@ -324,7 +324,7 @@ The operator returns batch prediction job id in :ref:`XCom ` unde The :class:`~airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job.CreateBatchPredictionJobOperator` also provides deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_batch_prediction_job_operator_def] @@ -334,7 +334,7 @@ also provides deferrable mode: To delete batch prediction job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job.DeleteBatchPredictionJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_batch_prediction_job_operator] @@ -343,7 +343,7 @@ To delete batch prediction job you can use To get a batch prediction job list you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job.ListBatchPredictionJobsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_batch_prediction_job_operator] @@ -356,7 +356,7 @@ To create a Google VertexAI endpoint you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.CreateEndpointOperator`. The operator returns endpoint id in :ref:`XCom ` under ``endpoint_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_endpoint_operator] @@ -365,7 +365,7 @@ The operator returns endpoint id in :ref:`XCom ` under ``endpoint After creating an endpoint you can use it to deploy some model using :class:`~airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.DeployModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_deploy_model_operator] @@ -374,7 +374,7 @@ After creating an endpoint you can use it to deploy some model using To un deploy model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.UndeployModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_undeploy_model_operator] @@ -383,7 +383,7 @@ To un deploy model you can use To delete endpoint you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.DeleteEndpointOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_endpoint_operator] @@ -392,7 +392,7 @@ To delete endpoint you can use To get an endpoint list you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.ListEndpointsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_endpoints_operator] @@ -405,7 +405,7 @@ To create a Google VertexAI hyperparameter tuning job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job.CreateHyperparameterTuningJobOperator`. The operator returns hyperparameter tuning job id in :ref:`XCom ` under ``hyperparameter_tuning_job_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_hyperparameter_tuning_job_operator] @@ -414,7 +414,7 @@ The operator returns hyperparameter tuning job id in :ref:`XCom ` :class:`~airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job.CreateHyperparameterTuningJobOperator` also supports deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_create_hyperparameter_tuning_job_operator_deferrable] @@ -423,7 +423,7 @@ also supports deferrable mode: To delete hyperparameter tuning job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job.DeleteHyperparameterTuningJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_hyperparameter_tuning_job_operator] @@ -432,7 +432,7 @@ To delete hyperparameter tuning job you can use To get hyperparameter tuning job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job.GetHyperparameterTuningJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_get_hyperparameter_tuning_job_operator] @@ -441,7 +441,7 @@ To get hyperparameter tuning job you can use To get a hyperparameter tuning job list you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job.ListHyperparameterTuningJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_hyperparameter_tuning_job_operator] @@ -454,7 +454,7 @@ To upload a Google VertexAI model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.UploadModelOperator`. The operator returns model id in :ref:`XCom ` under ``model_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_upload_model_operator] @@ -463,7 +463,7 @@ The operator returns model id in :ref:`XCom ` under ``model_id`` To export model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.ExportModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_export_model_operator] @@ -472,7 +472,7 @@ To export model you can use To delete model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_model_operator] @@ -481,7 +481,7 @@ To delete model you can use To get a model list you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.ListModelsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_models_operator] @@ -490,7 +490,7 @@ To get a model list you can use To retrieve model by its ID you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.GetModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_get_model_operator] @@ -499,7 +499,7 @@ To retrieve model by its ID you can use To list all model versions you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.ListModelVersionsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_model_versions_operator] @@ -508,7 +508,7 @@ To list all model versions you can use To set a specific version of model as a default one you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.SetDefaultVersionOnModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_set_version_as_default_operator] @@ -517,7 +517,7 @@ To set a specific version of model as a default one you can use To add aliases to specific version of model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.AddVersionAliasesOnModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_add_version_aliases_operator] @@ -526,7 +526,7 @@ To add aliases to specific version of model you can use To delete aliases from specific version of model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteVersionAliasesOnModelOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_version_aliases_operator] @@ -535,7 +535,7 @@ To delete aliases from specific version of model you can use To delete specific version of model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteModelVersionOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_version_operator] @@ -548,7 +548,7 @@ To run a Google VertexAI Pipeline Job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.pipeline_job.RunPipelineJobOperator`. The operator returns pipeline job id in :ref:`XCom ` under ``pipeline_job_id`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_run_pipeline_job_operator] @@ -557,7 +557,7 @@ The operator returns pipeline job id in :ref:`XCom ` under ``pipe To delete pipeline job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.pipeline_job.DeletePipelineJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_delete_pipeline_job_operator] @@ -566,7 +566,7 @@ To delete pipeline job you can use To get pipeline job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.pipeline_job.GetPipelineJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_get_pipeline_job_operator] @@ -575,7 +575,7 @@ To get pipeline job you can use To get a pipeline job list you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.pipeline_job.ListPipelineJobOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_list_pipeline_job_operator] @@ -588,7 +588,7 @@ To generate a prediction via language model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.TextGenerationModelPredictOperator`. The operator returns the model's response in :ref:`XCom ` under ``model_response`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_text_generation_model_predict_operator] @@ -598,7 +598,7 @@ To generate text embeddings you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.TextEmbeddingModelGetEmbeddingsOperator`. The operator returns the model's response in :ref:`XCom ` under ``model_response`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_text_embedding_model_get_embeddings_operator] @@ -608,7 +608,7 @@ To generate content with a generative model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.GenerativeModelGenerateContentOperator`. The operator returns the model's response in :ref:`XCom ` under ``model_response`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_generative_model_generate_content_operator] @@ -618,7 +618,7 @@ To run a supervised fine tuning job you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.SupervisedFineTuningTrainOperator`. The operator returns the tuned model's endpoint name in :ref:`XCom ` under ``tuned_model_endpoint_name`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_supervised_fine_tuning_train_operator] @@ -629,7 +629,7 @@ To calculates the number of input tokens before sending a request to the Gemini :class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.CountTokensOperator`. The operator returns the total tokens in :ref:`XCom ` under ``total_tokens`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_count_tokens_operator] @@ -639,7 +639,7 @@ To evaluate a model you can use :class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.RunEvaluationOperator`. The operator returns the evaluation summary metrics in :ref:`XCom ` under ``summary_metrics`` key. -.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py :language: python :dedent: 4 :start-after: [START how_to_cloud_vertex_ai_run_evaluation_operator] diff --git a/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst b/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst index 6b7bd2412e6ef..ea9ca48bd04ae 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst @@ -38,12 +38,12 @@ Using the operator Input uri is an uri to a file in Google Cloud Storage -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :start-after: [START howto_operator_video_intelligence_other_args] :end-before: [END howto_operator_video_intelligence_other_args] -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_labels] @@ -51,7 +51,7 @@ Input uri is an uri to a file in Google Cloud Storage You can use the annotation output via Xcom: -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_labels_result] @@ -60,7 +60,7 @@ You can use the annotation output via Xcom: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/video_intelligence.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/video_intelligence.py :language: python :dedent: 4 :start-after: [START gcp_video_intelligence_detect_labels_template_fields] @@ -87,7 +87,7 @@ Arguments Input uri is an uri to a file in Google Cloud Storage -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :start-after: [START howto_operator_video_intelligence_other_args] :end-before: [END howto_operator_video_intelligence_other_args] @@ -95,7 +95,7 @@ Input uri is an uri to a file in Google Cloud Storage Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_explicit_content] @@ -103,7 +103,7 @@ Using the operator You can use the annotation output via Xcom: -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_explicit_content_result] @@ -112,7 +112,7 @@ You can use the annotation output via Xcom: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/video_intelligence.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/video_intelligence.py :language: python :dedent: 4 :start-after: [START gcp_video_intelligence_detect_explicit_content_template_fields] @@ -139,7 +139,7 @@ Arguments Input uri is an uri to a file in Google Cloud Storage -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :start-after: [START howto_operator_video_intelligence_other_args] :end-before: [END howto_operator_video_intelligence_other_args] @@ -147,7 +147,7 @@ Input uri is an uri to a file in Google Cloud Storage Using the operator """""""""""""""""" -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_video_shots] @@ -155,7 +155,7 @@ Using the operator You can use the annotation output via Xcom: -.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_video_shots_result] @@ -164,7 +164,7 @@ You can use the annotation output via Xcom: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/video_intelligence.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/video_intelligence.py :language: python :dedent: 4 :start-after: [START gcp_video_intelligence_detect_video_shots_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/vision.rst b/docs/apache-airflow-providers-google/operators/cloud/vision.rst index 393f637dc3c55..897b4ead4f16d 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/vision.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/vision.rst @@ -42,17 +42,17 @@ We are using the :class:`~google.cloud.vision_v1.types.Product`, :class:`~google.cloud.vision_v1.types.ProductSet` and :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_set_import] :end-before: [END howto_operator_vision_product_set_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_import] :end-before: [END howto_operator_vision_product_import] @@ -60,7 +60,7 @@ Google libraries: If ``product_set_id`` and ``product_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_add_product_to_product_set] @@ -68,7 +68,7 @@ If ``product_set_id`` and ``product_id`` was generated by the API it can be extr Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_add_product_to_product_set_2] @@ -78,7 +78,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_add_product_to_product_set_template_fields] @@ -107,18 +107,18 @@ Using the operator We are using the :class:`~google.cloud.vision.enums` and :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :start-after: [START howto_operator_vision_enums_import] :end-before: [END howto_operator_vision_enums_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_annotate_image] @@ -126,7 +126,7 @@ Google libraries: The result can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_annotate_image_result] @@ -136,7 +136,7 @@ The result can be extracted from XCOM: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_annotate_image_template_fields] @@ -169,24 +169,24 @@ Using the operator We are using the ``Product`` and :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_import] :end-before: [END howto_operator_vision_product_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product] :end-before: [END howto_operator_vision_product] The ``product_id`` argument can be omitted (it will be generated by the API): -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_create] @@ -194,7 +194,7 @@ The ``product_id`` argument can be omitted (it will be generated by the API): Or it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_create_2] @@ -204,7 +204,7 @@ Or it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_product_create_template_fields] @@ -239,7 +239,7 @@ Using the operator If ``product_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_delete] @@ -247,7 +247,7 @@ If ``product_id`` was generated by the API it can be extracted from XCOM: Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_delete_2] @@ -256,7 +256,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_product_delete_template_fields] @@ -287,7 +287,7 @@ Using the operator If ``product_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_get] @@ -295,7 +295,7 @@ If ``product_id`` was generated by the API it can be extracted from XCOM: Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_get_2] @@ -304,7 +304,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_product_get_template_fields] @@ -331,24 +331,24 @@ Using the operator We are using the ``ProductSet`` and :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_set_import] :end-before: [END howto_operator_vision_product_set_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_set] :end-before: [END howto_operator_vision_product_set] The ``product_set_id`` argument can be omitted (it will be generated by the API): -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_create] @@ -356,7 +356,7 @@ The ``product_set_id`` argument can be omitted (it will be generated by the API) Or it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_create_2] @@ -366,7 +366,7 @@ Or it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_productset_create_template_fields] @@ -395,7 +395,7 @@ Using the operator If ``product_set_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_delete] @@ -403,7 +403,7 @@ If ``product_set_id`` was generated by the API it can be extracted from XCOM: Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_delete_2] @@ -412,7 +412,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_productset_delete_template_fields] @@ -439,7 +439,7 @@ Using the operator If ``product_set_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_get] @@ -447,7 +447,7 @@ If ``product_set_id`` was generated by the API it can be extracted from XCOM: Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_get_2] @@ -456,7 +456,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_productset_get_template_fields] @@ -495,12 +495,12 @@ Using the operator We are using the ``ProductSet`` object from the Google Cloud Vision library: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_set_import] :end-before: [END howto_operator_vision_product_set_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_set] :end-before: [END howto_operator_vision_product_set] @@ -509,7 +509,7 @@ Initialization of the task: If ``product_set_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_update] @@ -517,7 +517,7 @@ If ``product_set_id`` was generated by the API it can be extracted from XCOM: Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_set_update_2] @@ -526,7 +526,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_productset_update_template_fields] @@ -576,19 +576,19 @@ Using the operator We are using the ``Product`` object from the Google Cloud Vision library: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_import] :end-before: [END howto_operator_vision_product_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product] :end-before: [END howto_operator_vision_product] If ``product_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_update] @@ -596,7 +596,7 @@ If ``product_id`` was generated by the API it can be extracted from XCOM: Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_product_update_2] @@ -605,7 +605,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_product_update_template_fields] @@ -632,24 +632,24 @@ Using the operator We are using the :class:`~google.cloud.vision_v1.types.ReferenceImage` and :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_reference_image_import] :end-before: [END howto_operator_vision_reference_image_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_reference_image] :end-before: [END howto_operator_vision_reference_image] The ``product_set_id`` argument can be omitted (it will be generated by the API): -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_reference_image_create] @@ -657,7 +657,7 @@ The ``product_set_id`` argument can be omitted (it will be generated by the API) Or it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_reference_image_create_2] @@ -667,7 +667,7 @@ Or it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_reference_image_create_template_fields] @@ -694,24 +694,24 @@ Using the operator We are using the :class:`~google.cloud.vision_v1.types.ReferenceImage` and :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_reference_image_import] :end-before: [END howto_operator_vision_reference_image_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_reference_image] :end-before: [END howto_operator_vision_reference_image] The ``product_set_id`` argument can be omitted (it will be generated by the API): -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_reference_image_delete] @@ -719,7 +719,7 @@ The ``product_set_id`` argument can be omitted (it will be generated by the API) Or it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_reference_image_delete_2] @@ -729,7 +729,7 @@ Or it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_reference_image_create_template_fields] @@ -758,17 +758,17 @@ We are using the :class:`~google.cloud.vision_v1.types.Product`, :class:`~google.cloud.vision_v1.types.ProductSet` and :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_set_import] :end-before: [END howto_operator_vision_product_set_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :start-after: [START howto_operator_vision_product_import] :end-before: [END howto_operator_vision_product_import] @@ -776,7 +776,7 @@ Google libraries: If ``product_set_id`` and ``product_id`` was generated by the API it can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_autogenerated.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_remove_product_from_product_set] @@ -784,7 +784,7 @@ If ``product_set_id`` and ``product_id`` was generated by the API it can be extr Otherwise it can be specified explicitly: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_explicit.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_explicit.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_remove_product_from_product_set_2] @@ -794,7 +794,7 @@ Otherwise it can be specified explicitly: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_remove_product_from_product_set_template_fields] @@ -824,12 +824,12 @@ Using the operator We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_detect_text] @@ -837,7 +837,7 @@ Google libraries: The result can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_detect_text_result] @@ -847,7 +847,7 @@ The result can be extracted from XCOM: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_detect_text_set_template_fields] @@ -876,12 +876,12 @@ Using the operator We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_document_detect_text] @@ -889,7 +889,7 @@ Google libraries: The result can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_document_detect_text_result] @@ -899,7 +899,7 @@ The result can be extracted from XCOM: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_document_detect_text_set_template_fields] @@ -929,12 +929,12 @@ Using the operator We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_detect_labels] @@ -942,7 +942,7 @@ Google libraries: The result can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_detect_labels_result] @@ -952,7 +952,7 @@ The result can be extracted from XCOM: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_detect_labels_template_fields] @@ -981,12 +981,12 @@ Using the operator We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :start-after: [START howto_operator_vision_retry_import] :end-before: [END howto_operator_vision_retry_import] -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_detect_safe_search] @@ -994,7 +994,7 @@ Google libraries: The result can be extracted from XCOM: -.. exampleinclude:: /../../tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/vision/example_vision_annotate_image.py :language: python :dedent: 4 :start-after: [START howto_operator_vision_detect_safe_search_result] @@ -1004,7 +1004,7 @@ The result can be extracted from XCOM: Templating """""""""" -.. literalinclude:: /../../airflow/providers/google/cloud/operators/vision.py +.. literalinclude:: /../../providers/src/airflow/providers/google/cloud/operators/vision.py :language: python :dedent: 4 :start-after: [START vision_detect_safe_search_template_fields] diff --git a/docs/apache-airflow-providers-google/operators/cloud/workflows.rst b/docs/apache-airflow-providers-google/operators/cloud/workflows.rst index 79c7a2072419b..aa391cb0189e0 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/workflows.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/workflows.rst @@ -39,7 +39,7 @@ Create workflow To create a workflow use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsCreateWorkflowOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_create_workflow] @@ -47,7 +47,7 @@ To create a workflow use The workflow should be define in similar why to this example: -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 0 :start-after: [START how_to_define_workflow] @@ -65,7 +65,7 @@ Update workflow To update a workflow use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsUpdateWorkflowOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_update_workflow] @@ -79,7 +79,7 @@ Get workflow To get a workflow use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsGetWorkflowOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_get_workflow] @@ -93,7 +93,7 @@ List workflows To list workflows use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsListWorkflowsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_list_workflows] @@ -107,7 +107,7 @@ Delete workflow To delete a workflow use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsDeleteWorkflowOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_delete_workflow] @@ -122,7 +122,7 @@ To create an execution use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsCreateExecutionOperator`. This operator is not idempotent due to API limitation. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_create_execution] @@ -131,7 +131,7 @@ This operator is not idempotent due to API limitation. The create operator does not wait for execution to complete. To wait for execution result use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowExecutionSensor`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_wait_for_execution] @@ -145,7 +145,7 @@ Get execution To get an execution use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsGetExecutionOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_get_execution] @@ -160,7 +160,7 @@ To list executions use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsListExecutionsOperator`. By default this operator will return only executions for last 60 minutes. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_list_executions] @@ -174,7 +174,7 @@ Cancel execution To cancel an execution use :class:`~airflow.providers.google.cloud.operators.dataproc.WorkflowsCancelExecutionOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/workflows/example_workflows.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/workflows/example_workflows.py :language: python :dedent: 4 :start-after: [START how_to_cancel_execution] diff --git a/docs/apache-airflow-providers-google/operators/firebase/firestore.rst b/docs/apache-airflow-providers-google/operators/firebase/firestore.rst index ba7469d9744d0..eb274491d2fd2 100644 --- a/docs/apache-airflow-providers-google/operators/firebase/firestore.rst +++ b/docs/apache-airflow-providers-google/operators/firebase/firestore.rst @@ -41,7 +41,7 @@ Export database Exports a copy of all or a subset of documents from Google Cloud Firestore to Google Cloud Storage is performed with the :class:`~airflow.providers.google.firebase.operators.firestore.CloudFirestoreExportDatabaseOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_firestore.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_firestore.py :language: python :dedent: 4 :start-after: [START howto_operator_export_database_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst b/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst index 2a4f8b41b3167..9aeb5fcae80bb 100644 --- a/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst +++ b/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst @@ -36,7 +36,7 @@ Put key Get, put, delete key or write_batch, create database with comparator or different options in LevelDB is performed with the :class:`~airflow.providers.google.leveldb.operators.leveldb.LevelDBOperator` operator. -.. exampleinclude:: /../../tests/system/providers/google/leveldb/example_leveldb.py +.. exampleinclude:: /../../providers/tests/system/google/leveldb/example_leveldb.py :language: python :dedent: 4 :start-after: [START howto_operator_leveldb_put_key] diff --git a/docs/apache-airflow-providers-google/operators/marketing_platform/analytics_admin.rst b/docs/apache-airflow-providers-google/operators/marketing_platform/analytics_admin.rst index b4aab19897659..fd1fc199b497b 100644 --- a/docs/apache-airflow-providers-google/operators/marketing_platform/analytics_admin.rst +++ b/docs/apache-airflow-providers-google/operators/marketing_platform/analytics_admin.rst @@ -35,7 +35,7 @@ List the Accounts To list accounts from Analytics you can use the :class:`~airflow.providers.google.marketing_platform.operators.analytics_admin.GoogleAnalyticsAdminListAccountsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics_admin.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_analytics_admin.py :language: python :dedent: 4 :start-after: [START howto_marketing_platform_list_accounts_operator] @@ -53,7 +53,7 @@ Creates a property. To create a property you can use the :class:`~airflow.providers.google.marketing_platform.operators.analytics_admin.GoogleAnalyticsAdminCreatePropertyOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics_admin.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_analytics_admin.py :language: python :dedent: 4 :start-after: [START howto_marketing_platform_create_property_operator] @@ -71,7 +71,7 @@ Deletes a property. To delete a property you can use the :class:`~airflow.providers.google.marketing_platform.operators.analytics_admin.GoogleAnalyticsAdminDeletePropertyOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics_admin.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_analytics_admin.py :language: python :dedent: 4 :start-after: [START howto_marketing_platform_delete_property_operator] @@ -89,7 +89,7 @@ Creates a data stream. To create a data stream you can use the :class:`~airflow.providers.google.marketing_platform.operators.analytics_admin.GoogleAnalyticsAdminCreateDataStreamOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics_admin.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_analytics_admin.py :language: python :dedent: 4 :start-after: [START howto_marketing_platform_create_data_stream_operator] @@ -107,7 +107,7 @@ Deletes a data stream. To delete a data stream you can use the :class:`~airflow.providers.google.marketing_platform.operators.analytics_admin.GoogleAnalyticsAdminDeleteDataStreamOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics_admin.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_analytics_admin.py :language: python :dedent: 4 :start-after: [START howto_marketing_platform_delete_data_stream_operator] @@ -124,7 +124,7 @@ List Google Ads Links To list Google Ads links you can use the :class:`~airflow.providers.google.marketing_platform.operators.analytics_admin.GoogleAnalyticsAdminListGoogleAdsLinksOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics_admin.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_analytics_admin.py :language: python :dedent: 4 :start-after: [START howto_marketing_platform_list_google_ads_links] @@ -141,7 +141,7 @@ Get the Google Ads link To list Google Ads links you can use the :class:`~airflow.providers.google.marketing_platform.operators.analytics_admin.GoogleAnalyticsAdminGetGoogleAdsLinkOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics_admin.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_analytics_admin.py :language: python :dedent: 4 :start-after: [START howto_marketing_platform_get_google_ad_link] diff --git a/docs/apache-airflow-providers-google/operators/marketing_platform/campaign_manager.rst b/docs/apache-airflow-providers-google/operators/marketing_platform/campaign_manager.rst index eb3b6b03549e5..15b08b72a57b4 100644 --- a/docs/apache-airflow-providers-google/operators/marketing_platform/campaign_manager.rst +++ b/docs/apache-airflow-providers-google/operators/marketing_platform/campaign_manager.rst @@ -36,7 +36,7 @@ To delete Campaign Manager report you can use the :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerDeleteReportOperator`. It deletes a report by its unique ID. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_campaign_manager.py :language: python :dedent: 4 :start-after: [START howto_campaign_manager_delete_report_operator] @@ -54,7 +54,7 @@ Downloading a report The :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerDownloadReportOperator`. allows you to download a Campaign Manager to Google Cloud Storage bucket. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_campaign_manager.py :language: python :dedent: 4 :start-after: [START howto_campaign_manager_get_report_operator] @@ -72,7 +72,7 @@ Waiting for a report Report are generated asynchronously. To wait for report to be ready for downloading you can use :class:`~airflow.providers.google.marketing_platform.sensors.campaign_manager.GoogleCampaignManagerReportSensor`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_campaign_manager.py :language: python :dedent: 4 :start-after: [START howto_campaign_manager_wait_for_operation] @@ -91,7 +91,7 @@ To insert a Campaign Manager report you can use the :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerInsertReportOperator`. Running this operator creates a new report. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_campaign_manager.py :language: python :dedent: 4 :start-after: [START howto_campaign_manager_insert_report_operator] @@ -111,7 +111,7 @@ Running a report To run Campaign Manager report you can use the :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerRunReportOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_campaign_manager.py :language: python :dedent: 4 :start-after: [START howto_campaign_manager_run_report_operator] @@ -130,7 +130,7 @@ Inserting conversions To insert Campaign Manager conversions you can use the :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerBatchInsertConversionsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_campaign_manager.py :language: python :dedent: 4 :start-after: [START howto_campaign_manager_insert_conversions] @@ -149,7 +149,7 @@ Updating conversions To update Campaign Manager conversions you can use the :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerBatchUpdateConversionsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_campaign_manager.py :language: python :dedent: 4 :start-after: [START howto_campaign_manager_update_conversions] diff --git a/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst b/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst index 1669b26cc1ced..4b1f4b81cac8e 100644 --- a/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst +++ b/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst @@ -33,7 +33,7 @@ Creating a Query To create Display&Video 360 query use :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateQueryOperator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_create_query_operator] @@ -52,7 +52,7 @@ Run Query :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360RunQueryOperator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_run_query_report_operator] @@ -71,7 +71,7 @@ Deleting a report To delete Display&Video 360 report use :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DeleteReportOperator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_delete_query_report_operator] @@ -89,7 +89,7 @@ Waiting for query To wait for the report use :class:`~airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360RunQuerySensor`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_wait_run_query_sensor] @@ -107,7 +107,7 @@ Downloading a report To download a report to GCS bucket use :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadReportV2Operator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_get_report_operator] @@ -135,7 +135,7 @@ The operator accepts body request: To download line items in CSV format report use :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadLineItemsOperator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_download_line_items_operator] @@ -154,7 +154,7 @@ Upload line items To run Display&Video 360 uploading line items use :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360UploadLineItemsOperator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_upload_line_items_operator] @@ -172,7 +172,7 @@ Create SDF download task To create SDF download task use :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateSDFDownloadTaskOperator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_create_sdf_download_task_operator] @@ -191,7 +191,7 @@ Save SDF files in the Google Cloud Storage To save SDF files and save them in the Google Cloud Storage use :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360SDFtoGCSOperator`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_save_sdf_in_gcs_operator] @@ -209,7 +209,7 @@ Waiting for SDF operation Wait for SDF operation is executed by: :class:`~airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360GetSDFDownloadOperationSensor`. -.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py :language: python :dedent: 4 :start-after: [START howto_google_display_video_wait_for_operation_sensor] diff --git a/docs/apache-airflow-providers-google/operators/marketing_platform/search_ads.rst b/docs/apache-airflow-providers-google/operators/marketing_platform/search_ads.rst index 928d60f671a54..00fb9d5ab7074 100644 --- a/docs/apache-airflow-providers-google/operators/marketing_platform/search_ads.rst +++ b/docs/apache-airflow-providers-google/operators/marketing_platform/search_ads.rst @@ -34,7 +34,7 @@ Querying a report To query a Search Ads report use the :class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsSearchOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_search_ads.py :language: python :dedent: 4 :start-after: [START howto_search_ads_search_query_reports] @@ -52,7 +52,7 @@ Retrieve a field metadata To retrieve metadata of a field use :class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsGetFieldOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_search_ads.py :language: python :dedent: 4 :start-after: [START howto_search_ads_get_field] @@ -70,7 +70,7 @@ Retrieve metadata for multiple fields To retrieve metadata of multiple fields use the :class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsSearchFieldsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_search_ads.py :language: python :dedent: 4 :start-after: [START howto_search_ads_search_fields] @@ -89,7 +89,7 @@ Retrieve a custom column details To retrieve details of a custom column use :class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsGetCustomColumnOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_search_ads.py :language: python :dedent: 4 :start-after: [START howto_search_ads_get_custom_column] @@ -108,7 +108,7 @@ To retrieve the list of all custom columns use :class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsListCustomColumnsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py +.. exampleinclude:: /../../providers/tests/system/google/marketing_platform/example_search_ads.py :language: python :dedent: 4 :start-after: [START howto_search_ads_list_custom_columns] diff --git a/docs/apache-airflow-providers-google/operators/suite/sheets.rst b/docs/apache-airflow-providers-google/operators/suite/sheets.rst index 0f64c2f0b1dd3..0f891a60a3cfe 100644 --- a/docs/apache-airflow-providers-google/operators/suite/sheets.rst +++ b/docs/apache-airflow-providers-google/operators/suite/sheets.rst @@ -44,7 +44,7 @@ Create spreadsheet To create new spreadsheet you can use the :class:`~airflow.providers.google.suite.operators.sheets.GoogleSheetsCreateSpreadsheetOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sheets.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_sheets.py :language: python :dedent: 4 :start-after: [START create_spreadsheet] @@ -55,7 +55,7 @@ You can use :ref:`Jinja templating ` with To get the URL of newly created spreadsheet use XCom value: -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sheets.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_sheets.py :language: python :dedent: 4 :start-after: [START print_spreadsheet_url] diff --git a/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst index 1a7fb4c3062b8..8bd909a809c27 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst @@ -38,7 +38,7 @@ to transfer data from Azure Blob Storage to Google Cloud Storage. Example usage: -.. exampleinclude:: /../../tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py :language: python :start-after: [START how_to_azure_blob_to_gcs] :end-before: [END how_to_azure_blob_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/azure_fileshare_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/azure_fileshare_to_gcs.rst index e962e2b88520b..6efc0d091cc48 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/azure_fileshare_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/azure_fileshare_to_gcs.rst @@ -36,7 +36,7 @@ All parameters are described in the reference documentation - :class:`~airflow.p An example operator call might look like this: -.. exampleinclude:: /../../tests/system/providers/google/cloud/azure/example_azure_fileshare_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_azure_fileshare_to_gcs_basic] diff --git a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_bigquery.rst b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_bigquery.rst index 23fe393994ebc..f9ce21df0dd00 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_bigquery.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_bigquery.rst @@ -50,7 +50,7 @@ Copying BigQuery tables The following Operator copies data from one or more BigQuery tables to another. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_to_bigquery.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_to_bigquery] diff --git a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_gcs.rst index b0eeb0ac5e346..2e2f3ad41a28d 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_gcs.rst @@ -53,7 +53,7 @@ Exporting tables The following Operator exports BigQuery table into a GCS. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mssql.rst b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mssql.rst index 5fb56ae8701db..382f6fa645886 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mssql.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mssql.rst @@ -53,7 +53,7 @@ Transferring data The following Operator copies data from a BigQuery table to MsSQL. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_to_mssql] diff --git a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mysql.rst b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mysql.rst index 263db92d053ad..0790cc65096f2 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mysql.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_mysql.rst @@ -51,7 +51,7 @@ Transferring data The following Operator copies data from a BigQuery table to MySQL. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_to_mysql.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_to_mysql] diff --git a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_postgres.rst b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_postgres.rst index 70f0c8c56d05a..d600017a05fb5 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_postgres.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/bigquery_to_postgres.rst @@ -55,7 +55,7 @@ Transferring data The following Operator copies data from a BigQuery table to PostgreSQL. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_to_postgres] @@ -63,7 +63,7 @@ The following Operator copies data from a BigQuery table to PostgreSQL. The Operator can also replace data in a PostgreSQL table with matching data from a BigQuery table. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py :language: python :dedent: 4 :start-after: [START howto_operator_bigquery_to_postgres_upsert] diff --git a/docs/apache-airflow-providers-google/operators/transfer/calendar_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/calendar_to_gcs.rst index 7b420f1e6a046..37569d6c895bc 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/calendar_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/calendar_to_gcs.rst @@ -37,7 +37,7 @@ Upload data from Google Calendar to GCS To upload data from Google Calendar to Google Cloud Storage you can use the :class:`~airflow.providers.google.cloud.transfers.calendar_to_gcs.GoogleCalendarToGCSOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_calendar_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py :language: python :dedent: 4 :start-after: [START upload_calendar_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/facebook_ads_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/facebook_ads_to_gcs.rst index 7132417cbb565..003239d676f0a 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/facebook_ads_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/facebook_ads_to_gcs.rst @@ -34,7 +34,7 @@ Use the :class:`~airflow.providers.google.cloud.transfers.facebook_ads_to_gcs.FacebookAdsReportToGcsOperator` to execute a Facebook ads report fetch and load to GCS. -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py :language: python :start-after: [START howto_operator_facebook_ads_to_gcs] :end-before: [END howto_operator_facebook_ads_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_bigquery.rst b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_bigquery.rst index a10b3d046c36d..a05ed77484a36 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_bigquery.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_bigquery.rst @@ -53,7 +53,7 @@ Transferring files The following Operator transfers one or more files from GCS into a BigQuery table. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_bigquery] @@ -61,7 +61,7 @@ The following Operator transfers one or more files from GCS into a BigQuery tabl Also you can use GCSToBigQueryOperator in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery_async.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_bigquery_async] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst index 6e49dc4229e26..3bb97af557d48 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst @@ -105,7 +105,7 @@ Note that if the flag ``exact_match=False`` then the ``source_object`` will be c in the ``BUCKET_1_SRC`` GCS bucket. That's why if any will be found, they will be copied as well. To prevent this from happening, please use ``exact_match=False``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gcs_single_file] @@ -121,7 +121,7 @@ are both deprecated. Thus, it is not recommended to use them - but to utilize `` The following example would copy the files that matches the glob pattern in ``data/`` folder from ``BUCKET_1_SRC`` GCS bucket to the ``backup/`` folder in ``BUCKET_1_DST`` bucket. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gcs_match_glob] @@ -130,7 +130,7 @@ The following example would copy the files that matches the glob pattern in ``da The following example would copy all the files in ``subdir/`` folder (i.e subdir/a.csv, subdir/b.csv, subdir/c.csv) from the ``BUCKET_1_SRC`` GCS bucket to the ``backup/`` folder in ``BUCKET_1_DST`` bucket. (i.e backup/a.csv, backup/b.csv, backup/c.csv) -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gcs_without_wildcard] @@ -138,7 +138,7 @@ the ``BUCKET_1_SRC`` GCS bucket to the ``backup/`` folder in ``BUCKET_1_DST`` bu -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gcs_list] @@ -156,7 +156,7 @@ Note that if the flag ``exact_match=False`` then the ``source_object`` will be c in the ``BUCKET_1_SRC`` GCS bucket. That's why if any will be found, they will be copied as well. To prevent this from happening, please use ``exact_match=False``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gcs_single_file_move] @@ -168,7 +168,7 @@ Move multiple files Multiple files may be moved by supplying ``True`` to the ``move`` argument. The same rules concerning wild cards and the ``delimiter`` argument apply to moves as well as copies. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gcs_list_move] @@ -201,7 +201,7 @@ The following example will ensure all files in ``BUCKET_1_SRC``, including any i ``BUCKET_1_DST``. It will not overwrite identically named files in ``BUCKET_1_DST`` if they already exist. It will not delete any files in ``BUCKET_1_DST`` not in ``BUCKET_1_SRC``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_synch_bucket] @@ -214,7 +214,7 @@ This example will ensure all files in ``BUCKET_1_SRC``, including any in subdire ``BUCKET_1_DST``. It will overwrite identically named files in ``BUCKET_1_DST`` if they already exist. It will delete any files in ``BUCKET_1_DST`` not in ``BUCKET_1_SRC``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_synch_full_bucket] @@ -227,7 +227,7 @@ The following example will ensure all files in ``BUCKET_1_SRC``, including any i ``subdir`` folder in ``BUCKET_1_DST``. It will not overwrite identically named files in ``BUCKET_1_DST/subdir`` if they already exist and it will not delete any files in ``BUCKET_1_DST/subdir`` not in ``BUCKET_1_SRC``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_synch_to_subdir] @@ -240,7 +240,7 @@ This example will ensure all files in ``BUCKET_1_SRC/subdir``, including any in in ``BUCKET_1_DST``. It will not overwrite identically named files in ``BUCKET_1_DST`` if they already exist and it will not delete any files in ``BUCKET_1_DST`` not in ``BUCKET_1_SRC/subdir``. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_sync_from_subdir] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gdrive.rst b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gdrive.rst index 8a8edd519af52..6b7467f96d1a7 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gdrive.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gdrive.rst @@ -47,7 +47,7 @@ Copy single files The following Operator would copy a single file. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gdrive.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gdrive_copy_single_file] @@ -58,7 +58,7 @@ Copy into an existing folder The following Operator would copy a single file into an existing folder with the specified ID. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gdrive.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gdrive_copy_single_file_into_folder] @@ -69,7 +69,7 @@ Copy multiple files The following Operator would copy all the multiples files (i.e. using wildcard). -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gdrive.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gdrive_copy_files] @@ -81,7 +81,7 @@ Move files Using the ``move_object`` parameter allows you to move the files. After copying the file to Google Drive, the original file from the bucket is deleted. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gdrive.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_gdrive_move_files] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_local.rst b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_local.rst index 9dfebb1608f7b..87a29e54c4984 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_local.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_local.rst @@ -37,7 +37,7 @@ data from GCS to local filesystem. Below is an example of using this operator to download a file from GCS. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_upload_download.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py :language: python :dedent: 0 :start-after: [START howto_operator_gcs_download_file_task] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sftp.rst b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sftp.rst index a98f537e2bc1d..f1f9ea731e881 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sftp.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sftp.rst @@ -48,7 +48,7 @@ Copying a single file The following Operator copies a single file. -.. exampleinclude:: /../../tests/system/providers/google/cloud/transfers/example_gcs_to_sftp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_sftp_copy_single_file] @@ -61,7 +61,7 @@ To move the file use the ``move_object`` parameter. Once the file is copied to S the original file from the Google Storage is deleted. The ``destination_path`` parameter defines the full path of the file on the SFTP server. -.. exampleinclude:: /../../tests/system/providers/google/cloud/transfers/example_gcs_to_sftp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_sftp_move_single_file_destination] @@ -73,7 +73,7 @@ Copying a directory Use the ``wildcard`` in ``source_path`` parameter to copy a directory. -.. exampleinclude:: /../../tests/system/providers/google/cloud/transfers/example_gcs_to_sftp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_sftp_copy_directory] @@ -85,7 +85,7 @@ Moving specific files Use the ``wildcard`` in ``source_path`` parameter to move the specific files. The ``destination_path`` defines the path that is prefixed to all copied files. -.. exampleinclude:: /../../tests/system/providers/google/cloud/transfers/example_gcs_to_sftp.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_sftp_move_specific_files] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sheets.rst b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sheets.rst index d8fdcfff3e6bc..bd2e47fd6b26e 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sheets.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_sheets.rst @@ -38,7 +38,7 @@ Upload data from GCS to Google Sheets To upload data from Google Cloud Storage to Google Spreadsheet you can use the :class:`~airflow.providers.google.suite.transfers.gcs_to_sheets.GCSToGoogleSheetsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_sheets.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py :language: python :dedent: 4 :start-after: [START upload_gcs_to_sheets] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_gcs.rst index c5f3a11808c87..260ec631011fd 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_gcs.rst @@ -47,7 +47,7 @@ The following Operator copies a single file from a shared Google Drive folder to Note that you can transfer a file from the root folder of a shared drive by passing the id of the shared drive to both the ``folder_id`` and ``drive_id`` parameters. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gdrive_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py :language: python :dedent: 4 :start-after: [START upload_gdrive_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_local.rst b/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_local.rst index 9cc69760a1ef2..57b589199d2cc 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_local.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gdrive_to_local.rst @@ -38,7 +38,7 @@ data from Google Drive to local filesystem. Below is an example of using this operator to download file from Google Drive to Local Filesystem. -.. exampleinclude:: /../../tests/system/providers/google/cloud/transfers/example_gdrive_to_local.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py :language: python :dedent: 4 :start-after: [START download_from_gdrive_to_local] diff --git a/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst b/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst index da2fefcacfe46..117a65dd1cc20 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst @@ -38,7 +38,7 @@ When you use this operator, you can upload a list of files. Below is an example of using this operator to upload data from local filesystem to Google Drive. -.. exampleinclude:: /../../tests/system/providers/google/suite/example_local_to_drive.py +.. exampleinclude:: /../../providers/tests/system/google/suite/example_local_to_drive.py :language: python :dedent: 0 :start-after: [START howto_operator_local_to_drive_upload_single_file] diff --git a/docs/apache-airflow-providers-google/operators/transfer/local_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/local_to_gcs.rst index e12726712beb1..c7b215901c5ce 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/local_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/local_to_gcs.rst @@ -38,7 +38,7 @@ When you use this operator, you can optionally compress the data being uploaded. Below is an example of using this operator to upload a file to GCS. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_upload_download.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py :language: python :dedent: 0 :start-after: [START howto_operator_local_filesystem_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst index 54ceb15ac1ea4..094973306a3ef 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst @@ -36,7 +36,7 @@ data from Microsoft SQL Server database to GCS. Below is an example of using this operator to upload data to GCS. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_mssql_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py :language: python :start-after: [START howto_operator_mssql_to_gcs] :end-before: [END howto_operator_mssql_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/mysql_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/mysql_to_gcs.rst index 14bca637c9c5d..391f9ae837222 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/mysql_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/mysql_to_gcs.rst @@ -38,7 +38,7 @@ When you use this operator, you can optionally compress the data being uploaded Below is an example of using this operator to upload data to GCS. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py :language: python :dedent: 0 :start-after: [START howto_operator_mysql_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/oracle_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/oracle_to_gcs.rst index 9760e594beef2..538cbce38d65c 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/oracle_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/oracle_to_gcs.rst @@ -38,7 +38,7 @@ When you use this operator, you can optionally compress the data being uploaded Below is an example of using this operator to upload data to GCS. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_oracle_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py :language: python :dedent: 0 :start-after: [START howto_operator_oracle_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst index b6b01ce864a80..2a5c3fac6dda5 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst @@ -38,7 +38,7 @@ When you use this operator, you can optionally compress the data being uploaded Below is an example of using this operator to upload data to GCS. -.. exampleinclude:: /../../tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py :language: python :dedent: 0 :start-after: [START howto_operator_postgres_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/presto_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/presto_to_gcs.rst index c462ee9abaa6c..538f2dec094d0 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/presto_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/presto_to_gcs.rst @@ -49,7 +49,7 @@ All parameters are described in the reference documentation - :class:`~airflow.p An example operator call might look like this: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_presto_to_gcs_basic] @@ -67,7 +67,7 @@ You can specify these options by the ``export_format`` parameter. If you want a CSV file to be created, your operator call might look like this: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_presto_to_gcs_csv] @@ -81,7 +81,7 @@ will be dumped from the database and upload to the bucket. If you want to create a schema file, then an example operator call might look like this: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_presto_to_gcs_multiple_types] @@ -102,7 +102,7 @@ maximum allowed file size for a single object. If you want to create 10 MB files, your code might look like this: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_read_data_from_gcs_many_chunks] @@ -123,7 +123,7 @@ For example, if you want to create an external table that allows you to create q read data directly from GCS, then you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateExternalTableOperator`. Using this operator looks like this: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_create_external_table_multiple_types] diff --git a/docs/apache-airflow-providers-google/operators/transfer/s3_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/s3_to_gcs.rst index dbd2f603bb9ed..9dff4cbb6ab9b 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/s3_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/s3_to_gcs.rst @@ -32,7 +32,7 @@ Prerequisite Tasks Use the :class:`~airflow.providers.google.cloud.transfers.s3_to_gcs.S3ToGCSOperator` to transfer data from Amazon S3 to Google Cloud Storage. -.. exampleinclude::/../tests/system/providers/google/cloud/gcs/example_s3_to_gcs.py +.. exampleinclude::/../providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py :language: python :start-after: [START howto_transfer_s3togcs_operator] :end-before: [END howto_transfer_s3togcs_operator] @@ -42,7 +42,7 @@ There is a possibility to start S3ToGCSOperator asynchronously using deferrable Transfer Service. By changing parameter ``poll_interval=10`` you can control frequency of polling a transfer job status. -.. exampleinclude::/../tests/system/providers/google/cloud/gcs/example_s3_to_gcs.py +.. exampleinclude::/../providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py :language: python :start-after: [START howto_transfer_s3togcs_operator_async] :end-before: [END howto_transfer_s3togcs_operator_async] diff --git a/docs/apache-airflow-providers-google/operators/transfer/salesforce_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/salesforce_to_gcs.rst index 62cb4c97c6df2..94f4180f91a64 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/salesforce_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/salesforce_to_gcs.rst @@ -32,7 +32,7 @@ Use the :class:`~airflow.providers.google.cloud.transfers.salesforce_to_gcs.SalesforceToGcsOperator` to execute a Salesforce query to fetch data and load it to GCS. -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py :language: python :start-after: [START howto_operator_salesforce_to_gcs] :end-before: [END howto_operator_salesforce_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/sftp_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/sftp_to_gcs.rst index 0fd8016b78f4d..cf4d237dd68d3 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/sftp_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/sftp_to_gcs.rst @@ -46,7 +46,7 @@ Copying single files The following Operator copies a single file. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_sftp_to_gcs_copy_single_file] @@ -59,7 +59,7 @@ To move the file use the ``move_object`` parameter. Once the file is copied to G the original file from the SFTP is deleted. The ``destination_path`` parameter defines the full path of the file in the bucket. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_sftp_to_gcs_move_single_file_destination] @@ -71,7 +71,7 @@ Copying directory Use the ``wildcard`` in ``source_path`` parameter to copy the directory. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_sftp_to_gcs_copy_directory] @@ -87,7 +87,7 @@ e.g. ``tests_sftp_hook_dir/subdir/parent-1.bin`` is copied to ``specific_files/p and ``tests_sftp_hook_dir/subdir/parent-2.bin`` is copied to ``specific_files/parent-2.bin`` . ``tests_sftp_hook_dir/subdir/parent-3.txt`` is skipped. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_sftp_to_gcs_move_specific_files] diff --git a/docs/apache-airflow-providers-google/operators/transfer/sheets_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/sheets_to_gcs.rst index 862028c544e02..0a3389b79b50c 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/sheets_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/sheets_to_gcs.rst @@ -38,7 +38,7 @@ Upload data from Google Sheets to GCS To upload data from Google Spreadsheet to Google Cloud Storage you can use the :class:`~airflow.providers.google.cloud.transfers.sheets_to_gcs.GoogleSheetsToGCSOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sheets_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py :language: python :dedent: 4 :start-after: [START upload_sheet_to_gcs] diff --git a/docs/apache-airflow-providers-google/operators/transfer/sql_to_sheets.rst b/docs/apache-airflow-providers-google/operators/transfer/sql_to_sheets.rst index 5ce4f19f01a8c..1758c0fc6a417 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/sql_to_sheets.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/sql_to_sheets.rst @@ -35,7 +35,7 @@ Upload data from SQL to Google Sheets To upload data from and Database using SQL to Google Spreadsheet you can use the :class:`~airflow.providers.google.suite.transfers.sql_to_sheets.SQLToGoogleSheetsOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py :language: python :dedent: 4 :start-after: [START upload_sql_to_sheets] diff --git a/docs/apache-airflow-providers-google/operators/transfer/trino_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/trino_to_gcs.rst index a1177eaad7a59..f881dab7f77d9 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/trino_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/trino_to_gcs.rst @@ -49,7 +49,7 @@ All parameters are described in the reference documentation - :class:`~airflow.p An example operator call might look like this: -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_trino_to_gcs_basic] @@ -67,7 +67,7 @@ You can specify these options by the ``export_format`` parameter. If you want a CSV file to be created, your operator call might look like this: -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_trino_to_gcs_csv] @@ -81,7 +81,7 @@ will be dumped from the database and upload to the bucket. If you want to create a schema file, then an example operator call might look like this: -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_trino_to_gcs_multiple_types] @@ -102,7 +102,7 @@ maximum allowed file size for a single object. If you want to create 10 MB files, your code might look like this: -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_read_data_from_gcs_many_chunks] @@ -123,7 +123,7 @@ For example, if you want to create an external table that allows you to create q read data directly from GCS, then you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateExternalTableOperator`. Using this operator looks like this: -.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py +.. exampleinclude:: /../../providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py :language: python :dedent: 4 :start-after: [START howto_operator_create_external_table_multiple_types] diff --git a/docs/apache-airflow-providers-google/sensors/google-cloud-tasks.rst b/docs/apache-airflow-providers-google/sensors/google-cloud-tasks.rst index 02dd8602e2024..3e015ba063481 100644 --- a/docs/apache-airflow-providers-google/sensors/google-cloud-tasks.rst +++ b/docs/apache-airflow-providers-google/sensors/google-cloud-tasks.rst @@ -32,7 +32,7 @@ Google Cloud Tasks Empty Sensor To sense Queue being empty use :class:`~airflow.providers.google.cloud.sensor.tasks.TaskQueueEmptySensor` -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_cloud_task.py +.. exampleinclude:: /../../providers/src/airflow/providers/google/cloud/example_dags/example_cloud_task.py :language: python :dedent: 4 :start-after: [START cloud_tasks_empty_sensor] diff --git a/docs/apache-airflow-providers-grpc/changelog.rst b/docs/apache-airflow-providers-grpc/changelog.rst index c5b39cf9bfc91..7c93191ed4f0d 100644 --- a/docs/apache-airflow-providers-grpc/changelog.rst +++ b/docs/apache-airflow-providers-grpc/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/grpc/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/grpc/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-hashicorp/changelog.rst b/docs/apache-airflow-providers-hashicorp/changelog.rst index 7e670c6443e31..a2ab3e789158c 100644 --- a/docs/apache-airflow-providers-hashicorp/changelog.rst +++ b/docs/apache-airflow-providers-hashicorp/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/hashicorp/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/hashicorp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-http/changelog.rst b/docs/apache-airflow-providers-http/changelog.rst index 31947ff163d81..631df66a668d6 100644 --- a/docs/apache-airflow-providers-http/changelog.rst +++ b/docs/apache-airflow-providers-http/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/http/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/http/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-http/index.rst b/docs/apache-airflow-providers-http/index.rst index c99999083f0db..2e7bdf6b78d88 100644 --- a/docs/apache-airflow-providers-http/index.rst +++ b/docs/apache-airflow-providers-http/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/http/index> + System Tests <_api/tests/system/http/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-http/operators.rst b/docs/apache-airflow-providers-http/operators.rst index 3f52ca0a62d58..87473dc48086a 100644 --- a/docs/apache-airflow-providers-http/operators.rst +++ b/docs/apache-airflow-providers-http/operators.rst @@ -32,14 +32,14 @@ to ``true``. Here we are poking until httpbin gives us a response text containing ``httpbin``. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_http_sensor_check] :end-before: [END howto_operator_http_http_sensor_check] This sensor can also be used in deferrable mode -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_http_sensor_check_deferrable] :end-before: [END howto_operator_http_http_sensor_check_deferrable] @@ -76,14 +76,14 @@ the response text back. In the first example we are calling a ``POST`` with json data and succeed when we get the same json data back otherwise the task will fail. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_task_post_op] :end-before: [END howto_operator_http_task_post_op] Here we are calling a ``GET`` request and pass params to it. The task will succeed regardless of the response text. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_task_get_op] :end-before: [END howto_operator_http_task_get_op] @@ -98,7 +98,7 @@ it on the next task downstream use ``response_filter``. This is useful if: Below is an example of retrieving data from a REST API and only returning a nested property instead of the full response body. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_task_get_op_response_filter] :end-before: [END howto_operator_http_task_get_op_response_filter] @@ -106,7 +106,7 @@ response body. In the third example we are performing a ``PUT`` operation to put / set data according to the data that is being provided to the request. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_task_put_op] :end-before: [END howto_operator_http_task_put_op] @@ -114,14 +114,14 @@ provided to the request. In this example we call a ``DELETE`` operation to the ``delete`` endpoint. This time we are passing form data to the request. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_task_del_op] :end-before: [END howto_operator_http_task_del_op] Here we pass form data to a ``POST`` operation which is equal to a usual form submit. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_task_post_op_formenc] :end-before: [END howto_operator_http_task_post_op_formenc] @@ -140,7 +140,7 @@ You can write a ``pagination_function`` that will receive the raw ``request.Resp generate new request parameters (as ``dict``) based on this cursor. The HttpOperator will repeat calls to the API until the function stop returning anything. -.. exampleinclude:: /../../tests/system/providers/http/example_http.py +.. exampleinclude:: /../../providers/tests/system/http/example_http.py :language: python :start-after: [START howto_operator_http_pagination_function] :end-before: [END howto_operator_http_pagination_function] diff --git a/docs/apache-airflow-providers-imap/changelog.rst b/docs/apache-airflow-providers-imap/changelog.rst index f10bae15755c4..1cfb71752efa5 100644 --- a/docs/apache-airflow-providers-imap/changelog.rst +++ b/docs/apache-airflow-providers-imap/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/imap/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/imap/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-influxdb/changelog.rst b/docs/apache-airflow-providers-influxdb/changelog.rst index 32fe8a7a55e38..e1798cda0d09a 100644 --- a/docs/apache-airflow-providers-influxdb/changelog.rst +++ b/docs/apache-airflow-providers-influxdb/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/influxdb/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/influxdb/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-influxdb/index.rst b/docs/apache-airflow-providers-influxdb/index.rst index 8df098b8481cd..3ab429376c05e 100644 --- a/docs/apache-airflow-providers-influxdb/index.rst +++ b/docs/apache-airflow-providers-influxdb/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/influxdb/index> + System Tests <_api/tests/system/influxdb/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-influxdb/operators/index.rst b/docs/apache-airflow-providers-influxdb/operators/index.rst index b3d0350477dca..d05fb6d1d9159 100644 --- a/docs/apache-airflow-providers-influxdb/operators/index.rst +++ b/docs/apache-airflow-providers-influxdb/operators/index.rst @@ -27,7 +27,7 @@ SQL commands in a `InfluxDB `__ database. An example of running the query using the operator: -.. exampleinclude:: /../../tests/system/providers/influxdb/example_influxdb_query.py +.. exampleinclude:: /../../providers/tests/system/influxdb/example_influxdb_query.py :language: python :start-after: [START howto_operator_influxdb] :end-before: [END howto_operator_influxdb] diff --git a/docs/apache-airflow-providers-jdbc/changelog.rst b/docs/apache-airflow-providers-jdbc/changelog.rst index 6697e52b7f0b6..916dba68d1b1f 100644 --- a/docs/apache-airflow-providers-jdbc/changelog.rst +++ b/docs/apache-airflow-providers-jdbc/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/jdbc/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/jdbc/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-jdbc/index.rst b/docs/apache-airflow-providers-jdbc/index.rst index e31369841cd7c..3010c1e5d0802 100644 --- a/docs/apache-airflow-providers-jdbc/index.rst +++ b/docs/apache-airflow-providers-jdbc/index.rst @@ -50,14 +50,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/jdbc/index> + System Tests <_api/tests/system/jdbc/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-jdbc/operators.rst b/docs/apache-airflow-providers-jdbc/operators.rst index ea7297635b915..eba54d065d0c7 100644 --- a/docs/apache-airflow-providers-jdbc/operators.rst +++ b/docs/apache-airflow-providers-jdbc/operators.rst @@ -73,7 +73,7 @@ commands against a database (or data storage) accessible via a JDBC driver. The :doc:`JDBC Connection ` must be passed as ``conn_id``. -.. exampleinclude:: /../../tests/system/providers/jdbc/example_jdbc_queries.py +.. exampleinclude:: /../../providers/tests/system/jdbc/example_jdbc_queries.py :language: python :start-after: [START howto_operator_jdbc] :end-before: [END howto_operator_jdbc] @@ -91,7 +91,7 @@ Templating You can use :ref:`Jinja templates ` to parameterize ``sql``. -.. exampleinclude:: /../../tests/system/providers/jdbc/example_jdbc_queries.py +.. exampleinclude:: /../../providers/tests/system/jdbc/example_jdbc_queries.py :language: python :start-after: [START howto_operator_jdbc_template] :end-before: [END howto_operator_jdbc_template] diff --git a/docs/apache-airflow-providers-jenkins/changelog.rst b/docs/apache-airflow-providers-jenkins/changelog.rst index 4f556ba51445e..40dd4c7db5d9b 100644 --- a/docs/apache-airflow-providers-jenkins/changelog.rst +++ b/docs/apache-airflow-providers-jenkins/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/jenkins/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/jenkins/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-jenkins/index.rst b/docs/apache-airflow-providers-jenkins/index.rst index d890406508df5..1aa841a5d8f84 100644 --- a/docs/apache-airflow-providers-jenkins/index.rst +++ b/docs/apache-airflow-providers-jenkins/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/jenkins/index> + System Tests <_api/tests/system/jenkins/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-microsoft-azure/changelog.rst b/docs/apache-airflow-providers-microsoft-azure/changelog.rst index 4456c720c8840..8b73842cf5a9e 100644 --- a/docs/apache-airflow-providers-microsoft-azure/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-azure/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/microsoft/azure/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/microsoft/azure/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst index 7a073a3b72341..5eb7c53813f3c 100644 --- a/docs/apache-airflow-providers-microsoft-azure/index.rst +++ b/docs/apache-airflow-providers-microsoft-azure/index.rst @@ -54,14 +54,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/microsoft/azure/index> + System Tests <_api/tests/system/microsoft/azure/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/adf_run_pipeline.rst b/docs/apache-airflow-providers-microsoft-azure/operators/adf_run_pipeline.rst index 024873d597a55..0feb5802093ad 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/adf_run_pipeline.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/adf_run_pipeline.rst @@ -31,7 +31,7 @@ This functionality can be disabled for an asynchronous wait -- typically with th Below is an example of using this operator to execute an Azure Data Factory pipeline. - .. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adf_run_pipeline.py + .. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adf_run_pipeline.py :language: python :dedent: 0 :start-after: [START howto_operator_adf_run_pipeline] @@ -40,7 +40,7 @@ Below is an example of using this operator to execute an Azure Data Factory pipe Below is an example of using this operator to execute an Azure Data Factory pipeline with a deferrable flag so that polling for the status of the pipeline run occurs on the Airflow Triggerer. - .. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adf_run_pipeline.py + .. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adf_run_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_adf_run_pipeline_with_deferrable_flag] @@ -48,7 +48,7 @@ so that polling for the status of the pipeline run occurs on the Airflow Trigger Here is a different example of using this operator to execute a pipeline but coupled with the :class:`~airflow.providers.microsoft.azure.sensors.data_factory.AzureDataFactoryPipelineRunStatusSensor` to perform an asynchronous wait. - .. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adf_run_pipeline.py + .. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adf_run_pipeline.py :language: python :dedent: 0 :start-after: [START howto_operator_adf_run_pipeline_async] @@ -56,7 +56,7 @@ Here is a different example of using this operator to execute a pipeline but cou Also you can use deferrable mode in :class:`~airflow.providers.microsoft.azure.sensors.data_factory.AzureDataFactoryPipelineRunStatusSensor` if you would like to free up the worker slots while the sensor is running. - .. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adf_run_pipeline.py + .. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adf_run_pipeline.py :language: python :dedent: 0 :start-after: [START howto_operator_adf_run_pipeline_async] @@ -72,7 +72,7 @@ status of a data factory pipeline run asynchronously. This sensor will free up t polling for job status happens on the Airflow triggerer, leading to efficient utilization of resources within Airflow. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adf_run_pipeline.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adf_run_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_adf_run_pipeline_async] diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/adls.rst b/docs/apache-airflow-providers-microsoft-azure/operators/adls.rst index 24b6a3ac5603e..4209bcf3096ac 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/adls.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/adls.rst @@ -35,7 +35,7 @@ upload data to Azure DataLake Storage Below is an example of using this operator to upload data to ADL. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adls_create.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adls_create.py :language: python :dedent: 0 :start-after: [START howto_operator_adls_create] @@ -52,7 +52,7 @@ file(s) from Azure DataLake Storage Below is an example of using this operator to delete a file from ADL. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adls_delete.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adls_delete.py :language: python :dedent: 0 :start-after: [START howto_operator_adls_delete] @@ -69,7 +69,7 @@ file(s) from Azure DataLake Storage Below is an example of using this operator to list files from ADL. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_adls_list.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_adls_list.py :language: python :dedent: 0 :start-after: [START howto_operator_adls_list] diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst b/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst index 9adbbddc4c92e..c7cdfec9e7598 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst @@ -41,7 +41,7 @@ To create Azure service bus queue with specific Parameter you can use Below is an example of using this operator to execute an Azure Service Bus Create Queue. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_create_service_bus_queue] @@ -58,7 +58,7 @@ To Send message or list of message or batch Message to the Azure Service Bus Que Below is an example of using this operator to execute an Azure Service Bus Send Message to Queue. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_send_message_to_service_bus_queue] @@ -75,7 +75,7 @@ To Receive Message or list of message or Batch message in a Queue you can use Below is an example of using this operator to execute an Azure Service Bus Create Queue. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_receive_message_service_bus_queue] @@ -92,7 +92,7 @@ To Delete the Azure service bus queue you can use Below is an example of using this operator to execute an Azure Service Bus Delete Queue. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_service_bus_queue] @@ -113,7 +113,7 @@ To create Azure service bus topic with specific Parameter you can use Below is an example of using this operator to execute an Azure Service Bus Create Topic. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_create_service_bus_topic] @@ -129,7 +129,7 @@ To Delete the Azure service bus topic you can use Below is an example of using this operator to execute an Azure Service Bus Delete topic. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_service_bus_topic] @@ -150,7 +150,7 @@ To create Azure service bus topic Subscription with specific Parameter you can u Below is an example of using this operator to execute an Azure Service Bus Create Subscription. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_create_service_bus_subscription] @@ -166,7 +166,7 @@ To Update the Azure service bus topic Subscription which is already created, wit Below is an example of using this operator to execute an Azure Service Bus Update Subscription. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_update_service_bus_subscription] @@ -182,7 +182,7 @@ To Receive a Batch messages from a Service Bus Subscription under specific Topic Below is an example of using this operator to execute an Azure Service Bus Receive Subscription Message. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_receive_message_service_bus_subscription] @@ -198,7 +198,7 @@ To Delete the Azure service bus topic Subscription you can use Below is an example of using this operator to execute an Azure Service Bus Delete Subscription under topic. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_service_bus.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_service_bus.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_service_bus_subscription] diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/azure_synapse.rst b/docs/apache-airflow-providers-microsoft-azure/operators/azure_synapse.rst index 440e173d04a78..f3a2a3affd760 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/azure_synapse.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/azure_synapse.rst @@ -34,7 +34,7 @@ terminate with a "Succeeded" status. Below is an example of using this operator to execute a Spark application on Azure Synapse. - .. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_synapse.py + .. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_synapse.py :language: python :dedent: 4 :start-after: [START howto_operator_azure_synapse] @@ -46,7 +46,7 @@ AzureSynapseRunPipelineOperator Use the: class:`~airflow.providers.microsoft.azure.operators.synapse.AzureSynapseRunPipelineOperator` to execute a pipeline application within Synapse Analytics. The operator will Execute a Synapse Pipeline. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_synapse_run_pipeline.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py :language: python :dedent: 4 :start-after: [START howto_operator_azure_synapse_run_pipeline] diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst b/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst index 5fa5a3808d906..32d40ecbe5ba7 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst @@ -26,7 +26,7 @@ Use the Below is an example of using this operator to trigger a task on Azure Batch -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_batch_operator.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_batch_operator.py :language: python :dedent: 0 :start-after: [START howto_azure_batch_operator] diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/msgraph.rst b/docs/apache-airflow-providers-microsoft-azure/operators/msgraph.rst index 56a9259f93143..a91a833855f70 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/msgraph.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/msgraph.rst @@ -34,7 +34,7 @@ Use the Below is an example of using this operator to get a Sharepoint site. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_msgraph.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_msgraph.py :language: python :dedent: 0 :start-after: [START howto_operator_graph_site] @@ -42,7 +42,7 @@ Below is an example of using this operator to get a Sharepoint site. Below is an example of using this operator to get a Sharepoint site pages. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_msgraph.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_msgraph.py :language: python :dedent: 0 :start-after: [START howto_operator_graph_site_pages] @@ -50,7 +50,7 @@ Below is an example of using this operator to get a Sharepoint site pages. Below is an example of using this operator to get PowerBI workspaces. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_powerbi.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_powerbi.py :language: python :dedent: 0 :start-after: [START howto_operator_powerbi_workspaces] @@ -58,7 +58,7 @@ Below is an example of using this operator to get PowerBI workspaces. Below is an example of using this operator to get PowerBI workspaces info. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_powerbi.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_powerbi.py :language: python :dedent: 0 :start-after: [START howto_operator_powerbi_workspaces_info] @@ -66,7 +66,7 @@ Below is an example of using this operator to get PowerBI workspaces info. Below is an example of using this operator to refresh PowerBI dataset. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_powerbi.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_powerbi.py :language: python :dedent: 0 :start-after: [START howto_operator_powerbi_refresh_dataset] @@ -74,7 +74,7 @@ Below is an example of using this operator to refresh PowerBI dataset. Below is an example of using this operator to create an item schedule in Fabric. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_msfabric.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_msfabric.py :language: python :dedent: 0 :start-after: [START howto_operator_ms_fabric_create_item_schedule] diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sensor.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sensor.rst index c0a7d140cc22c..f15064d4852ff 100644 --- a/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sensor.rst +++ b/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sensor.rst @@ -28,7 +28,7 @@ Azure Cosmos Document Sensor Checks for the existence of a document which matches the given query in CosmosDB. :class:`~airflow.providers.microsoft.azure.sensors.cosmos.AzureCosmosDocumentSensor` -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_cosmosdb.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_azure_cosmosdb.py :language: python :dedent: 4 :start-after: [START cosmos_document_sensor] diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/msgraph.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/msgraph.rst index 4ddad88f19fa1..f3a6e5c8c99e6 100644 --- a/docs/apache-airflow-providers-microsoft-azure/sensors/msgraph.rst +++ b/docs/apache-airflow-providers-microsoft-azure/sensors/msgraph.rst @@ -27,7 +27,7 @@ Use the Below is an example of using this sensor to poll the status of a PowerBI workspace. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_powerbi.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_powerbi.py :language: python :dedent: 0 :start-after: [START howto_sensor_powerbi_scan_status] diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst index c01adeab1a89f..2f79f6df6b20b 100644 --- a/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst +++ b/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst @@ -29,7 +29,7 @@ Wasb Blob Sensor Waits for a blob to arrive on Azure Blob Storage. :class:`~airflow.providers.microsoft.azure.sensors.wasb.WasbBlobSensor` -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_wasb_sensors.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_wasb_sensors.py :language: python :dedent: 4 :start-after: [START wasb_blob_sensor] @@ -41,7 +41,7 @@ Wasb Prefix Sensor Waits for blobs matching a prefix to arrive on Azure Blob Storage. :class:`~airflow.providers.microsoft.azure.sensors.wasb.WasbPrefixSensor` -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_wasb_sensors.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_wasb_sensors.py :language: python :dedent: 4 :start-after: [START wasb_prefix_sensor] diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst index 76a06133d21db..19f4c36d15c27 100644 --- a/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst +++ b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst @@ -38,7 +38,7 @@ upload data from local filesystem to ADL. Below is an example of using this operator to upload a file to ADL. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_local_to_adls.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_local_to_adls.py :language: python :dedent: 0 :start-after: [START howto_operator_local_to_adls] diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst index 970a6298342ed..fed25a5797ae6 100644 --- a/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst +++ b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst @@ -36,7 +36,7 @@ upload data from local filesystem to Azure Blob Storage. Below is an example of using this operator to upload a file to Azure Blob Storage. -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_local_to_wasb.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_local_to_wasb.py :language: python :dedent: 0 :start-after: [START howto_operator_local_to_wasb] diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst index 59a394b0935fb..e206611389530 100644 --- a/docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst +++ b/docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst @@ -47,7 +47,7 @@ To copy data from an Amazon AWS S3 Bucket to an Azure Blob Storage container, th Example usage: -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_s3_to_wasb.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_s3_to_wasb.py :language: python :dedent: 4 :start-after: [START howto_transfer_s3_to_wasb] diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst index a79500de6a3bc..a4699655a3d3d 100644 --- a/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst +++ b/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst @@ -54,7 +54,7 @@ To get information about jobs within a Azure Blob Storage use: :class:`~airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPToWasbOperator` Example usage: -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_sftp_to_wasb.py +.. exampleinclude:: /../../providers/tests/system/microsoft/azure/example_sftp_to_wasb.py :language: python :dedent: 4 :start-after: [START how_to_sftp_to_wasb] diff --git a/docs/apache-airflow-providers-microsoft-mssql/changelog.rst b/docs/apache-airflow-providers-microsoft-mssql/changelog.rst index d7b97646ce1a3..5c5b6c990874b 100644 --- a/docs/apache-airflow-providers-microsoft-mssql/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-mssql/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/microsoft/mssql/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/microsoft/mssql/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-mssql/index.rst b/docs/apache-airflow-providers-microsoft-mssql/index.rst index a1032303afe23..f14c3caf7f2d1 100644 --- a/docs/apache-airflow-providers-microsoft-mssql/index.rst +++ b/docs/apache-airflow-providers-microsoft-mssql/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/microsoft/mssql/index> + System Tests <_api/tests/system/microsoft/mssql/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-microsoft-mssql/operators.rst b/docs/apache-airflow-providers-microsoft-mssql/operators.rst index 9d2189c7719bf..560e196731b57 100644 --- a/docs/apache-airflow-providers-microsoft-mssql/operators.rst +++ b/docs/apache-airflow-providers-microsoft-mssql/operators.rst @@ -41,7 +41,7 @@ The code snippets below are based on Airflow-2.2 An example usage of the SQLExecuteQueryOperator to connect to MSSQL is as follows: -.. exampleinclude:: /../../tests/system/providers/microsoft/mssql/example_mssql.py +.. exampleinclude:: /../../providers/tests/system/microsoft/mssql/example_mssql.py :language: python :start-after: [START howto_operator_mssql] :end-before: [END howto_operator_mssql] @@ -49,7 +49,7 @@ An example usage of the SQLExecuteQueryOperator to connect to MSSQL is as follow You can also use an external file to execute the SQL commands. Script folder must be at the same level as DAG.py file. This way you can easily maintain the SQL queries separated from the code. -.. exampleinclude:: /../../tests/system/providers/microsoft/mssql/example_mssql.py +.. exampleinclude:: /../../providers/tests/system/microsoft/mssql/example_mssql.py :language: python :start-after: [START mssql_operator_howto_guide_create_table_mssql_from_external_file] :end-before: [END mssql_operator_howto_guide_create_table_mssql_from_external_file] @@ -71,7 +71,7 @@ Inserting data into a MSSQL database table --------------------------------------------- We can then create a SQLExecuteQueryOperator task that populate the ``Users`` table. -.. exampleinclude:: /../../tests/system/providers/microsoft/mssql/example_mssql.py +.. exampleinclude:: /../../providers/tests/system/microsoft/mssql/example_mssql.py :language: python :start-after: [START mssql_operator_howto_guide_populate_user_table] :end-before: [END mssql_operator_howto_guide_populate_user_table] @@ -82,7 +82,7 @@ Fetching records from your MSSQL database table Fetching records from your MSSQL database table can be as simple as: -.. exampleinclude:: /../../tests/system/providers/microsoft/mssql/example_mssql.py +.. exampleinclude:: /../../providers/tests/system/microsoft/mssql/example_mssql.py :language: python :start-after: [START mssql_operator_howto_guide_get_all_countries] :end-before: [END mssql_operator_howto_guide_get_all_countries] @@ -96,7 +96,7 @@ SQL requests during runtime. To find the countries in Asian continent: -.. exampleinclude:: /../../tests/system/providers/microsoft/mssql/example_mssql.py +.. exampleinclude:: /../../providers/tests/system/microsoft/mssql/example_mssql.py :language: python :start-after: [START mssql_operator_howto_guide_params_passing_get_query] :end-before: [END mssql_operator_howto_guide_params_passing_get_query] @@ -107,7 +107,7 @@ The complete SQLExecuteQueryOperator DAG to connect to MSSQL When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/microsoft/mssql/example_mssql.py +.. exampleinclude:: /../../providers/tests/system/microsoft/mssql/example_mssql.py :language: python :start-after: [START mssql_operator_howto_guide] :end-before: [END mssql_operator_howto_guide] diff --git a/docs/apache-airflow-providers-microsoft-psrp/changelog.rst b/docs/apache-airflow-providers-microsoft-psrp/changelog.rst index bba78b70734f4..ed4648eded19f 100644 --- a/docs/apache-airflow-providers-microsoft-psrp/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-psrp/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/microsoft/psrp/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/microsoft/psrp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-winrm/changelog.rst b/docs/apache-airflow-providers-microsoft-winrm/changelog.rst index 05d79981df50d..fb0faf44d10fb 100644 --- a/docs/apache-airflow-providers-microsoft-winrm/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-winrm/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/microsoft/winrm/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/microsoft/winrm/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-winrm/index.rst b/docs/apache-airflow-providers-microsoft-winrm/index.rst index b3b7cc0afcfb2..6bc6954fe11e6 100644 --- a/docs/apache-airflow-providers-microsoft-winrm/index.rst +++ b/docs/apache-airflow-providers-microsoft-winrm/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/microsoft/winrm/index> + System Tests <_api/tests/system/microsoft/winrm/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-microsoft-winrm/operators.rst b/docs/apache-airflow-providers-microsoft-winrm/operators.rst index e70d124f6262c..2e7fdc6633486 100644 --- a/docs/apache-airflow-providers-microsoft-winrm/operators.rst +++ b/docs/apache-airflow-providers-microsoft-winrm/operators.rst @@ -22,7 +22,7 @@ use the WinRMOperator to execute commands on a given remote host using the winrm create a hook -.. exampleinclude:: /../../tests/system/providers/microsoft/winrm/example_winrm.py +.. exampleinclude:: /../../providers/tests/system/microsoft/winrm/example_winrm.py :language: python :dedent: 4 :start-after: [START create_hook] @@ -30,7 +30,7 @@ create a hook Run the operator, pass the hook, and pass a command to do something -.. exampleinclude:: /../../tests/system/providers/microsoft/winrm/example_winrm.py +.. exampleinclude:: /../../providers/tests/system/microsoft/winrm/example_winrm.py :language: python :dedent: 4 :start-after: [START run_operator] diff --git a/docs/apache-airflow-providers-mongo/changelog.rst b/docs/apache-airflow-providers-mongo/changelog.rst index dc2b2dead5d62..a1714d0e2b49e 100644 --- a/docs/apache-airflow-providers-mongo/changelog.rst +++ b/docs/apache-airflow-providers-mongo/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/mongo/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/mongo/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-mysql/changelog.rst b/docs/apache-airflow-providers-mysql/changelog.rst index 40b5da4fbad92..807ca2ce0730c 100644 --- a/docs/apache-airflow-providers-mysql/changelog.rst +++ b/docs/apache-airflow-providers-mysql/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/mysql/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/mysql/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-mysql/index.rst b/docs/apache-airflow-providers-mysql/index.rst index 622084d88957c..d8d6129134129 100644 --- a/docs/apache-airflow-providers-mysql/index.rst +++ b/docs/apache-airflow-providers-mysql/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/mysql/index> + System Tests <_api/tests/system/mysql/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-mysql/operators.rst b/docs/apache-airflow-providers-mysql/operators.rst index 2ee7b885cd7bc..55b3727822973 100644 --- a/docs/apache-airflow-providers-mysql/operators.rst +++ b/docs/apache-airflow-providers-mysql/operators.rst @@ -53,14 +53,14 @@ the connection metadata is structured as follows: An example usage of the SQLExecuteQueryOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/mysql/example_mysql.py +.. exampleinclude:: /../../providers/tests/system/mysql/example_mysql.py :language: python :start-after: [START howto_operator_mysql] :end-before: [END howto_operator_mysql] You can also use an external file to execute the SQL commands. Script folder must be at the same level as DAG.py file. -.. exampleinclude:: /../../tests/system/providers/mysql/example_mysql.py +.. exampleinclude:: /../../providers/tests/system/mysql/example_mysql.py :language: python :start-after: [START howto_operator_mysql_external_file] :end-before: [END howto_operator_mysql_external_file] diff --git a/docs/apache-airflow-providers-neo4j/changelog.rst b/docs/apache-airflow-providers-neo4j/changelog.rst index 124d526faa220..8b3d87f653f39 100644 --- a/docs/apache-airflow-providers-neo4j/changelog.rst +++ b/docs/apache-airflow-providers-neo4j/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/neo4j/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/neo4j/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-neo4j/index.rst b/docs/apache-airflow-providers-neo4j/index.rst index 83b78bcd892c9..1b78a2bdba8bd 100644 --- a/docs/apache-airflow-providers-neo4j/index.rst +++ b/docs/apache-airflow-providers-neo4j/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/neo4j/index> + System Tests <_api/tests/system/neo4j/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-neo4j/operators/neo4j.rst b/docs/apache-airflow-providers-neo4j/operators/neo4j.rst index 5bc2362e91089..14f3e3da2c93e 100644 --- a/docs/apache-airflow-providers-neo4j/operators/neo4j.rst +++ b/docs/apache-airflow-providers-neo4j/operators/neo4j.rst @@ -49,7 +49,7 @@ the connection metadata is structured as follows: * - Port: int - Neo4j port -.. exampleinclude:: /../../tests/system/providers/neo4j/example_neo4j.py +.. exampleinclude:: /../../providers/tests/system/neo4j/example_neo4j.py :language: python :dedent: 4 :start-after: [START run_query_neo4j_operator] diff --git a/docs/apache-airflow-providers-odbc/changelog.rst b/docs/apache-airflow-providers-odbc/changelog.rst index 1c24fdd8fe6b0..6028c91353b1d 100644 --- a/docs/apache-airflow-providers-odbc/changelog.rst +++ b/docs/apache-airflow-providers-odbc/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/odbc/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/odbc/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-openai/changelog.rst b/docs/apache-airflow-providers-openai/changelog.rst index 64206f909af45..50402f1c9929b 100644 --- a/docs/apache-airflow-providers-openai/changelog.rst +++ b/docs/apache-airflow-providers-openai/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/openai/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/openai/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-openai/index.rst b/docs/apache-airflow-providers-openai/index.rst index ac0130e9d93ed..8e07d82ca6c16 100644 --- a/docs/apache-airflow-providers-openai/index.rst +++ b/docs/apache-airflow-providers-openai/index.rst @@ -52,7 +52,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/openai/index> + System Tests <_api/tests/system/openai/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-openai/operators/openai.rst b/docs/apache-airflow-providers-openai/operators/openai.rst index fef8521188df6..d82a8058f298d 100644 --- a/docs/apache-airflow-providers-openai/operators/openai.rst +++ b/docs/apache-airflow-providers-openai/operators/openai.rst @@ -32,7 +32,7 @@ connect to your account. An example using the operator is in way: -.. exampleinclude:: /../../tests/system/providers/openai/example_openai.py +.. exampleinclude:: /../../providers/tests/system/openai/example_openai.py :language: python :start-after: [START howto_operator_openai_embedding] :end-before: [END howto_operator_openai_embedding] @@ -57,7 +57,7 @@ The OpenAITriggerBatchOperator An example using the operator is in way: -.. exampleinclude:: /../../tests/system/providers/openai/example_trigger_batch_operator.py +.. exampleinclude:: /../../providers/tests/system/openai/example_trigger_batch_operator.py :language: python :start-after: [START howto_operator_openai_trigger_operator] :end-before: [END howto_operator_openai_trigger_operator] diff --git a/docs/apache-airflow-providers-openfaas/changelog.rst b/docs/apache-airflow-providers-openfaas/changelog.rst index 48653876678d8..507f650340ba2 100644 --- a/docs/apache-airflow-providers-openfaas/changelog.rst +++ b/docs/apache-airflow-providers-openfaas/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/openfaas/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/openfaas/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-openlineage/changelog.rst b/docs/apache-airflow-providers-openlineage/changelog.rst index 9c9060dd21a8e..f410ab7a2eaf7 100644 --- a/docs/apache-airflow-providers-openlineage/changelog.rst +++ b/docs/apache-airflow-providers-openlineage/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/openlineage/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/openlineage/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-openlineage/guides/developer.rst b/docs/apache-airflow-providers-openlineage/guides/developer.rst index ccab215fc1846..d826ccde9a5ec 100644 --- a/docs/apache-airflow-providers-openlineage/guides/developer.rst +++ b/docs/apache-airflow-providers-openlineage/guides/developer.rst @@ -290,7 +290,7 @@ To learn more about how Operators and Extractors work together under the hood, c When testing an Extractor, we want to firstly verify if ``OperatorLineage`` object is being created, specifically verifying that the object is being built with the correct input and output datasets and relevant facets. This is done in OpenLineage via pytest, with appropriate mocking and patching for connections and objects. -Check out `example tests `_. +Check out `example tests `_. Testing each facet is also important, as data or graphs in the UI can render incorrectly if the facets are wrong. For example, if the facet name is created incorrectly in the Extractor, then the Operator's task will not show up in the lineage graph, diff --git a/docs/apache-airflow-providers-opensearch/changelog.rst b/docs/apache-airflow-providers-opensearch/changelog.rst index 2ce00fd5fdcff..21f39a6c1f6ea 100644 --- a/docs/apache-airflow-providers-opensearch/changelog.rst +++ b/docs/apache-airflow-providers-opensearch/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/opensearch/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/opensearch/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-opensearch/index.rst b/docs/apache-airflow-providers-opensearch/index.rst index de79657141fad..efa238c993ef1 100644 --- a/docs/apache-airflow-providers-opensearch/index.rst +++ b/docs/apache-airflow-providers-opensearch/index.rst @@ -51,7 +51,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/opensearch/index> + System Tests <_api/tests/system/opensearch/index> .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-opensearch/operators/opensearch.rst b/docs/apache-airflow-providers-opensearch/operators/opensearch.rst index cd39a59566fbb..b85a014ebefee 100644 --- a/docs/apache-airflow-providers-opensearch/operators/opensearch.rst +++ b/docs/apache-airflow-providers-opensearch/operators/opensearch.rst @@ -35,7 +35,7 @@ to create a new index in an OpenSearch domain. -.. exampleinclude:: /../../tests/system/providers/opensearch/example_opensearch.py +.. exampleinclude:: /../../providers/tests/system/opensearch/example_opensearch.py :language: python :start-after: [START howto_operator_opensearch_create_index] :dedent: 4 @@ -50,7 +50,7 @@ Add a Document to an Index on OpenSearch Use :class:`~airflow.providers.opensearch.operators.opensearch.OpenSearchAddDocumentOperator` to add single documents to an OpenSearch Index -.. exampleinclude:: /../../tests/system/providers/opensearch/example_opensearch.py +.. exampleinclude:: /../../providers/tests/system/opensearch/example_opensearch.py :language: python :start-after: [START howto_operator_opensearch_add_document] :dedent: 4 @@ -65,7 +65,7 @@ Run a query against an OpenSearch Index Use :class:`~airflow.providers.opensearch.operators.opensearch.OpenSearchQueryOperator` to run a query against an OpenSearch index. -.. exampleinclude:: /../../tests/system/providers/opensearch/example_opensearch.py +.. exampleinclude:: /../../providers/tests/system/opensearch/example_opensearch.py :language: python :start-after: [START howto_operator_opensearch_query] :dedent: 4 diff --git a/docs/apache-airflow-providers-opsgenie/changelog.rst b/docs/apache-airflow-providers-opsgenie/changelog.rst index 4c9f227f1dd62..abb3ab86138c5 100644 --- a/docs/apache-airflow-providers-opsgenie/changelog.rst +++ b/docs/apache-airflow-providers-opsgenie/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/opsgenie/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/opsgenie/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-opsgenie/index.rst b/docs/apache-airflow-providers-opsgenie/index.rst index b09e294bbc85e..30b35890e97c2 100644 --- a/docs/apache-airflow-providers-opsgenie/index.rst +++ b/docs/apache-airflow-providers-opsgenie/index.rst @@ -50,14 +50,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/opsgenie/index> + System Tests <_api/tests/system/opsgenie/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-opsgenie/notifications/opsgenie_notifier.rst b/docs/apache-airflow-providers-opsgenie/notifications/opsgenie_notifier.rst index 54cff970d4a68..2172be24faa41 100644 --- a/docs/apache-airflow-providers-opsgenie/notifications/opsgenie_notifier.rst +++ b/docs/apache-airflow-providers-opsgenie/notifications/opsgenie_notifier.rst @@ -27,7 +27,7 @@ Using the Notifier ^^^^^^^^^^^^^^^^^^ Send an alert to Opsgenie with a specific message. -.. exampleinclude:: /../../tests/system/providers/opsgenie/example_opsgenie_notifier.py +.. exampleinclude:: /../../providers/tests/system/opsgenie/example_opsgenie_notifier.py :language: python :start-after: [START howto_notifier_opsgenie] :end-before: [END howto_notifier_opsgenie] diff --git a/docs/apache-airflow-providers-opsgenie/operators/opsgenie_alert.rst b/docs/apache-airflow-providers-opsgenie/operators/opsgenie_alert.rst index 5ba7cb6a8952d..ebed7548c4f03 100644 --- a/docs/apache-airflow-providers-opsgenie/operators/opsgenie_alert.rst +++ b/docs/apache-airflow-providers-opsgenie/operators/opsgenie_alert.rst @@ -27,7 +27,7 @@ Using the Operator ^^^^^^^^^^^^^^^^^^ Send an alert to Opsgenie with a specific message. -.. exampleinclude:: /../../tests/system/providers/opsgenie/example_opsgenie_alert.py +.. exampleinclude:: /../../providers/tests/system/opsgenie/example_opsgenie_alert.py :language: python :start-after: [START howto_opsgenie_create_alert_operator] :end-before: [END howto_opsgenie_create_alert_operator] @@ -44,7 +44,7 @@ Using the Operator ^^^^^^^^^^^^^^^^^^ Close alert in Opsgenie. -.. exampleinclude:: /../../tests/system/providers/opsgenie/example_opsgenie_alert.py +.. exampleinclude:: /../../providers/tests/system/opsgenie/example_opsgenie_alert.py :language: python :start-after: [START howto_opsgenie_close_alert_operator] :end-before: [END howto_opsgenie_close_alert_operator] @@ -61,7 +61,7 @@ Using the Operator ^^^^^^^^^^^^^^^^^^ Delete alert in Opsgenie. -.. exampleinclude:: /../../tests/system/providers/opsgenie/example_opsgenie_alert.py +.. exampleinclude:: /../../providers/tests/system/opsgenie/example_opsgenie_alert.py :language: python :dedent: 4 :start-after: [START howto_opsgenie_delete_alert_operator] diff --git a/docs/apache-airflow-providers-oracle/changelog.rst b/docs/apache-airflow-providers-oracle/changelog.rst index 4711449e6c6ce..b6cdc5037ba7c 100644 --- a/docs/apache-airflow-providers-oracle/changelog.rst +++ b/docs/apache-airflow-providers-oracle/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/oracle/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/oracle/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-oracle/operators/index.rst b/docs/apache-airflow-providers-oracle/operators/index.rst index af440c7d19b3c..990cb2e6503b2 100644 --- a/docs/apache-airflow-providers-oracle/operators/index.rst +++ b/docs/apache-airflow-providers-oracle/operators/index.rst @@ -30,7 +30,7 @@ To execute arbitrary SQL in an Oracle database, use the An example of executing a simple query is as follows: -.. exampleinclude:: /../../airflow/providers/oracle/example_dags/example_oracle.py +.. exampleinclude:: /../../providers/src/airflow/providers/oracle/example_dags/example_oracle.py :language: python :start-after: [START howto_oracle_operator] :end-before: [END howto_oracle_operator] @@ -58,7 +58,7 @@ a single integer argument, val_out. This can be represented with the following call using :class:`~airflow.providers.oracle.operators.oracle.OracleStoredProcedureOperator` with parameters passed positionally as a list: -.. exampleinclude:: /../../airflow/providers/oracle/example_dags/example_oracle.py +.. exampleinclude:: /../../providers/src/airflow/providers/oracle/example_dags/example_oracle.py :language: python :start-after: [START howto_oracle_stored_procedure_operator_with_list_inout] :end-before: [END howto_oracle_stored_procedure_operator_with_list_inout] @@ -67,7 +67,7 @@ with parameters passed positionally as a list: Alternatively, parameters can be passed as keyword arguments using a dictionary as well. -.. exampleinclude:: /../../airflow/providers/oracle/example_dags/example_oracle.py +.. exampleinclude:: /../../providers/src/airflow/providers/oracle/example_dags/example_oracle.py :language: python :start-after: [START howto_oracle_stored_procedure_operator_with_dict_inout] :end-before: [END howto_oracle_stored_procedure_operator_with_dict_inout] diff --git a/docs/apache-airflow-providers-pagerduty/changelog.rst b/docs/apache-airflow-providers-pagerduty/changelog.rst index ddd707b84f9fa..049844772748b 100644 --- a/docs/apache-airflow-providers-pagerduty/changelog.rst +++ b/docs/apache-airflow-providers-pagerduty/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/pagerduty/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/pagerduty/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-papermill/changelog.rst b/docs/apache-airflow-providers-papermill/changelog.rst index c022939a40099..3c11af574b2fe 100644 --- a/docs/apache-airflow-providers-papermill/changelog.rst +++ b/docs/apache-airflow-providers-papermill/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/papermill/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/papermill/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-papermill/index.rst b/docs/apache-airflow-providers-papermill/index.rst index b32d63bc9c89d..5217f7119e332 100644 --- a/docs/apache-airflow-providers-papermill/index.rst +++ b/docs/apache-airflow-providers-papermill/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/papermill/index> + System Tests <_api/tests/system/papermill/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-papermill/operators.rst b/docs/apache-airflow-providers-papermill/operators.rst index ed1cf580c85a4..fd17fba350602 100644 --- a/docs/apache-airflow-providers-papermill/operators.rst +++ b/docs/apache-airflow-providers-papermill/operators.rst @@ -50,7 +50,7 @@ Example DAG Use the :class:`~airflow.providers.papermill.operators.papermill.PapermillOperator` to execute a jupyter notebook: -.. exampleinclude:: /../../tests/system/providers/papermill/example_papermill.py +.. exampleinclude:: /../../providers/tests/system/papermill/example_papermill.py :language: python :dedent: 4 :start-after: [START howto_operator_papermill] @@ -58,7 +58,7 @@ to execute a jupyter notebook: Example DAG to Verify the message in the notebook: -.. exampleinclude:: /../../tests/system/providers/papermill/example_papermill_verify.py +.. exampleinclude:: /../../providers/tests/system/papermill/example_papermill_verify.py :language: python :start-after: [START howto_verify_operator_papermill] :end-before: [END howto_verify_operator_papermill] @@ -66,7 +66,7 @@ Example DAG to Verify the message in the notebook: Example DAG to Verify the message in the notebook using a remote jupyter kernel: -.. exampleinclude:: /../../tests/system/providers/papermill/example_papermill_remote_verify.py +.. exampleinclude:: /../../providers/tests/system/papermill/example_papermill_remote_verify.py :language: python :start-after: [START howto_verify_operator_papermill_remote_kernel] :end-before: [END howto_verify_operator_papermill_remote_kernel] diff --git a/docs/apache-airflow-providers-pgvector/changelog.rst b/docs/apache-airflow-providers-pgvector/changelog.rst index 9e33a9e9e6065..15c377d7cfc85 100644 --- a/docs/apache-airflow-providers-pgvector/changelog.rst +++ b/docs/apache-airflow-providers-pgvector/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/pgvector/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/pgvector/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-pgvector/index.rst b/docs/apache-airflow-providers-pgvector/index.rst index a7b280a9981e5..18d69634b876e 100644 --- a/docs/apache-airflow-providers-pgvector/index.rst +++ b/docs/apache-airflow-providers-pgvector/index.rst @@ -60,7 +60,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/pgvector/index> + System Tests <_api/tests/system/pgvector/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-pgvector/operators/pgvector.rst b/docs/apache-airflow-providers-pgvector/operators/pgvector.rst index dfee1c9fef2a5..b8ef6b27fccee 100644 --- a/docs/apache-airflow-providers-pgvector/operators/pgvector.rst +++ b/docs/apache-airflow-providers-pgvector/operators/pgvector.rst @@ -36,7 +36,7 @@ See https://github.com/pgvector/pgvector#installation for installation instructi An example using the operator to ingest data is shown below: -.. exampleinclude:: /../../tests/system/providers/pgvector/example_pgvector.py +.. exampleinclude:: /../../providers/tests/system/pgvector/example_pgvector.py :language: python :start-after: [START howto_operator_pgvector_ingest] :end-before: [END howto_operator_pgvector_ingest] diff --git a/docs/apache-airflow-providers-pinecone/changelog.rst b/docs/apache-airflow-providers-pinecone/changelog.rst index 73d9f14c9a674..4d21af53ed850 100644 --- a/docs/apache-airflow-providers-pinecone/changelog.rst +++ b/docs/apache-airflow-providers-pinecone/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/pinecone/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/pinecone/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-pinecone/index.rst b/docs/apache-airflow-providers-pinecone/index.rst index ea00879975c9c..8c329abde18e6 100644 --- a/docs/apache-airflow-providers-pinecone/index.rst +++ b/docs/apache-airflow-providers-pinecone/index.rst @@ -52,7 +52,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/pinecone/index> + System Tests <_api/tests/system/pinecone/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-pinecone/operators/pinecone.rst b/docs/apache-airflow-providers-pinecone/operators/pinecone.rst index b50e5300f09a9..0f8e49442fe5d 100644 --- a/docs/apache-airflow-providers-pinecone/operators/pinecone.rst +++ b/docs/apache-airflow-providers-pinecone/operators/pinecone.rst @@ -36,7 +36,7 @@ the original text corresponding to the vectors that could be ingested into the d An example using the operator in this way: -.. exampleinclude:: /../../tests/system/providers/pinecone/example_dag_pinecone.py +.. exampleinclude:: /../../providers/tests/system/pinecone/example_dag_pinecone.py :language: python :dedent: 4 :start-after: [START howto_operator_pinecone_ingest] @@ -58,7 +58,7 @@ passed via arguments to the operator or via the connection. An example using the operator in this way: -.. exampleinclude:: /../../tests/system/providers/pinecone/example_create_pod_index.py +.. exampleinclude:: /../../providers/tests/system/pinecone/example_create_pod_index.py :language: python :dedent: 4 :start-after: [START howto_operator_create_pod_index] @@ -81,7 +81,7 @@ passed via arguments to the operator or via the connection. An example using the operator in this way: -.. exampleinclude:: /../../tests/system/providers/pinecone/example_create_serverless_index.py +.. exampleinclude:: /../../providers/tests/system/pinecone/example_create_serverless_index.py :language: python :dedent: 4 :start-after: [START howto_operator_create_serverless_index] diff --git a/docs/apache-airflow-providers-postgres/changelog.rst b/docs/apache-airflow-providers-postgres/changelog.rst index f82d2144e7b0a..d2d34f24db829 100644 --- a/docs/apache-airflow-providers-postgres/changelog.rst +++ b/docs/apache-airflow-providers-postgres/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/postgres/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/postgres/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-postgres/index.rst b/docs/apache-airflow-providers-postgres/index.rst index 6fbd69ba1c5aa..2864276f37893 100644 --- a/docs/apache-airflow-providers-postgres/index.rst +++ b/docs/apache-airflow-providers-postgres/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/postgres/index> + System Tests <_api/tests/system/postgres/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst b/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst index 09402178aa057..d1b20dcea172f 100644 --- a/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst +++ b/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst @@ -45,7 +45,7 @@ Creating a Postgres database table The code snippets below are based on Airflow-2.0 -.. exampleinclude:: /../../tests/system/providers/postgres/example_postgres.py +.. exampleinclude:: /../../providers/tests/system/postgres/example_postgres.py :language: python :start-after: [START postgres_sql_execute_query_operator_howto_guide] :end-before: [END postgres_sql_execute_query_operator_howto_guide_create_pet_table] @@ -181,7 +181,7 @@ SQLExecuteQueryOperator provides ``hook_params`` attribute that allows you to pa You can pass ``options`` argument this way so that you specify `command-line options `_ sent to the server at connection start. -.. exampleinclude:: /../../tests/system/providers/postgres/example_postgres.py +.. exampleinclude:: /../../providers/tests/system/postgres/example_postgres.py :language: python :start-after: [START postgres_sql_execute_query_operator_howto_guide_get_birth_date] :end-before: [END postgres_sql_execute_query_operator_howto_guide_get_birth_date] @@ -192,7 +192,7 @@ The complete Postgres Operator DAG When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/postgres/example_postgres.py +.. exampleinclude:: /../../providers/tests/system/postgres/example_postgres.py :language: python :start-after: [START postgres_sql_execute_query_operator_howto_guide] :end-before: [END postgres_sql_execute_query_operator_howto_guide] diff --git a/docs/apache-airflow-providers-presto/changelog.rst b/docs/apache-airflow-providers-presto/changelog.rst index e430ef542f538..7cd5d865d7c23 100644 --- a/docs/apache-airflow-providers-presto/changelog.rst +++ b/docs/apache-airflow-providers-presto/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/presto/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/presto/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-presto/index.rst b/docs/apache-airflow-providers-presto/index.rst index 9d814fde8d915..a67acce0f81bc 100644 --- a/docs/apache-airflow-providers-presto/index.rst +++ b/docs/apache-airflow-providers-presto/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/presto/index> + System Tests <_api/tests/system/presto/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-presto/operators/transfer/gcs_to_presto.rst b/docs/apache-airflow-providers-presto/operators/transfer/gcs_to_presto.rst index de717cdaee8d2..d85844956039b 100644 --- a/docs/apache-airflow-providers-presto/operators/transfer/gcs_to_presto.rst +++ b/docs/apache-airflow-providers-presto/operators/transfer/gcs_to_presto.rst @@ -39,7 +39,7 @@ This operator assumes that CSV does not have headers and the data is correspondi pre-existing presto table. Optionally, you can provide schema as tuple/list of strings or as a path to a JSON file in the same bucket as the CSV file. -.. exampleinclude:: /../../tests/system/providers/presto/example_gcs_to_presto.py +.. exampleinclude:: /../../providers/tests/system/presto/example_gcs_to_presto.py :language: python :dedent: 4 :start-after: [START gcs_csv_to_presto_table] diff --git a/docs/apache-airflow-providers-qdrant/changelog.rst b/docs/apache-airflow-providers-qdrant/changelog.rst index 56dd0a8b55a8e..b6079158bc81c 100644 --- a/docs/apache-airflow-providers-qdrant/changelog.rst +++ b/docs/apache-airflow-providers-qdrant/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/qdrant/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/qdrant/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-qdrant/index.rst b/docs/apache-airflow-providers-qdrant/index.rst index 774dbaab014a1..32ce6934c872d 100644 --- a/docs/apache-airflow-providers-qdrant/index.rst +++ b/docs/apache-airflow-providers-qdrant/index.rst @@ -51,7 +51,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/qdrant/index> + System Tests <_api/tests/system/qdrant/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-qdrant/operators/qdrant.rst b/docs/apache-airflow-providers-qdrant/operators/qdrant.rst index 22d82d5958f12..5e7bbda10a645 100644 --- a/docs/apache-airflow-providers-qdrant/operators/qdrant.rst +++ b/docs/apache-airflow-providers-qdrant/operators/qdrant.rst @@ -33,7 +33,7 @@ the original text corresponding to the vectors that could be ingested into the d An example using the operator in this way: -.. exampleinclude:: /../../tests/system/providers/qdrant/example_dag_qdrant.py +.. exampleinclude:: /../../providers/tests/system/qdrant/example_dag_qdrant.py :language: python :dedent: 4 :start-after: [START howto_operator_qdrant_ingest] diff --git a/docs/apache-airflow-providers-redis/changelog.rst b/docs/apache-airflow-providers-redis/changelog.rst index 2fe7da2668e99..fa694a4a9b8d6 100644 --- a/docs/apache-airflow-providers-redis/changelog.rst +++ b/docs/apache-airflow-providers-redis/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/redis/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/redis/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-redis/index.rst b/docs/apache-airflow-providers-redis/index.rst index 1231aa2df3e98..fe9b374f68833 100644 --- a/docs/apache-airflow-providers-redis/index.rst +++ b/docs/apache-airflow-providers-redis/index.rst @@ -49,7 +49,7 @@ :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources @@ -58,7 +58,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/redis/index> + System Tests <_api/tests/system/redis/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-salesforce/changelog.rst b/docs/apache-airflow-providers-salesforce/changelog.rst index 192eeeb3d5ed6..ba7063ba2d814 100644 --- a/docs/apache-airflow-providers-salesforce/changelog.rst +++ b/docs/apache-airflow-providers-salesforce/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/salesforce/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/salesforce/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-salesforce/index.rst b/docs/apache-airflow-providers-salesforce/index.rst index cf4ea7a539bb5..610be66bbc277 100644 --- a/docs/apache-airflow-providers-salesforce/index.rst +++ b/docs/apache-airflow-providers-salesforce/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/salesforce/index> + System Tests <_api/tests/system/salesforce/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-salesforce/operators/bulk.rst b/docs/apache-airflow-providers-salesforce/operators/bulk.rst index 037f819c3c7bd..8e1bdd8415c91 100644 --- a/docs/apache-airflow-providers-salesforce/operators/bulk.rst +++ b/docs/apache-airflow-providers-salesforce/operators/bulk.rst @@ -28,7 +28,7 @@ Using the Operator You can use this operator to access Bulk Insert API: -.. exampleinclude:: /../../tests/system/providers/salesforce/example_bulk.py +.. exampleinclude:: /../../providers/tests/system/salesforce/example_bulk.py :language: python :dedent: 4 :start-after: [START howto_salesforce_bulk_insert_operation] @@ -36,7 +36,7 @@ You can use this operator to access Bulk Insert API: You can use this operator to access Bulk Update API: -.. exampleinclude:: /../../tests/system/providers/salesforce/example_bulk.py +.. exampleinclude:: /../../providers/tests/system/salesforce/example_bulk.py :language: python :dedent: 4 :start-after: [START howto_salesforce_bulk_update_operation] @@ -44,7 +44,7 @@ You can use this operator to access Bulk Update API: You can use this operator to access Bulk Upsert API: -.. exampleinclude:: /../../tests/system/providers/salesforce/example_bulk.py +.. exampleinclude:: /../../providers/tests/system/salesforce/example_bulk.py :language: python :dedent: 4 :start-after: [START howto_salesforce_bulk_upsert_operation] @@ -52,7 +52,7 @@ You can use this operator to access Bulk Upsert API: You can use this operator to access Bulk Delete API: -.. exampleinclude:: /../../tests/system/providers/salesforce/example_bulk.py +.. exampleinclude:: /../../providers/tests/system/salesforce/example_bulk.py :language: python :dedent: 4 :start-after: [START howto_salesforce_bulk_delete_operation] diff --git a/docs/apache-airflow-providers-salesforce/operators/salesforce_apex_rest.rst b/docs/apache-airflow-providers-salesforce/operators/salesforce_apex_rest.rst index 00b4b4c643434..372a4efff538d 100644 --- a/docs/apache-airflow-providers-salesforce/operators/salesforce_apex_rest.rst +++ b/docs/apache-airflow-providers-salesforce/operators/salesforce_apex_rest.rst @@ -30,7 +30,7 @@ You can also use this library to call custom Apex methods: This would call the endpoint ``https://.salesforce.com/services/apexrest/User/Activity`` with ``payload`` as the body content encoded with ``json.dumps`` -.. exampleinclude:: /../../tests/system/providers/salesforce/example_salesforce_apex_rest.py +.. exampleinclude:: /../../providers/tests/system/salesforce/example_salesforce_apex_rest.py :language: python :start-after: [START howto_salesforce_apex_rest_operator] :end-before: [END howto_salesforce_apex_rest_operator] diff --git a/docs/apache-airflow-providers-samba/changelog.rst b/docs/apache-airflow-providers-samba/changelog.rst index e22abbadb41c1..8bcfbb1b1f012 100644 --- a/docs/apache-airflow-providers-samba/changelog.rst +++ b/docs/apache-airflow-providers-samba/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/samba/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/samba/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-samba/index.rst b/docs/apache-airflow-providers-samba/index.rst index a252f74481e84..11ac45f8c1d97 100644 --- a/docs/apache-airflow-providers-samba/index.rst +++ b/docs/apache-airflow-providers-samba/index.rst @@ -57,7 +57,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/samba/index> + System Tests <_api/tests/system/samba/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-samba/transfer/gcs_to_samba.rst b/docs/apache-airflow-providers-samba/transfer/gcs_to_samba.rst index c63f468bd665e..86f6de1926ef3 100644 --- a/docs/apache-airflow-providers-samba/transfer/gcs_to_samba.rst +++ b/docs/apache-airflow-providers-samba/transfer/gcs_to_samba.rst @@ -42,7 +42,7 @@ Copying a single file The following Operator copies a single file. -.. exampleinclude:: /../../tests/system/providers/samba/example_gcs_to_samba.py +.. exampleinclude:: /../../providers/tests/system/samba/example_gcs_to_samba.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_samba_copy_single_file] @@ -55,7 +55,7 @@ To move the file use the ``move_object`` parameter. Once the file is copied to S the original file from the Google Storage is deleted. The ``destination_path`` parameter defines the full path of the file on the Samba server. -.. exampleinclude:: /../../tests/system/providers/samba/example_gcs_to_samba.py +.. exampleinclude:: /../../providers/tests/system/samba/example_gcs_to_samba.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_samba_move_single_file_destination] @@ -67,7 +67,7 @@ Copying a directory Use the ``wildcard`` in ``source_path`` parameter to copy a directory. -.. exampleinclude:: /../../tests/system/providers/samba/example_gcs_to_samba.py +.. exampleinclude:: /../../providers/tests/system/samba/example_gcs_to_samba.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_samba_copy_directory] @@ -79,7 +79,7 @@ Moving specific files Use the ``wildcard`` in ``source_path`` parameter to move the specific files. The ``destination_path`` defines the path that is prefixed to all copied files. -.. exampleinclude:: /../../tests/system/providers/samba/example_gcs_to_samba.py +.. exampleinclude:: /../../providers/tests/system/samba/example_gcs_to_samba.py :language: python :dedent: 4 :start-after: [START howto_operator_gcs_to_samba_move_specific_files] diff --git a/docs/apache-airflow-providers-segment/changelog.rst b/docs/apache-airflow-providers-segment/changelog.rst index 9a9e27c111972..b8eafd739d1fa 100644 --- a/docs/apache-airflow-providers-segment/changelog.rst +++ b/docs/apache-airflow-providers-segment/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/segment/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/segment/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-sendgrid/changelog.rst b/docs/apache-airflow-providers-sendgrid/changelog.rst index 43ddc029119e1..913b7477b13a2 100644 --- a/docs/apache-airflow-providers-sendgrid/changelog.rst +++ b/docs/apache-airflow-providers-sendgrid/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/sendgrid/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/sendgrid/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-sftp/changelog.rst b/docs/apache-airflow-providers-sftp/changelog.rst index 2ed184d07da7c..3915d213a711c 100644 --- a/docs/apache-airflow-providers-sftp/changelog.rst +++ b/docs/apache-airflow-providers-sftp/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/sftp/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/sftp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-sftp/index.rst b/docs/apache-airflow-providers-sftp/index.rst index 9acb9773fadc1..a69f41a678800 100644 --- a/docs/apache-airflow-providers-sftp/index.rst +++ b/docs/apache-airflow-providers-sftp/index.rst @@ -51,7 +51,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/sftp/index> + System Tests <_api/tests/system/sftp/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-sftp/sensors/sftp_sensor.rst b/docs/apache-airflow-providers-sftp/sensors/sftp_sensor.rst index b99bc36672d36..32f2a20696a73 100644 --- a/docs/apache-airflow-providers-sftp/sensors/sftp_sensor.rst +++ b/docs/apache-airflow-providers-sftp/sensors/sftp_sensor.rst @@ -21,7 +21,7 @@ SFTP Sensor Looks for either a specific file or files with a specific pattern in a server using SFTP protocol. To get more information about this sensor visit :class:`~airflow.providers.sftp.sensors.sftp.SFTPSensor` -.. exampleinclude:: /../../tests/system/providers/sftp/example_sftp_sensor.py +.. exampleinclude:: /../../providers/tests/system/sftp/example_sftp_sensor.py :language: python :dedent: 4 :start-after: [START howto_operator_sftp_sensor] @@ -39,7 +39,7 @@ op_kwargs (optional) Whatever returned by the python callable is put into XCom. -.. exampleinclude:: /../../tests/system/providers/sftp/example_sftp_sensor.py +.. exampleinclude:: /../../providers/tests/system/sftp/example_sftp_sensor.py :language: python :dedent: 4 :start-after: [START howto_operator_sftp_sensor_decorator] @@ -47,7 +47,7 @@ Whatever returned by the python callable is put into XCom. Checks for the existence of a file on an SFTP server in the deferrable mode: -.. exampleinclude:: /../../tests/system/providers/sftp/example_sftp_sensor.py +.. exampleinclude:: /../../providers/tests/system/sftp/example_sftp_sensor.py :language: python :dedent: 4 :start-after: [START howto_sensor_sftp_deferrable] diff --git a/docs/apache-airflow-providers-singularity/changelog.rst b/docs/apache-airflow-providers-singularity/changelog.rst index 1da16678d8e1b..eaa3226835fd6 100644 --- a/docs/apache-airflow-providers-singularity/changelog.rst +++ b/docs/apache-airflow-providers-singularity/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/singularity/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/singularity/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-singularity/index.rst b/docs/apache-airflow-providers-singularity/index.rst index 3dd7c559bc833..656a7a64a27a6 100644 --- a/docs/apache-airflow-providers-singularity/index.rst +++ b/docs/apache-airflow-providers-singularity/index.rst @@ -41,14 +41,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/singularity/index> + System Tests <_api/tests/system/singularity/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-slack/changelog.rst b/docs/apache-airflow-providers-slack/changelog.rst index ec3a2f20d6577..75fb0c94dfe75 100644 --- a/docs/apache-airflow-providers-slack/changelog.rst +++ b/docs/apache-airflow-providers-slack/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/slack/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/slack/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-slack/index.rst b/docs/apache-airflow-providers-slack/index.rst index f9e6e02e9fd19..7fca57690168d 100644 --- a/docs/apache-airflow-providers-slack/index.rst +++ b/docs/apache-airflow-providers-slack/index.rst @@ -34,7 +34,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/slack/index> + System Tests <_api/tests/system/slack/index> .. toctree:: :hidden: @@ -51,7 +51,7 @@ :caption: References Python API <_api/airflow/providers/slack/index> - Example DAGs + Example DAGs .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-slack/operators/slack_api.rst b/docs/apache-airflow-providers-slack/operators/slack_api.rst index 6fafa46f79418..7ab98f6b7eb81 100644 --- a/docs/apache-airflow-providers-slack/operators/slack_api.rst +++ b/docs/apache-airflow-providers-slack/operators/slack_api.rst @@ -34,7 +34,7 @@ Using the Operator You could send simple text message -.. exampleinclude:: /../../tests/system/providers/slack/example_slack.py +.. exampleinclude:: /../../providers/tests/system/slack/example_slack.py :language: python :dedent: 4 :start-after: [START slack_api_post_operator_text_howto_guide] @@ -43,7 +43,7 @@ You could send simple text message Or you could use `Block Kit `_ for create app layouts -.. exampleinclude:: /../../tests/system/providers/slack/example_slack.py +.. exampleinclude:: /../../providers/tests/system/slack/example_slack.py :language: python :dedent: 4 :start-after: [START slack_api_post_operator_blocks_howto_guide] @@ -81,7 +81,7 @@ Using the Operator You could send file attachment by specifying file path -.. exampleinclude:: /../../tests/system/providers/slack/example_slack.py +.. exampleinclude:: /../../providers/tests/system/slack/example_slack.py :language: python :start-after: [START slack_api_file_operator_howto_guide] :end-before: [END slack_api_file_operator_howto_guide] @@ -89,7 +89,7 @@ You could send file attachment by specifying file path Or by directly providing file contents -.. exampleinclude:: /../../tests/system/providers/slack/example_slack.py +.. exampleinclude:: /../../providers/tests/system/slack/example_slack.py :language: python :start-after: [START slack_api_file_operator_content_howto_guide] :end-before: [END slack_api_file_operator_content_howto_guide] diff --git a/docs/apache-airflow-providers-slack/operators/slack_webhook.rst b/docs/apache-airflow-providers-slack/operators/slack_webhook.rst index aded2a76bb7c6..0b599312ba61c 100644 --- a/docs/apache-airflow-providers-slack/operators/slack_webhook.rst +++ b/docs/apache-airflow-providers-slack/operators/slack_webhook.rst @@ -30,7 +30,7 @@ Using the Operator You could send simple text message -.. exampleinclude:: /../../tests/system/providers/slack/example_slack_webhook.py +.. exampleinclude:: /../../providers/tests/system/slack/example_slack_webhook.py :language: python :dedent: 4 :start-after: [START slack_webhook_operator_text_howto_guide] @@ -39,7 +39,7 @@ You could send simple text message Or you could use `Block Kit `_ for create app layouts -.. exampleinclude:: /../../tests/system/providers/slack/example_slack_webhook.py +.. exampleinclude:: /../../providers/tests/system/slack/example_slack_webhook.py :language: python :dedent: 4 :start-after: [START slack_webhook_operator_blocks_howto_guide] diff --git a/docs/apache-airflow-providers-slack/operators/sql_to_slack.rst b/docs/apache-airflow-providers-slack/operators/sql_to_slack.rst index 3182fd7e0b785..5da6a2bd593d5 100644 --- a/docs/apache-airflow-providers-slack/operators/sql_to_slack.rst +++ b/docs/apache-airflow-providers-slack/operators/sql_to_slack.rst @@ -51,7 +51,7 @@ This operator will execute a custom query in the provided SQL connection and pub An example usage of the SqlToSlackApiFileOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/slack/example_sql_to_slack.py +.. exampleinclude:: /../../providers/tests/system/slack/example_sql_to_slack.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_to_slack_api_file] diff --git a/docs/apache-airflow-providers-slack/operators/sql_to_slack_webhook.rst b/docs/apache-airflow-providers-slack/operators/sql_to_slack_webhook.rst index 3ad72e2a9ad77..aa46fe7b03509 100644 --- a/docs/apache-airflow-providers-slack/operators/sql_to_slack_webhook.rst +++ b/docs/apache-airflow-providers-slack/operators/sql_to_slack_webhook.rst @@ -31,7 +31,7 @@ and contain the resulting dataset (e.g. ASCII formatted dataframe). An example usage of the SqlToSlackWebhookOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/slack/example_sql_to_slack_webhook.py +.. exampleinclude:: /../../providers/tests/system/slack/example_sql_to_slack_webhook.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_to_slack_webhook] diff --git a/docs/apache-airflow-providers-smtp/changelog.rst b/docs/apache-airflow-providers-smtp/changelog.rst index 105ae30115633..3ada8a5cf420d 100644 --- a/docs/apache-airflow-providers-smtp/changelog.rst +++ b/docs/apache-airflow-providers-smtp/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/smtp/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/smtp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-snowflake/changelog.rst b/docs/apache-airflow-providers-snowflake/changelog.rst index 6ccd3e61dc37a..932a260df7188 100644 --- a/docs/apache-airflow-providers-snowflake/changelog.rst +++ b/docs/apache-airflow-providers-snowflake/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/snowflake/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/snowflake/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-snowflake/decorators/snowpark.rst b/docs/apache-airflow-providers-snowflake/decorators/snowpark.rst index 09be01e3efc8f..947cfec52b5e4 100644 --- a/docs/apache-airflow-providers-snowflake/decorators/snowpark.rst +++ b/docs/apache-airflow-providers-snowflake/decorators/snowpark.rst @@ -51,7 +51,7 @@ Use the ``snowflake_conn_id`` argument to specify connection used. If not specif An example usage of the ``@task.snowpark`` is as follows: -.. exampleinclude:: /../../tests/system/providers/snowflake/example_snowpark_decorator.py +.. exampleinclude:: /../../providers/tests/system/snowflake/example_snowpark_decorator.py :language: python :start-after: [START howto_decorator_snowpark] :end-before: [END howto_decorator_snowpark] diff --git a/docs/apache-airflow-providers-snowflake/index.rst b/docs/apache-airflow-providers-snowflake/index.rst index c9746f7806943..c9f3b8f8872ef 100644 --- a/docs/apache-airflow-providers-snowflake/index.rst +++ b/docs/apache-airflow-providers-snowflake/index.rst @@ -50,14 +50,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/snowflake/index> + System Tests <_api/tests/system/snowflake/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-snowflake/operators/copy_into_snowflake.rst b/docs/apache-airflow-providers-snowflake/operators/copy_into_snowflake.rst index 49fca623f2f9e..30236121abbb3 100644 --- a/docs/apache-airflow-providers-snowflake/operators/copy_into_snowflake.rst +++ b/docs/apache-airflow-providers-snowflake/operators/copy_into_snowflake.rst @@ -43,7 +43,7 @@ a file format (see `docs + System Tests <_api/tests/system/sqlite/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-sqlite/operators.rst b/docs/apache-airflow-providers-sqlite/operators.rst index 472ec71452499..5b113162ae4c3 100644 --- a/docs/apache-airflow-providers-sqlite/operators.rst +++ b/docs/apache-airflow-providers-sqlite/operators.rst @@ -45,14 +45,14 @@ the connection metadata is structured as follows: An example usage of the SQLExecuteQueryOperator to connect to Sqlite is as follows: -.. exampleinclude:: /../../tests/system/providers/sqlite/example_sqlite.py +.. exampleinclude:: /../../providers/tests/system/sqlite/example_sqlite.py :language: python :start-after: [START howto_operator_sqlite] :end-before: [END howto_operator_sqlite] Furthermore, you can use an external file to execute the SQL commands. Script folder must be at the same level as DAG.py file. -.. exampleinclude:: /../../tests/system/providers/sqlite/example_sqlite.py +.. exampleinclude:: /../../providers/tests/system/sqlite/example_sqlite.py :language: python :start-after: [START howto_operator_sqlite_external_file] :end-before: [END howto_operator_sqlite_external_file] diff --git a/docs/apache-airflow-providers-ssh/changelog.rst b/docs/apache-airflow-providers-ssh/changelog.rst index b2212a396cc68..33bcb1f1b4086 100644 --- a/docs/apache-airflow-providers-ssh/changelog.rst +++ b/docs/apache-airflow-providers-ssh/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/ssh/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/ssh/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-standard/changelog.rst b/docs/apache-airflow-providers-standard/changelog.rst index 3d9d5b25f5540..2e1b328f5e5e2 100644 --- a/docs/apache-airflow-providers-standard/changelog.rst +++ b/docs/apache-airflow-providers-standard/changelog.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../../airflow/providers/standard/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/standard/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-tableau/changelog.rst b/docs/apache-airflow-providers-tableau/changelog.rst index 527d56d2c4242..7716b6ab1bf64 100644 --- a/docs/apache-airflow-providers-tableau/changelog.rst +++ b/docs/apache-airflow-providers-tableau/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/tableau/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/tableau/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-tableau/index.rst b/docs/apache-airflow-providers-tableau/index.rst index 0a4789db73124..e0f140e0fae4e 100644 --- a/docs/apache-airflow-providers-tableau/index.rst +++ b/docs/apache-airflow-providers-tableau/index.rst @@ -43,14 +43,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/tableau/index> + System Tests <_api/tests/system/tableau/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-tableau/operators.rst b/docs/apache-airflow-providers-tableau/operators.rst index ba250e1be41fe..7452d7899f4dc 100644 --- a/docs/apache-airflow-providers-tableau/operators.rst +++ b/docs/apache-airflow-providers-tableau/operators.rst @@ -68,7 +68,7 @@ Using the Operator An example usage of the TableauOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/tableau/example_tableau.py +.. exampleinclude:: /../../providers/tests/system/tableau/example_tableau.py :language: python :start-after: [START howto_operator_tableau] :end-before: [END howto_operator_tableau] diff --git a/docs/apache-airflow-providers-telegram/changelog.rst b/docs/apache-airflow-providers-telegram/changelog.rst index 704e8de2f6f26..073d54c1b5e83 100644 --- a/docs/apache-airflow-providers-telegram/changelog.rst +++ b/docs/apache-airflow-providers-telegram/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/telegram/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/telegram/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-telegram/index.rst b/docs/apache-airflow-providers-telegram/index.rst index 5c38d229986b0..58efd772bb28f 100644 --- a/docs/apache-airflow-providers-telegram/index.rst +++ b/docs/apache-airflow-providers-telegram/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/telegram/index> + System Tests <_api/tests/system/telegram/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-telegram/operators.rst b/docs/apache-airflow-providers-telegram/operators.rst index 0f3ad86b19d09..de693f54001e4 100644 --- a/docs/apache-airflow-providers-telegram/operators.rst +++ b/docs/apache-airflow-providers-telegram/operators.rst @@ -48,7 +48,7 @@ the connection metadata is structured as follows: An example usage of the TelegramOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/telegram/example_telegram.py +.. exampleinclude:: /../../providers/tests/system/telegram/example_telegram.py :language: python :start-after: [START howto_operator_telegram] :end-before: [END howto_operator_telegram] diff --git a/docs/apache-airflow-providers-teradata/changelog.rst b/docs/apache-airflow-providers-teradata/changelog.rst index 90c80663b40d8..577e6333e2053 100644 --- a/docs/apache-airflow-providers-teradata/changelog.rst +++ b/docs/apache-airflow-providers-teradata/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/teradata/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/teradata/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-teradata/index.rst b/docs/apache-airflow-providers-teradata/index.rst index 44310a10b1e70..ecf00232484d1 100644 --- a/docs/apache-airflow-providers-teradata/index.rst +++ b/docs/apache-airflow-providers-teradata/index.rst @@ -49,7 +49,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/teradata/index> + System Tests <_api/tests/system/teradata/index> System Tests Dashboard .. toctree:: diff --git a/docs/apache-airflow-providers-teradata/operators/azure_blob_to_teradata.rst b/docs/apache-airflow-providers-teradata/operators/azure_blob_to_teradata.rst index 194eabd0cd001..4cc3da46a902f 100644 --- a/docs/apache-airflow-providers-teradata/operators/azure_blob_to_teradata.rst +++ b/docs/apache-airflow-providers-teradata/operators/azure_blob_to_teradata.rst @@ -60,7 +60,7 @@ Transferring data from public Azure Blob Storage to Teradata An example usage of the AzureBlobStorageToTeradataOperator to transfer CSV data format from public Azure Blob Storage to teradata table is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py :language: python :start-after: [START azure_blob__to_teradata_transfer_operator_howto_guide_transfer_data_public_blob_to_teradata_csv] :end-before: [END azure_blob__to_teradata_transfer_operator_howto_guide_transfer_data_public_blob_to_teradata_csv] @@ -71,7 +71,7 @@ Transferring data from private Azure Blob Storage to Teradata with AWS connectio An example usage of the AzureBlobStorageToTeradataOperator to transfer CSV data format from private S3 object store to teradata with AWS credentials defined as AWS connection: -.. exampleinclude:: /../../tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py :language: python :start-after: [START azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_access_blob_to_teradata_csv] :end-before: [END azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_access_blob_to_teradata_csv] @@ -85,7 +85,7 @@ database object should exists in Teradata database to use it in transferring dat An example usage of the AzureBlobStorageToTeradataOperator to transfer CSV data format from private S3 object store to teradata with Authorization database object defined in Teradata. -.. exampleinclude:: /../../tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py :language: python :start-after: [START azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_authorization_blob_to_teradata_csv] :end-before: [END azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_authorization_blob_to_teradata_csv] @@ -96,7 +96,7 @@ Transferring data in CSV format from Azure Blob Storage to Teradata An example usage of the AzureBlobStorageToTeradataOperator to transfer CSV data format from Azure Blob Storage to teradata table is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py :language: python :start-after: [START azure_blob__to_teradata_transfer_operator_howto_guide_transfer_data_public_blob_to_teradata_csv] :end-before: [END azure_blob__to_teradata_transfer_operator_howto_guide_transfer_data_public_blob_to_teradata_csv] @@ -107,7 +107,7 @@ Transferring data in JSON format from Azure Blob Storage to Teradata An example usage of the AzureBlobStorageToTeradataOperator to transfer JSON data format from Azure Blob Storage to teradata table is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py :language: python :start-after: [START azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_blob_to_teradata_json] :end-before: [END azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_blob_to_teradata_json] @@ -118,7 +118,7 @@ Transferring data in PARQUET format from Azure Blob Storage to Teradata An example usage of the AzureBlobStorageToTeradataOperator to transfer PARQUET data format from Azure Blob Storage to teradata table is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py :language: python :start-after: [START azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_blob_to_teradata_parquet] :end-before: [END azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_blob_to_teradata_parquet] @@ -128,7 +128,7 @@ The complete ``AzureBlobStorageToTeradataOperator`` Operator DAG When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py :language: python :start-after: [START azure_blob_to_teradata_transfer_operator_howto_guide] :end-before: [END azure_blob_to_teradata_transfer_operator_howto_guide] diff --git a/docs/apache-airflow-providers-teradata/operators/compute_cluster.rst b/docs/apache-airflow-providers-teradata/operators/compute_cluster.rst index ceaf27ee74676..507aac5c90ac6 100644 --- a/docs/apache-airflow-providers-teradata/operators/compute_cluster.rst +++ b/docs/apache-airflow-providers-teradata/operators/compute_cluster.rst @@ -33,7 +33,7 @@ to provision the new Compute Cluster in Teradata Vantage Cloud Lake. An example usage of the TeradataComputeClusterProvisionOperator to provision the new Compute Cluster in Teradata Vantage Cloud Lake is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_compute_cluster.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_compute_cluster.py :language: python :start-after: [START teradata_vantage_lake_compute_cluster_provision_howto_guide] :end-before: [END teradata_vantage_lake_compute_cluster_provision_howto_guide] @@ -56,7 +56,7 @@ to decommission the specified Teradata Vantage Cloud Lake Compute Cluster. An example usage of the TeradataComputeClusterDecommissionOperator to decommission the specified Teradata Vantage Cloud Lake Compute Cluster is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_compute_cluster.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_compute_cluster.py :language: python :start-after: [START teradata_vantage_lake_compute_cluster_decommission_howto_guide] :end-before: [END teradata_vantage_lake_compute_cluster_decommission_howto_guide] @@ -79,7 +79,7 @@ to start the specified Compute Cluster in Teradata Vantage Cloud Lake. An example usage of the TeradataComputeClusterSuspendOperator to start the specified Compute Cluster in Teradata Vantage Cloud Lake is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_compute_cluster.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_compute_cluster.py :language: python :start-after: [START teradata_vantage_lake_compute_cluster_resume_howto_guide] :end-before: [END teradata_vantage_lake_compute_cluster_resume_howto_guide] @@ -101,7 +101,7 @@ to suspend the specified Compute Cluster in Teradata Vantage Cloud Lake. An example usage of the TeradataComputeClusterSuspendOperator to suspend the specified Compute Cluster in Teradata Vantage Cloud Lake is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_compute_cluster.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_compute_cluster.py :language: python :start-after: [START teradata_vantage_lake_compute_cluster_suspend_howto_guide] :end-before: [END teradata_vantage_lake_compute_cluster_suspend_howto_guide] diff --git a/docs/apache-airflow-providers-teradata/operators/s3_to_teradata.rst b/docs/apache-airflow-providers-teradata/operators/s3_to_teradata.rst index da52e2841bfa3..e24887126a2a5 100644 --- a/docs/apache-airflow-providers-teradata/operators/s3_to_teradata.rst +++ b/docs/apache-airflow-providers-teradata/operators/s3_to_teradata.rst @@ -45,7 +45,7 @@ Transferring data in CSV format from S3 to Teradata An example usage of the S3ToTeradataOperator to transfer CSV data format from S3 to teradata table is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_s3_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_s3_to_teradata_transfer.py :language: python :start-after: [START s3_to_teradata_transfer_operator_howto_guide_transfer_data_public_s3_to_teradata_csv] :end-before: [END s3_to_teradata_transfer_operator_howto_guide_transfer_data_public_s3_to_teradata_csv] @@ -55,7 +55,7 @@ Transferring data in JSON format from S3 to Teradata An example usage of the S3ToTeradataOperator to transfer JSON data format from S3 to teradata table is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_s3_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_s3_to_teradata_transfer.py :language: python :start-after: [START s3_to_teradata_transfer_operator_howto_guide_transfer_data_s3_to_teradata_json] :end-before: [END s3_to_teradata_transfer_operator_howto_guide_transfer_data_s3_to_teradata_json] @@ -65,7 +65,7 @@ Transferring data in PARQUET format from S3 to Teradata An example usage of the S3ToTeradataOperator to transfer PARQUET data format from S3 to teradata table is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_s3_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_s3_to_teradata_transfer.py :language: python :start-after: [START s3_to_teradata_transfer_operator_howto_guide_transfer_data_s3_to_teradata_parquet] :end-before: [END s3_to_teradata_transfer_operator_howto_guide_transfer_data_s3_to_teradata_parquet] @@ -75,7 +75,7 @@ The complete ``S3ToTeradataOperator`` Operator DAG When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/teradata/example_s3_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_s3_to_teradata_transfer.py :language: python :start-after: [START s3_to_teradata_transfer_operator_howto_guide] :end-before: [END s3_to_teradata_transfer_operator_howto_guide] diff --git a/docs/apache-airflow-providers-teradata/operators/teradata.rst b/docs/apache-airflow-providers-teradata/operators/teradata.rst index 6fd7d371a7b09..78e2e1b12e800 100644 --- a/docs/apache-airflow-providers-teradata/operators/teradata.rst +++ b/docs/apache-airflow-providers-teradata/operators/teradata.rst @@ -33,7 +33,7 @@ Creating a Teradata database table An example usage of the TeradataOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :dedent: 4 :start-after: [START teradata_operator_howto_guide_create_table] @@ -42,7 +42,7 @@ An example usage of the TeradataOperator is as follows: You can also use an external file to execute the SQL commands. External file must be at the same level as DAG.py file. This way you can easily maintain the SQL queries separated from the code. -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :start-after: [START teradata_operator_howto_guide_create_table_from_external_file] :end-before: [END teradata_operator_howto_guide_create_table_from_external_file] @@ -63,7 +63,7 @@ Inserting data into a Teradata database table --------------------------------------------- We can then create a TeradataOperator task that populate the ``Users`` table. -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :start-after: [START teradata_operator_howto_guide_populate_table] :end-before: [END teradata_operator_howto_guide_populate_table] @@ -74,7 +74,7 @@ Fetching records from your Teradata database table Fetching records from your Teradata database table can be as simple as: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :start-after: [START teradata_operator_howto_guide_get_all_countries] :end-before: [END teradata_operator_howto_guide_get_all_countries] @@ -88,7 +88,7 @@ SQL requests during runtime. To find the countries in Asian continent: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :start-after: [START teradata_operator_howto_guide_params_passing_get_query] :end-before: [END teradata_operator_howto_guide_params_passing_get_query] @@ -99,7 +99,7 @@ Dropping a Teradata database table We can then create a TeradataOperator task that drops the ``Users`` table. -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :start-after: [START teradata_operator_howto_guide_drop_users_table] :end-before: [END teradata_operator_howto_guide_drop_users_table] @@ -109,7 +109,7 @@ The complete Teradata Operator DAG When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :start-after: [START teradata_operator_howto_guide] :end-before: [END teradata_operator_howto_guide] @@ -152,21 +152,21 @@ This stored procedure can be invoked using One approach involves passing parameters positionally as a list, with output parameters specified as Python data types: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_call_sp.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_call_sp.py :language: python :start-after: [START howto_call_teradata_stored_procedure_operator_with_types] :end-before: [END howto_call_teradata_stored_procedure_operator_with_types] Alternatively, parameters can be passed positionally as a list, with output parameters designated as placeholders: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_call_sp.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_call_sp.py :language: python :start-after: [START howto_call_teradata_stored_procedure_operator_with_place_holder] :end-before: [END howto_call_teradata_stored_procedure_operator_with_place_holder] Another method entails passing parameters positionally as a dictionary: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_call_sp.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_call_sp.py :language: python :start-after: [START howto_call_teradata_stored_procedure_operator_with_dict_input] :end-before: [END howto_call_teradata_stored_procedure_operator_with_dict_input] @@ -186,7 +186,7 @@ This stored procedure yields a singular timestamp argument, out_timestamp, and i :class:`~airflow.providers.teradata.operators.teradata.TeradataStoredProcedureOperator` with parameters passed positionally as a list: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_call_sp.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_call_sp.py :language: python :start-after: [START howto_call_teradata_stored_procedure_operator_timestamp] :end-before: [END howto_call_teradata_stored_procedure_operator_timestamp] @@ -213,7 +213,7 @@ This stored procedure can be invoked using :class:`~airflow.providers.teradata.operators.teradata.TeradataStoredProcedureOperator` with parameters passed positionally as a list: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_call_sp.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_call_sp.py :language: python :start-after: [START howto_teradata_stored_procedure_operator_with_in_out_dynamic_result] :end-before: [END howto_teradata_stored_procedure_operator_with_in_out_dynamic_result] @@ -223,7 +223,7 @@ The complete TeradataStoredProcedureOperator DAG When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_call_sp.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_call_sp.py :language: python :start-after: [START howto_teradata_operator_for_sp] :end-before: [END howto_teradata_operator_for_sp] diff --git a/docs/apache-airflow-providers-teradata/operators/teradata_to_teradata.rst b/docs/apache-airflow-providers-teradata/operators/teradata_to_teradata.rst index 6c551427f0737..ec2a414ff7d0a 100644 --- a/docs/apache-airflow-providers-teradata/operators/teradata_to_teradata.rst +++ b/docs/apache-airflow-providers-teradata/operators/teradata_to_teradata.rst @@ -32,7 +32,7 @@ To transfer data between two Teradata instances, use the An example usage of the TeradataToTeradataOperator is as follows: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata_to_teradata_transfer.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata_to_teradata_transfer.py :language: python :start-after: [START teradata_to_teradata_transfer_operator_howto_guide_transfer_data] :end-before: [END teradata_to_teradata_transfer_operator_howto_guide_transfer_data] @@ -42,7 +42,7 @@ The complete TeradataToTeradata Transfer Operator DAG When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/teradata/example_teradata.py +.. exampleinclude:: /../../providers/tests/system/teradata/example_teradata.py :language: python :start-after: [START teradata_operator_howto_guide] :end-before: [END teradata_operator_howto_guide] diff --git a/docs/apache-airflow-providers-trino/changelog.rst b/docs/apache-airflow-providers-trino/changelog.rst index e29b41a315293..c3d1995dd1ea6 100644 --- a/docs/apache-airflow-providers-trino/changelog.rst +++ b/docs/apache-airflow-providers-trino/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/trino/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/trino/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-trino/index.rst b/docs/apache-airflow-providers-trino/index.rst index 9a17b5907e5db..70a8d85c9a2ea 100644 --- a/docs/apache-airflow-providers-trino/index.rst +++ b/docs/apache-airflow-providers-trino/index.rst @@ -50,14 +50,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/trino/index> + System Tests <_api/tests/system/trino/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-trino/operators/transfer/gcs_to_trino.rst b/docs/apache-airflow-providers-trino/operators/transfer/gcs_to_trino.rst index 39224b3ca3d05..4d1cbca9d14d2 100644 --- a/docs/apache-airflow-providers-trino/operators/transfer/gcs_to_trino.rst +++ b/docs/apache-airflow-providers-trino/operators/transfer/gcs_to_trino.rst @@ -39,7 +39,7 @@ This operator assumes that CSV does not have headers and the data is correspondi pre-existing presto table. Optionally, you can provide schema as tuple/list of strings or as a path to a JSON file in the same bucket as the CSV file. -.. exampleinclude:: /../../tests/system/providers/trino/example_gcs_to_trino.py +.. exampleinclude:: /../../providers/tests/system/trino/example_gcs_to_trino.py :language: python :dedent: 4 :start-after: [START gcs_csv_to_trino_table] diff --git a/docs/apache-airflow-providers-trino/operators/trino.rst b/docs/apache-airflow-providers-trino/operators/trino.rst index d0e901ebed9fd..dc076c82c7c2d 100644 --- a/docs/apache-airflow-providers-trino/operators/trino.rst +++ b/docs/apache-airflow-providers-trino/operators/trino.rst @@ -34,7 +34,7 @@ Use the ``trino_conn_id`` argument to connect to your Trino instance An example usage of the SQLExecuteQueryOperator to connect to Trino is as follows: -.. exampleinclude:: /../../tests/system/providers/trino/example_trino.py +.. exampleinclude:: /../../providers/tests/system/trino/example_trino.py :language: python :start-after: [START howto_operator_trino] :end-before: [END howto_operator_trino] diff --git a/docs/apache-airflow-providers-vertica/changelog.rst b/docs/apache-airflow-providers-vertica/changelog.rst index 1f8dd4aa880e2..2c77a34914de2 100644 --- a/docs/apache-airflow-providers-vertica/changelog.rst +++ b/docs/apache-airflow-providers-vertica/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/vertica/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/vertica/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-weaviate/changelog.rst b/docs/apache-airflow-providers-weaviate/changelog.rst index c2fc65bc5ceb4..dd6e11ef6876d 100644 --- a/docs/apache-airflow-providers-weaviate/changelog.rst +++ b/docs/apache-airflow-providers-weaviate/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/weaviate/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/weaviate/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-weaviate/index.rst b/docs/apache-airflow-providers-weaviate/index.rst index c2a4965414fe3..fd7998daac395 100644 --- a/docs/apache-airflow-providers-weaviate/index.rst +++ b/docs/apache-airflow-providers-weaviate/index.rst @@ -60,7 +60,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/weaviate/index> + System Tests <_api/tests/system/weaviate/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/docs/apache-airflow-providers-weaviate/operators/weaviate.rst b/docs/apache-airflow-providers-weaviate/operators/weaviate.rst index 5ec262ab7a2d3..ae9fcdba58eed 100644 --- a/docs/apache-airflow-providers-weaviate/operators/weaviate.rst +++ b/docs/apache-airflow-providers-weaviate/operators/weaviate.rst @@ -33,28 +33,28 @@ connect to your account. An example using the operator to ingest data with custom vectors retrieved from XCOM: -.. exampleinclude:: /../../tests/system/providers/weaviate/example_weaviate_operator.py +.. exampleinclude:: /../../providers/tests/system/weaviate/example_weaviate_operator.py :language: python :start-after: [START howto_operator_weaviate_embedding_and_ingest_xcom_data_with_vectors] :end-before: [END howto_operator_weaviate_embedding_and_ingest_xcom_data_with_vectors] An example using the operator to ingest data with custom vectors retrieved from a python callable: -.. exampleinclude:: /../../tests/system/providers/weaviate/example_weaviate_operator.py +.. exampleinclude:: /../../providers/tests/system/weaviate/example_weaviate_operator.py :language: python :start-after: [START howto_operator_weaviate_embedding_and_ingest_callable_data_with_vectors] :end-before: [END howto_operator_weaviate_embedding_and_ingest_callable_data_with_vectors] An example using the operator to ingest data without vectors retrieved from XCOM for which the operator would generate embedding vectors: -.. exampleinclude:: /../../tests/system/providers/weaviate/example_weaviate_operator.py +.. exampleinclude:: /../../providers/tests/system/weaviate/example_weaviate_operator.py :language: python :start-after: [START howto_operator_weaviate_ingest_xcom_data_without_vectors] :end-before: [END howto_operator_weaviate_ingest_xcom_data_without_vectors] An example using the operator to ingest data without vectors retrieved from a python callable for which the operator would generate embedding vectors: -.. exampleinclude:: /../../tests/system/providers/weaviate/example_weaviate_operator.py +.. exampleinclude:: /../../providers/tests/system/weaviate/example_weaviate_operator.py :language: python :start-after: [START howto_operator_weaviate_ingest_callable_data_without_vectors] :end-before: [END howto_operator_weaviate_ingest_callable_data_without_vectors] diff --git a/docs/apache-airflow-providers-yandex/changelog.rst b/docs/apache-airflow-providers-yandex/changelog.rst index 066c3c5be0be2..9bcad616eb83d 100644 --- a/docs/apache-airflow-providers-yandex/changelog.rst +++ b/docs/apache-airflow-providers-yandex/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/yandex/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/yandex/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-yandex/index.rst b/docs/apache-airflow-providers-yandex/index.rst index 03495bcef4611..08f3953aefeb4 100644 --- a/docs/apache-airflow-providers-yandex/index.rst +++ b/docs/apache-airflow-providers-yandex/index.rst @@ -51,14 +51,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/yandex/index> + System Tests <_api/tests/system/yandex/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-yandex/operators/dataproc.rst b/docs/apache-airflow-providers-yandex/operators/dataproc.rst index 2bb08d859de4f..03dfd3acae817 100644 --- a/docs/apache-airflow-providers-yandex/operators/dataproc.rst +++ b/docs/apache-airflow-providers-yandex/operators/dataproc.rst @@ -34,4 +34,4 @@ that can be integrated with Apache Hadoop and other storage systems. Using the operators ^^^^^^^^^^^^^^^^^^^ To learn how to use Data Proc operators, -see `example DAGs `_. +see `example DAGs `_. diff --git a/docs/apache-airflow-providers-yandex/operators/yq.rst b/docs/apache-airflow-providers-yandex/operators/yq.rst index 78bdb733ee1ff..23bd4ac336160 100644 --- a/docs/apache-airflow-providers-yandex/operators/yq.rst +++ b/docs/apache-airflow-providers-yandex/operators/yq.rst @@ -25,4 +25,4 @@ Yandex Query Operators Using the operators ^^^^^^^^^^^^^^^^^^^ To learn how to use Yandex Query operator, -see `example DAG `__. +see `example DAG `__. diff --git a/docs/apache-airflow-providers-ydb/changelog.rst b/docs/apache-airflow-providers-ydb/changelog.rst index 801c69978c6b6..e77ade1c93725 100644 --- a/docs/apache-airflow-providers-ydb/changelog.rst +++ b/docs/apache-airflow-providers-ydb/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/ydb/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/ydb/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-ydb/index.rst b/docs/apache-airflow-providers-ydb/index.rst index 4dff0e421f9bd..30b8e90d97531 100644 --- a/docs/apache-airflow-providers-ydb/index.rst +++ b/docs/apache-airflow-providers-ydb/index.rst @@ -49,14 +49,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/ydb/index> + System Tests <_api/tests/system/ydb/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-ydb/operators/ydb_operator_howto_guide.rst b/docs/apache-airflow-providers-ydb/operators/ydb_operator_howto_guide.rst index 894be8101d5b6..9416894e53324 100644 --- a/docs/apache-airflow-providers-ydb/operators/ydb_operator_howto_guide.rst +++ b/docs/apache-airflow-providers-ydb/operators/ydb_operator_howto_guide.rst @@ -50,7 +50,7 @@ Creating an YDB table The code snippets below are based on Airflow-2.0 -.. exampleinclude:: /../../tests/system/providers/ydb/example_ydb.py +.. exampleinclude:: /../../providers/tests/system/ydb/example_ydb.py :language: python :start-after: [START ydb_operator_howto_guide] :end-before: [END ydb_operator_howto_guide_create_pet_table] @@ -187,7 +187,7 @@ The complete YDB Operator DAG When we put everything together, our DAG should look like this: -.. exampleinclude:: /../../tests/system/providers/ydb/example_ydb.py +.. exampleinclude:: /../../providers/tests/system/ydb/example_ydb.py :language: python :start-after: [START ydb_operator_howto_guide] :end-before: [END ydb_operator_howto_guide] diff --git a/docs/apache-airflow-providers-zendesk/changelog.rst b/docs/apache-airflow-providers-zendesk/changelog.rst index 3be2afdb2306b..eacd3ee51d1bd 100644 --- a/docs/apache-airflow-providers-zendesk/changelog.rst +++ b/docs/apache-airflow-providers-zendesk/changelog.rst @@ -22,4 +22,4 @@ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -.. include:: ../../airflow/providers/zendesk/CHANGELOG.rst +.. include:: ../../providers/src/airflow/providers/zendesk/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-zendesk/index.rst b/docs/apache-airflow-providers-zendesk/index.rst index 0b2852be24280..68d9576441888 100644 --- a/docs/apache-airflow-providers-zendesk/index.rst +++ b/docs/apache-airflow-providers-zendesk/index.rst @@ -48,14 +48,14 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/zendesk/index> + System Tests <_api/tests/system/zendesk/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index aac04f9b53454..e15e1c78045c2 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -307,7 +307,7 @@ Below is an example of using the ``@task.docker`` decorator to run a Python task .. _taskflow/docker_example: -.. exampleinclude:: /../../tests/system/providers/docker/example_taskflow_api_docker_virtualenv.py +.. exampleinclude:: /../../providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py :language: python :dedent: 4 :start-after: [START transform_docker] @@ -338,7 +338,7 @@ Below is an example of using the ``@task.kubernetes`` decorator to run a Python .. _taskflow/kubernetes_example: -.. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes_decorator.py +.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_kubernetes] diff --git a/docs/build_docs.py b/docs/build_docs.py index f856d6828e8ed..82edc9d1632d3 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -35,7 +35,7 @@ from tabulate import tabulate from docs.exts.docs_build import dev_index_generator, lint_checks -from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, PROVIDER_INIT_FILE +from docs.exts.docs_build.code_utils import CONSOLE_WIDTH from docs.exts.docs_build.docs_builder import DOCS_DIR, AirflowDocsBuilder, get_available_packages from docs.exts.docs_build.errors import DocBuildError, display_errors_summary from docs.exts.docs_build.fetch_inventories import fetch_inventories @@ -566,9 +566,6 @@ def main(): if not package_filters: _promote_new_flags() - if os.path.exists(PROVIDER_INIT_FILE): - os.remove(PROVIDER_INIT_FILE) - print_build_errors_and_exit( all_build_errors, all_spelling_errors, diff --git a/docs/conf.py b/docs/conf.py index 4d01e402195a5..a09e54db63d20 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,6 +32,7 @@ # All configuration values have a default; values that are commented out # serve to show the default. import json +import logging import os import pathlib import re @@ -74,13 +75,17 @@ ) except StopIteration: raise RuntimeError(f"Could not find provider.yaml file for package: {PACKAGE_NAME}") - PACKAGE_DIR = pathlib.Path(CURRENT_PROVIDER["package-dir"]) + + # Oddity: since we set autoapi_python_use_implicit_namespaces for provider packages, it does a "../"on the + # dir we give it. So we want to set the package dir to be airflow so it goes up to src, else we end up + # with "src" in the output paths of modules which we don't want + PACKAGE_DIR = ROOT_DIR / "providers" / "src" / "airflow" PACKAGE_VERSION = CURRENT_PROVIDER["versions"][0] - SYSTEM_TESTS_DIR = CURRENT_PROVIDER["system-tests-dir"] + SYSTEM_TESTS_DIR = ROOT_DIR / "providers" / "tests" / "system" elif PACKAGE_NAME == "apache-airflow-providers": from provider_yaml_utils import load_package_data - PACKAGE_DIR = ROOT_DIR / "airflow" / "providers" + PACKAGE_DIR = ROOT_DIR / "providers" / "src" PACKAGE_VERSION = "devel" ALL_PROVIDER_YAMLS = load_package_data() SYSTEM_TESTS_DIR = None @@ -108,8 +113,13 @@ global_substitutions = { "version": PACKAGE_VERSION, "airflow-version": airflow.__version__, + "experimental": "This is an :ref:`experimental feature `.", } +if PACKAGE_NAME != "apache-airflow": + global_substitutions["experimental"] = "This is an :external:ref:`experimental feature `." + + # == Sphinx configuration ====================================================== # -- Project information ------------------------------------------------------- @@ -125,13 +135,7 @@ # -- General configuration ----------------------------------------------------- # See: https://www.sphinx-doc.org/en/master/usage/configuration.html -rst_epilog = "\n".join( - f".. |{key}| replace:: {replace}" - for key, replace in { - **global_substitutions, - "experimental": "This is an :ref:`experimental feature `.", - }.items() -) +rst_epilog = "\n".join(f".. |{key}| replace:: {replace}" for key, replace in global_substitutions.items()) smartquotes_excludes = {"builders": ["man", "text", "spelling"]} @@ -139,7 +143,6 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "provider_init_hack", "sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinxarg.ext", @@ -740,8 +743,6 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "") if PACKAGE_NAME != "docker-stack": autoapi_dirs.append(PACKAGE_DIR) -if SYSTEM_TESTS_DIR and os.path.exists(SYSTEM_TESTS_DIR): - autoapi_dirs.append(SYSTEM_TESTS_DIR) # A directory that has user-defined templates to override our default templates. if PACKAGE_NAME == "apache-airflow": @@ -755,16 +756,74 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "") "*/node_modules/*", "*/migrations/*", "*/contrib/*", - "**/example_taskflow_api_docker_virtualenv.py", - "**/example_dag_decorator.py", + "*/example_taskflow_api_docker_virtualenv.py", + "*/example_dag_decorator.py", + "*/conftest.py", + "*/tests/__init__.py", + "*/tests/system/__init__.py", + "*/test_aws_auth_manager.py", + # These sub-folders aren't really providers, but we need __init__.py files else various tools (ruff, mypy) + # get confused by providers/tests/systems/cncf/kubernetes and think that folder is the top level + # kubernetes module! + "*/providers/tests/__init__.py", + "*/providers/tests/cncf/__init__.py", + "*/providers/tests/common/__init__.py", + "*/providers/tests/apache/__init__.py", + "*/providers/tests/dbt/__init__.py", + "*/providers/tests/microsoft/__init__.py", + "*/providers/tests/system/__init__.py", + "*/providers/tests/system/apache/__init__.py", + "*/providers/tests/system/cncf/__init__.py", + "*/providers/tests/system/common/__init__.py", + "*/providers/tests/system/dbt/__init__.py", + "*/providers/tests/system/microsoft/__init__.py", ] -if PACKAGE_NAME == "apache-airflow": - autoapi_ignore.append("*/airflow/providers/*") -elif PACKAGE_NAME == "docker-stack": - autoapi_ignore.append("*/airflow/providers/*") + +ignore_re = re.compile(r"\[AutoAPI\] .* Ignoring \s (?P/[\w/.]*)", re.VERBOSE) + + +# Make the "Ignoring /..." log messages slightly less verbose +def filter_ignore(record: logging.LogRecord) -> bool: + matches = ignore_re.search(record.msg) + if not matches: + return True + if matches["path"].endswith("__init__.py"): + record.msg = record.msg.replace("__init__.py", "") + return True + return False + + +autoapi_log = logging.getLogger("sphinx.autoapi.mappers.base") +autoapi_log.addFilter(filter_ignore) + +if PACKAGE_NAME.startswith("apache-airflow-providers-"): + autoapi_python_use_implicit_namespaces = True + from provider_yaml_utils import load_package_data + + autoapi_ignore.extend( + ( + "*/airflow/__init__.py", + "*/airflow/providiers/__init__.py", + "*/example_dags/*", + "*/airflow/providers/cncf/kubernetes/backcompat/*", + ) + ) + + for p in load_package_data(include_suspended=True): + if p["package-name"] == PACKAGE_NAME: + continue + autoapi_ignore.extend((p["package-dir"] + "/*", p["system-tests-dir"] + "/*")) + + autoapi_keep_files = True + + if SYSTEM_TESTS_DIR and os.path.exists(SYSTEM_TESTS_DIR): + test_dir = SYSTEM_TESTS_DIR.parent + autoapi_dirs.append(test_dir) + + autoapi_ignore.extend(f"{d}/*" for d in test_dir.glob("*") if d.is_dir() and d.name != "system") else: - autoapi_ignore.append("*/airflow/providers/cncf/kubernetes/backcompat/*") - autoapi_ignore.append("*/example_dags/*") + if SYSTEM_TESTS_DIR and os.path.exists(SYSTEM_TESTS_DIR): + autoapi_dirs.append(SYSTEM_TESTS_DIR) # Keep the AutoAPI generated files on the filesystem after the run. # Useful for debugging. autoapi_keep_files = True diff --git a/docs/exts/docs_build/code_utils.py b/docs/exts/docs_build/code_utils.py index 6aef1ab1f3051..3bbade4beb59e 100644 --- a/docs/exts/docs_build/code_utils.py +++ b/docs/exts/docs_build/code_utils.py @@ -24,7 +24,6 @@ ROOT_PROJECT_DIR = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir) ) -PROVIDER_INIT_FILE = os.path.join(ROOT_PROJECT_DIR, "airflow", "providers", "__init__.py") DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs") AIRFLOW_DIR = os.path.join(ROOT_PROJECT_DIR, "airflow") diff --git a/docs/exts/exampleinclude.py b/docs/exts/exampleinclude.py index eade660515db1..ee5d9329d2a25 100644 --- a/docs/exts/exampleinclude.py +++ b/docs/exts/exampleinclude.py @@ -188,7 +188,18 @@ def create_node(env, relative_path, show_button): :param show_button: whether to show "view code" button :return paragraph with the node """ - pagename = "_modules/" + relative_path[:-3] + + # Strip "providers" out of the example title that we include/link to. The full path needs to include + # it so we can pull in the code, but we don't want it to show up in the rendered docs + if relative_path.startswith("providers/src/"): + relative_path = relative_path.replace("providers/src/", "", 1) + elif relative_path.startswith("providers/"): + relative_path = relative_path.replace("providers/", "", 1) + + if relative_path.endswith(".py"): + pagename = "_modules/" + relative_path[:-3] + else: + pagename = "_modules/" + relative_path header_classes = ["example-header"] if show_button: diff --git a/docs/exts/provider_init_hack.py b/docs/exts/provider_init_hack.py deleted file mode 100644 index 819a78b95d7bf..0000000000000 --- a/docs/exts/provider_init_hack.py +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Bugs in sphinx-autoapi using metaclasses prevent us from upgrading to 1.3 -which has implicit namespace support. Until that time, we make it look -like a real package for building docs -""" - -from __future__ import annotations - -import os -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from sphinx.application import Sphinx - -ROOT_PROJECT_DIR = os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) -) - -PROVIDER_INIT_FILE = os.path.join(ROOT_PROJECT_DIR, "airflow", "providers", "__init__.py") - - -def _create_init_py(app, config): - del app - del config - # This file is deleted by /docs/build_docs.py. If you are not using the script, the file will be - # deleted by pre-commit. - with open(PROVIDER_INIT_FILE, "w"): - pass - - -def setup(app: Sphinx): - """ - Sets the plugin up and returns configuration of the plugin. - - :param app: application. - :return json description of the configuration that is needed by the plugin. - """ - app.connect("config-inited", _create_init_py) - - return {"version": "builtin", "parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index 1eaf93a6986e7..6e5d3a835e1d0 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -18,7 +18,7 @@ import json import os -from glob import glob +from functools import lru_cache from pathlib import Path from typing import Any @@ -26,6 +26,8 @@ import yaml ROOT_DIR = Path(__file__).parents[2].resolve() +AIRFLOW_PROVIDERS_SRC = ROOT_DIR / "providers" / "src" +AIRFLOW_PROVIDERS_NS_PACKAGE = AIRFLOW_PROVIDERS_SRC / "airflow" / "providers" PROVIDER_DATA_SCHEMA_PATH = ROOT_DIR / "airflow" / "provider.yaml.schema.json" @@ -36,24 +38,22 @@ def _load_schema() -> dict[str, Any]: def _filepath_to_module(filepath: str): - return str(Path(filepath).relative_to(ROOT_DIR)).replace("/", ".") + return str(Path(filepath).relative_to(AIRFLOW_PROVIDERS_SRC)).replace("/", ".") def _filepath_to_system_tests(filepath: str): return str( - ROOT_DIR - / "tests" - / "system" - / "providers" - / Path(filepath).relative_to(ROOT_DIR / "airflow" / "providers") + ROOT_DIR / "providers" / "tests" / "system" / Path(filepath).relative_to(AIRFLOW_PROVIDERS_NS_PACKAGE) ) +@lru_cache def get_provider_yaml_paths(): """Returns list of provider.yaml files""" - return sorted(glob(f"{ROOT_DIR}/airflow/providers/**/provider.yaml", recursive=True)) + return sorted(AIRFLOW_PROVIDERS_NS_PACKAGE.rglob("**/provider.yaml")) +@lru_cache def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: """ Load all data from providers files diff --git a/docs/exts/providers_extensions.py b/docs/exts/providers_extensions.py index 5de6dcb8eb7b2..1b59cc00aca3f 100644 --- a/docs/exts/providers_extensions.py +++ b/docs/exts/providers_extensions.py @@ -23,10 +23,8 @@ from pathlib import Path from typing import Any, Iterable -import yaml - # No stub exists for docutils.parsers.rst.directives. See https://github.com/python/typeshed/issues/5755. -from provider_yaml_utils import get_provider_yaml_paths +from provider_yaml_utils import load_package_data from docs.exts.operators_and_hooks_ref import ( DEFAULT_HEADER_SEPARATOR, @@ -64,7 +62,7 @@ def get_import_mappings(tree): def _get_module_class_registry( - module_filepath: str, class_extras: dict[str, Any] + module_filepath: Path, module_name: str, class_extras: dict[str, Any] ) -> dict[str, dict[str, Any]]: """Extracts classes and its information from a Python module file. @@ -80,7 +78,6 @@ def _get_module_class_registry( with open(module_filepath) as file: ast_obj = ast.parse(file.read()) - module_name = module_filepath.replace("/", ".").replace(".py", "").lstrip(".") import_mappings = get_import_mappings(ast_obj) module_class_registry = { f"{module_name}.{node.name}": { @@ -140,16 +137,26 @@ def _get_providers_class_registry() -> dict[str, dict[str, Any]]: :return: A dictionary with provider names as keys and a dictionary of classes as values. """ class_registry = {} - for provider_yaml_path in get_provider_yaml_paths(): - provider_yaml_content = yaml.safe_load(Path(provider_yaml_path).read_text()) - for root, _, file_names in os.walk(Path(provider_yaml_path).parent): + for provider_yaml_content in load_package_data(): + provider_pkg_root = Path(provider_yaml_content["package-dir"]) + for root, _, file_names in os.walk(provider_pkg_root): + folder = Path(root) for file_name in file_names: - module_filepath = f"{os.path.relpath(root)}/{file_name}" - if not module_filepath.endswith(".py") or module_filepath == "__init__.py": + if not file_name.endswith(".py") or file_name == "__init__.py": continue + module_filepath = folder.joinpath(file_name) + module_registry = _get_module_class_registry( module_filepath=module_filepath, + module_name=( + provider_yaml_content["python-module"] + + "." + + module_filepath.relative_to(provider_pkg_root) + .with_suffix("") + .as_posix() + .replace("/", ".") + ), class_extras={"provider_name": provider_yaml_content["package-name"]}, ) class_registry.update(module_registry) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 2d0ab90a3508b..e6a0deca23cbf 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -345,8 +345,7 @@ "devel-deps": [], "plugins": [], "cross-providers-deps": [ - "cncf.kubernetes", - "standard" + "cncf.kubernetes" ], "excluded-python-versions": [], "state": "ready" @@ -534,9 +533,7 @@ "plugin-class": "airflow.providers.edge.plugins.edge_executor_plugin.EdgeExecutorPlugin" } ], - "cross-providers-deps": [ - "standard" - ], + "cross-providers-deps": [], "excluded-python-versions": [], "state": "not-ready" }, diff --git a/providers/.gitignore b/providers/.gitignore new file mode 100644 index 0000000000000..2924614f3a149 --- /dev/null +++ b/providers/.gitignore @@ -0,0 +1,7 @@ +# Ignore init files in these non-provider folders. I.e. we relrease atlassian/jira, but not atlassian +src/airflow/providers/apache/__init__.py +src/airflow/providers/atlassian/__init__.py +src/airflow/providers/cncf/__init__.py +src/airflow/providers/common/__init__.py +src/airflow/providers/dbt/__init__.py +src/airflow/providers/microsoft/__init__.py diff --git a/tests/test_utils/__init__.py b/providers/__init__.py similarity index 74% rename from tests/test_utils/__init__.py rename to providers/__init__.py index 4c4790fa4b479..c53de5451bd08 100644 --- a/tests/test_utils/__init__.py +++ b/providers/__init__.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,10 +14,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - -import os -AIRFLOW_MAIN_FOLDER = os.path.realpath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) -) +# This exists so that pytest doesn't get confused about namespace packages +# and think that `tests/conftest.py` and `providers/tests/conftest.py` are +# both "tests.conftest" +# +# This is a temporary solution until https://github.com/apache/airflow/issues/42632 +# is done diff --git a/providers/pyproject.toml b/providers/pyproject.toml new file mode 100644 index 0000000000000..093cbbd547141 --- /dev/null +++ b/providers/pyproject.toml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "local-providers" +version = "0.1.0" +description = "Placeholder package for local/from-sources providers." +requires-python = ">=3.8, <3.13" +classifiers = [ + "Private :: Do Not Upload", +] + +[tool.hatch.publish.index] +# Lets make doubly sure this never goes to PyPi +disable = true + +[tool.hatch.build.targets.wheel] +packages = ["src/airflow"] +exclude = [ + ".gitignore", + ".latest-doc-only-change.txt", + "CHANGELOG.rst", + "MANAGING_PROVIDERS_LIFECYCLE.rst", +] + +[tool.ruff] +extend = "../pyproject.toml" +src = ["src"] +namespace-packages = ["src/airflow/providers"] +extend-exclude = [ + # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can + # ignore them in ruff + "src/airflow/providers/common/sql/*/*.pyi", +] + +[tool.ruff.lint.per-file-ignores] + +# Ignore Doc rules et al for anything outside of tests +"!src/*" = ["D", "TID253", "S101", "TRY002"] + +# https://github.com/apache/airflow/issues/39252 +"src/airflow/providers/amazon/aws/hooks/eks.py" = ["W605"] + +# All of the modules which have an extra license header (i.e. that we copy from another project) need to +# ignore E402 -- module level import not at top level +"tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py" = ["E402"] +"tests/common/io/xcom/test_backend.py" = ["E402"] +"tests/elasticsearch/log/elasticmock/__init__.py" = ["E402"] +"tests/elasticsearch/log/elasticmock/utilities/__init__.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_auto_ml.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_custom_job.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_dataset.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_endpoint_service.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_generative_model.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_model_service.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_pipeline_job.py" = ["E402"] +"tests/google/cloud/hooks/vertex_ai/test_prediction_service.py" = ["E402"] +"tests/google/cloud/links/test_translate.py" = ["E402"] +"tests/google/cloud/operators/test_automl.py"= ["E402"] +"tests/google/cloud/operators/test_vertex_ai.py" = ["E402"] +"tests/google/cloud/operators/vertex_ai/test_generative_model.py" = ["E402"] +"tests/google/cloud/triggers/test_vertex_ai.py" = ["E402"] +"tests/openai/hooks/test_openai.py" = ["E402"] +"tests/openai/operators/test_openai.py" = ["E402"] +"tests/openai/triggers/test_openai.py" = ["E402"] +"tests/opensearch/conftest.py" = ["E402"] +"tests/opensearch/hooks/test_opensearch.py" = ["E402"] +"tests/opensearch/log/test_os_json_formatter.py" = ["E402"] +"tests/opensearch/log/test_os_response.py" = ["E402"] +"tests/opensearch/log/test_os_task_handler.py" = ["E402"] +"tests/opensearch/operators/test_opensearch.py" = ["E402"] +"tests/qdrant/hooks/test_qdrant.py" = ["E402"] +"tests/qdrant/operators/test_qdrant.py" = ["E402"] +"tests/snowflake/operators/test_snowflake_sql.py" = ["E402"] +"tests/yandex/**/*.py" = ["E402"] + +# https://github.com/apache/airflow/issues/39252 +"airflow/providers/amazon/aws/hooks/eks.py" = ["W605"] diff --git a/providers/src/airflow/providers/.gitignore b/providers/src/airflow/providers/.gitignore new file mode 100644 index 0000000000000..528066d9003e2 --- /dev/null +++ b/providers/src/airflow/providers/.gitignore @@ -0,0 +1 @@ +/__init__.py diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/providers/src/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst similarity index 98% rename from airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst rename to providers/src/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst index 3d3e95c28b172..c5e6ec1287854 100644 --- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst +++ b/providers/src/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst @@ -108,7 +108,7 @@ breeze and I'll run unit tests for my Hook. .. code-block:: bash - root@fafd8d630e46:/opt/airflow# python -m pytest tests/providers//hook/test_*.py + root@fafd8d630e46:/opt/airflow# python -m pytest providers/tests//hook/test_*.py Adding chicken-egg providers ---------------------------- @@ -341,23 +341,23 @@ Example failing collection after ``google`` provider has been suspended: .. code-block:: txt - _____ ERROR collecting tests/providers/apache/beam/operators/test_beam.py ______ - ImportError while importing test module '/opt/airflow/tests/providers/apache/beam/operators/test_beam.py'. + _____ ERROR collecting providers/tests/apache/beam/operators/test_beam.py ______ + ImportError while importing test module '/opt/airflow/providers/tests/apache/beam/operators/test_beam.py'. Hint: make sure your test modules/packages have valid Python names. Traceback: /usr/local/lib/python3.8/importlib/__init__.py:127: in import_module return _bootstrap._gcd_import(name[level:], package, level) - tests/providers/apache/beam/operators/test_beam.py:25: in + providers/tests/apache/beam/operators/test_beam.py:25: in from airflow.providers.apache.beam.operators.beam import ( airflow/providers/apache/beam/operators/beam.py:35: in from airflow.providers.google.cloud.hooks.dataflow import ( airflow/providers/google/cloud/hooks/dataflow.py:32: in from google.cloud.dataflow_v1beta3 import GetJobRequest, Job, JobState, JobsV1Beta3AsyncClient, JobView E ModuleNotFoundError: No module named 'google.cloud.dataflow_v1beta3' - _ ERROR collecting tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py _ + _ ERROR collecting providers/tests/microsoft/azure/transfers/test_azure_blob_to_gcs.py _ -The fix is to add this line at the top of the ``tests/providers/apache/beam/operators/test_beam.py`` module: +The fix is to add this line at the top of the ``providers/tests/apache/beam/operators/test_beam.py`` module: .. code-block:: python diff --git a/airflow/providers/airbyte/.latest-doc-only-change.txt b/providers/src/airflow/providers/airbyte/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/airbyte/.latest-doc-only-change.txt rename to providers/src/airflow/providers/airbyte/.latest-doc-only-change.txt diff --git a/airflow/providers/airbyte/CHANGELOG.rst b/providers/src/airflow/providers/airbyte/CHANGELOG.rst similarity index 100% rename from airflow/providers/airbyte/CHANGELOG.rst rename to providers/src/airflow/providers/airbyte/CHANGELOG.rst diff --git a/airflow/providers/airbyte/__init__.py b/providers/src/airflow/providers/airbyte/__init__.py similarity index 100% rename from airflow/providers/airbyte/__init__.py rename to providers/src/airflow/providers/airbyte/__init__.py diff --git a/airflow/providers/airbyte/hooks/__init__.py b/providers/src/airflow/providers/airbyte/hooks/__init__.py similarity index 100% rename from airflow/providers/airbyte/hooks/__init__.py rename to providers/src/airflow/providers/airbyte/hooks/__init__.py diff --git a/airflow/providers/airbyte/hooks/airbyte.py b/providers/src/airflow/providers/airbyte/hooks/airbyte.py similarity index 100% rename from airflow/providers/airbyte/hooks/airbyte.py rename to providers/src/airflow/providers/airbyte/hooks/airbyte.py diff --git a/airflow/providers/airbyte/operators/__init__.py b/providers/src/airflow/providers/airbyte/operators/__init__.py similarity index 100% rename from airflow/providers/airbyte/operators/__init__.py rename to providers/src/airflow/providers/airbyte/operators/__init__.py diff --git a/airflow/providers/airbyte/operators/airbyte.py b/providers/src/airflow/providers/airbyte/operators/airbyte.py similarity index 100% rename from airflow/providers/airbyte/operators/airbyte.py rename to providers/src/airflow/providers/airbyte/operators/airbyte.py diff --git a/airflow/providers/airbyte/provider.yaml b/providers/src/airflow/providers/airbyte/provider.yaml similarity index 100% rename from airflow/providers/airbyte/provider.yaml rename to providers/src/airflow/providers/airbyte/provider.yaml diff --git a/airflow/providers/alibaba/cloud/operators/__init__.py b/providers/src/airflow/providers/airbyte/sensors/__init__.py similarity index 100% rename from airflow/providers/alibaba/cloud/operators/__init__.py rename to providers/src/airflow/providers/airbyte/sensors/__init__.py diff --git a/airflow/providers/airbyte/sensors/airbyte.py b/providers/src/airflow/providers/airbyte/sensors/airbyte.py similarity index 100% rename from airflow/providers/airbyte/sensors/airbyte.py rename to providers/src/airflow/providers/airbyte/sensors/airbyte.py diff --git a/airflow/providers/alibaba/cloud/sensors/__init__.py b/providers/src/airflow/providers/airbyte/triggers/__init__.py similarity index 100% rename from airflow/providers/alibaba/cloud/sensors/__init__.py rename to providers/src/airflow/providers/airbyte/triggers/__init__.py diff --git a/airflow/providers/airbyte/triggers/airbyte.py b/providers/src/airflow/providers/airbyte/triggers/airbyte.py similarity index 100% rename from airflow/providers/airbyte/triggers/airbyte.py rename to providers/src/airflow/providers/airbyte/triggers/airbyte.py diff --git a/airflow/providers/alibaba/.latest-doc-only-change.txt b/providers/src/airflow/providers/alibaba/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/alibaba/.latest-doc-only-change.txt rename to providers/src/airflow/providers/alibaba/.latest-doc-only-change.txt diff --git a/airflow/providers/alibaba/CHANGELOG.rst b/providers/src/airflow/providers/alibaba/CHANGELOG.rst similarity index 100% rename from airflow/providers/alibaba/CHANGELOG.rst rename to providers/src/airflow/providers/alibaba/CHANGELOG.rst diff --git a/airflow/providers/alibaba/__init__.py b/providers/src/airflow/providers/alibaba/__init__.py similarity index 100% rename from airflow/providers/alibaba/__init__.py rename to providers/src/airflow/providers/alibaba/__init__.py diff --git a/airflow/providers/alibaba/cloud/__init__.py b/providers/src/airflow/providers/alibaba/cloud/__init__.py similarity index 100% rename from airflow/providers/alibaba/cloud/__init__.py rename to providers/src/airflow/providers/alibaba/cloud/__init__.py diff --git a/airflow/providers/alibaba/cloud/hooks/__init__.py b/providers/src/airflow/providers/alibaba/cloud/hooks/__init__.py similarity index 100% rename from airflow/providers/alibaba/cloud/hooks/__init__.py rename to providers/src/airflow/providers/alibaba/cloud/hooks/__init__.py diff --git a/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py b/providers/src/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py similarity index 100% rename from airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py rename to providers/src/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py diff --git a/airflow/providers/alibaba/cloud/hooks/oss.py b/providers/src/airflow/providers/alibaba/cloud/hooks/oss.py similarity index 100% rename from airflow/providers/alibaba/cloud/hooks/oss.py rename to providers/src/airflow/providers/alibaba/cloud/hooks/oss.py diff --git a/airflow/providers/amazon/aws/__init__.py b/providers/src/airflow/providers/alibaba/cloud/log/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/__init__.py rename to providers/src/airflow/providers/alibaba/cloud/log/__init__.py diff --git a/airflow/providers/alibaba/cloud/log/oss_task_handler.py b/providers/src/airflow/providers/alibaba/cloud/log/oss_task_handler.py similarity index 100% rename from airflow/providers/alibaba/cloud/log/oss_task_handler.py rename to providers/src/airflow/providers/alibaba/cloud/log/oss_task_handler.py diff --git a/airflow/providers/amazon/aws/assets/__init__.py b/providers/src/airflow/providers/alibaba/cloud/operators/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/assets/__init__.py rename to providers/src/airflow/providers/alibaba/cloud/operators/__init__.py diff --git a/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py b/providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py similarity index 100% rename from airflow/providers/alibaba/cloud/operators/analyticdb_spark.py rename to providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py diff --git a/airflow/providers/alibaba/cloud/operators/oss.py b/providers/src/airflow/providers/alibaba/cloud/operators/oss.py similarity index 100% rename from airflow/providers/alibaba/cloud/operators/oss.py rename to providers/src/airflow/providers/alibaba/cloud/operators/oss.py diff --git a/airflow/providers/amazon/aws/auth_manager/__init__.py b/providers/src/airflow/providers/alibaba/cloud/sensors/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/__init__.py rename to providers/src/airflow/providers/alibaba/cloud/sensors/__init__.py diff --git a/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py b/providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py similarity index 100% rename from airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py rename to providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py diff --git a/airflow/providers/alibaba/cloud/sensors/oss_key.py b/providers/src/airflow/providers/alibaba/cloud/sensors/oss_key.py similarity index 100% rename from airflow/providers/alibaba/cloud/sensors/oss_key.py rename to providers/src/airflow/providers/alibaba/cloud/sensors/oss_key.py diff --git a/airflow/providers/alibaba/provider.yaml b/providers/src/airflow/providers/alibaba/provider.yaml similarity index 100% rename from airflow/providers/alibaba/provider.yaml rename to providers/src/airflow/providers/alibaba/provider.yaml diff --git a/airflow/providers/amazon/.latest-doc-only-change.txt b/providers/src/airflow/providers/amazon/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/amazon/.latest-doc-only-change.txt rename to providers/src/airflow/providers/amazon/.latest-doc-only-change.txt diff --git a/airflow/providers/amazon/CHANGELOG.rst b/providers/src/airflow/providers/amazon/CHANGELOG.rst similarity index 100% rename from airflow/providers/amazon/CHANGELOG.rst rename to providers/src/airflow/providers/amazon/CHANGELOG.rst diff --git a/airflow/providers/amazon/__init__.py b/providers/src/airflow/providers/amazon/__init__.py similarity index 100% rename from airflow/providers/amazon/__init__.py rename to providers/src/airflow/providers/amazon/__init__.py diff --git a/airflow/providers/amazon/aws/auth_manager/avp/__init__.py b/providers/src/airflow/providers/amazon/aws/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/avp/__init__.py rename to providers/src/airflow/providers/amazon/aws/__init__.py diff --git a/airflow/providers/amazon/aws/auth_manager/cli/__init__.py b/providers/src/airflow/providers/amazon/aws/assets/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/cli/__init__.py rename to providers/src/airflow/providers/amazon/aws/assets/__init__.py diff --git a/airflow/providers/amazon/aws/assets/s3.py b/providers/src/airflow/providers/amazon/aws/assets/s3.py similarity index 100% rename from airflow/providers/amazon/aws/assets/s3.py rename to providers/src/airflow/providers/amazon/aws/assets/s3.py diff --git a/airflow/providers/amazon/aws/auth_manager/security_manager/__init__.py b/providers/src/airflow/providers/amazon/aws/auth_manager/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/security_manager/__init__.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/__init__.py diff --git a/airflow/providers/amazon/aws/auth_manager/views/__init__.py b/providers/src/airflow/providers/amazon/aws/auth_manager/avp/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/views/__init__.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/avp/__init__.py diff --git a/airflow/providers/amazon/aws/auth_manager/avp/entities.py b/providers/src/airflow/providers/amazon/aws/auth_manager/avp/entities.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/avp/entities.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/avp/entities.py diff --git a/airflow/providers/amazon/aws/auth_manager/avp/facade.py b/providers/src/airflow/providers/amazon/aws/auth_manager/avp/facade.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/avp/facade.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/avp/facade.py diff --git a/airflow/providers/amazon/aws/auth_manager/avp/schema.json b/providers/src/airflow/providers/amazon/aws/auth_manager/avp/schema.json similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/avp/schema.json rename to providers/src/airflow/providers/amazon/aws/auth_manager/avp/schema.json diff --git a/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py b/providers/src/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py diff --git a/airflow/providers/amazon/aws/executors/__init__.py b/providers/src/airflow/providers/amazon/aws/auth_manager/cli/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/executors/__init__.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/cli/__init__.py diff --git a/airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py b/providers/src/airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py diff --git a/airflow/providers/amazon/aws/auth_manager/cli/definition.py b/providers/src/airflow/providers/amazon/aws/auth_manager/cli/definition.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/cli/definition.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/cli/definition.py diff --git a/airflow/providers/amazon/aws/auth_manager/constants.py b/providers/src/airflow/providers/amazon/aws/auth_manager/constants.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/constants.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/constants.py diff --git a/airflow/providers/amazon/aws/executors/utils/__init__.py b/providers/src/airflow/providers/amazon/aws/auth_manager/security_manager/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/executors/utils/__init__.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/security_manager/__init__.py diff --git a/airflow/providers/amazon/aws/auth_manager/security_manager/aws_security_manager_override.py b/providers/src/airflow/providers/amazon/aws/auth_manager/security_manager/aws_security_manager_override.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/security_manager/aws_security_manager_override.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/security_manager/aws_security_manager_override.py diff --git a/airflow/providers/amazon/aws/auth_manager/user.py b/providers/src/airflow/providers/amazon/aws/auth_manager/user.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/user.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/user.py diff --git a/airflow/providers/amazon/aws/fs/__init__.py b/providers/src/airflow/providers/amazon/aws/auth_manager/views/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/fs/__init__.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/views/__init__.py diff --git a/airflow/providers/amazon/aws/auth_manager/views/auth.py b/providers/src/airflow/providers/amazon/aws/auth_manager/views/auth.py similarity index 100% rename from airflow/providers/amazon/aws/auth_manager/views/auth.py rename to providers/src/airflow/providers/amazon/aws/auth_manager/views/auth.py diff --git a/airflow/providers/amazon/aws/exceptions.py b/providers/src/airflow/providers/amazon/aws/exceptions.py similarity index 100% rename from airflow/providers/amazon/aws/exceptions.py rename to providers/src/airflow/providers/amazon/aws/exceptions.py diff --git a/airflow/providers/amazon/aws/executors/Dockerfile b/providers/src/airflow/providers/amazon/aws/executors/Dockerfile similarity index 100% rename from airflow/providers/amazon/aws/executors/Dockerfile rename to providers/src/airflow/providers/amazon/aws/executors/Dockerfile diff --git a/airflow/providers/amazon/aws/hooks/__init__.py b/providers/src/airflow/providers/amazon/aws/executors/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/__init__.py rename to providers/src/airflow/providers/amazon/aws/executors/__init__.py diff --git a/airflow/providers/amazon/aws/executors/batch/__init__.py b/providers/src/airflow/providers/amazon/aws/executors/batch/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/executors/batch/__init__.py rename to providers/src/airflow/providers/amazon/aws/executors/batch/__init__.py diff --git a/airflow/providers/amazon/aws/executors/batch/batch_executor.py b/providers/src/airflow/providers/amazon/aws/executors/batch/batch_executor.py similarity index 100% rename from airflow/providers/amazon/aws/executors/batch/batch_executor.py rename to providers/src/airflow/providers/amazon/aws/executors/batch/batch_executor.py diff --git a/airflow/providers/amazon/aws/executors/batch/batch_executor_config.py b/providers/src/airflow/providers/amazon/aws/executors/batch/batch_executor_config.py similarity index 100% rename from airflow/providers/amazon/aws/executors/batch/batch_executor_config.py rename to providers/src/airflow/providers/amazon/aws/executors/batch/batch_executor_config.py diff --git a/airflow/providers/amazon/aws/executors/batch/boto_schema.py b/providers/src/airflow/providers/amazon/aws/executors/batch/boto_schema.py similarity index 100% rename from airflow/providers/amazon/aws/executors/batch/boto_schema.py rename to providers/src/airflow/providers/amazon/aws/executors/batch/boto_schema.py diff --git a/airflow/providers/amazon/aws/executors/batch/utils.py b/providers/src/airflow/providers/amazon/aws/executors/batch/utils.py similarity index 100% rename from airflow/providers/amazon/aws/executors/batch/utils.py rename to providers/src/airflow/providers/amazon/aws/executors/batch/utils.py diff --git a/airflow/providers/amazon/aws/executors/ecs/__init__.py b/providers/src/airflow/providers/amazon/aws/executors/ecs/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/executors/ecs/__init__.py rename to providers/src/airflow/providers/amazon/aws/executors/ecs/__init__.py diff --git a/airflow/providers/amazon/aws/executors/ecs/boto_schema.py b/providers/src/airflow/providers/amazon/aws/executors/ecs/boto_schema.py similarity index 100% rename from airflow/providers/amazon/aws/executors/ecs/boto_schema.py rename to providers/src/airflow/providers/amazon/aws/executors/ecs/boto_schema.py diff --git a/airflow/providers/amazon/aws/executors/ecs/ecs_executor.py b/providers/src/airflow/providers/amazon/aws/executors/ecs/ecs_executor.py similarity index 100% rename from airflow/providers/amazon/aws/executors/ecs/ecs_executor.py rename to providers/src/airflow/providers/amazon/aws/executors/ecs/ecs_executor.py diff --git a/airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py b/providers/src/airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py similarity index 100% rename from airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py rename to providers/src/airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py diff --git a/airflow/providers/amazon/aws/executors/ecs/utils.py b/providers/src/airflow/providers/amazon/aws/executors/ecs/utils.py similarity index 100% rename from airflow/providers/amazon/aws/executors/ecs/utils.py rename to providers/src/airflow/providers/amazon/aws/executors/ecs/utils.py diff --git a/airflow/providers/amazon/aws/links/__init__.py b/providers/src/airflow/providers/amazon/aws/executors/utils/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/links/__init__.py rename to providers/src/airflow/providers/amazon/aws/executors/utils/__init__.py diff --git a/airflow/providers/amazon/aws/executors/utils/base_config_keys.py b/providers/src/airflow/providers/amazon/aws/executors/utils/base_config_keys.py similarity index 100% rename from airflow/providers/amazon/aws/executors/utils/base_config_keys.py rename to providers/src/airflow/providers/amazon/aws/executors/utils/base_config_keys.py diff --git a/airflow/providers/amazon/aws/executors/utils/exponential_backoff_retry.py b/providers/src/airflow/providers/amazon/aws/executors/utils/exponential_backoff_retry.py similarity index 100% rename from airflow/providers/amazon/aws/executors/utils/exponential_backoff_retry.py rename to providers/src/airflow/providers/amazon/aws/executors/utils/exponential_backoff_retry.py diff --git a/airflow/providers/amazon/aws/log/__init__.py b/providers/src/airflow/providers/amazon/aws/fs/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/log/__init__.py rename to providers/src/airflow/providers/amazon/aws/fs/__init__.py diff --git a/airflow/providers/amazon/aws/fs/s3.py b/providers/src/airflow/providers/amazon/aws/fs/s3.py similarity index 100% rename from airflow/providers/amazon/aws/fs/s3.py rename to providers/src/airflow/providers/amazon/aws/fs/s3.py diff --git a/airflow/providers/amazon/aws/notifications/__init__.py b/providers/src/airflow/providers/amazon/aws/hooks/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/notifications/__init__.py rename to providers/src/airflow/providers/amazon/aws/hooks/__init__.py diff --git a/airflow/providers/amazon/aws/hooks/appflow.py b/providers/src/airflow/providers/amazon/aws/hooks/appflow.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/appflow.py rename to providers/src/airflow/providers/amazon/aws/hooks/appflow.py diff --git a/airflow/providers/amazon/aws/hooks/athena.py b/providers/src/airflow/providers/amazon/aws/hooks/athena.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/athena.py rename to providers/src/airflow/providers/amazon/aws/hooks/athena.py diff --git a/airflow/providers/amazon/aws/hooks/athena_sql.py b/providers/src/airflow/providers/amazon/aws/hooks/athena_sql.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/athena_sql.py rename to providers/src/airflow/providers/amazon/aws/hooks/athena_sql.py diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py b/providers/src/airflow/providers/amazon/aws/hooks/base_aws.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/base_aws.py rename to providers/src/airflow/providers/amazon/aws/hooks/base_aws.py diff --git a/airflow/providers/amazon/aws/hooks/batch_client.py b/providers/src/airflow/providers/amazon/aws/hooks/batch_client.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/batch_client.py rename to providers/src/airflow/providers/amazon/aws/hooks/batch_client.py diff --git a/airflow/providers/amazon/aws/hooks/batch_waiters.json b/providers/src/airflow/providers/amazon/aws/hooks/batch_waiters.json similarity index 100% rename from airflow/providers/amazon/aws/hooks/batch_waiters.json rename to providers/src/airflow/providers/amazon/aws/hooks/batch_waiters.json diff --git a/airflow/providers/amazon/aws/hooks/batch_waiters.py b/providers/src/airflow/providers/amazon/aws/hooks/batch_waiters.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/batch_waiters.py rename to providers/src/airflow/providers/amazon/aws/hooks/batch_waiters.py diff --git a/airflow/providers/amazon/aws/hooks/bedrock.py b/providers/src/airflow/providers/amazon/aws/hooks/bedrock.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/bedrock.py rename to providers/src/airflow/providers/amazon/aws/hooks/bedrock.py diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/providers/src/airflow/providers/amazon/aws/hooks/chime.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/chime.py rename to providers/src/airflow/providers/amazon/aws/hooks/chime.py diff --git a/airflow/providers/amazon/aws/hooks/cloud_formation.py b/providers/src/airflow/providers/amazon/aws/hooks/cloud_formation.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/cloud_formation.py rename to providers/src/airflow/providers/amazon/aws/hooks/cloud_formation.py diff --git a/airflow/providers/amazon/aws/hooks/comprehend.py b/providers/src/airflow/providers/amazon/aws/hooks/comprehend.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/comprehend.py rename to providers/src/airflow/providers/amazon/aws/hooks/comprehend.py diff --git a/airflow/providers/amazon/aws/hooks/datasync.py b/providers/src/airflow/providers/amazon/aws/hooks/datasync.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/datasync.py rename to providers/src/airflow/providers/amazon/aws/hooks/datasync.py diff --git a/airflow/providers/amazon/aws/hooks/dms.py b/providers/src/airflow/providers/amazon/aws/hooks/dms.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/dms.py rename to providers/src/airflow/providers/amazon/aws/hooks/dms.py diff --git a/airflow/providers/amazon/aws/hooks/dynamodb.py b/providers/src/airflow/providers/amazon/aws/hooks/dynamodb.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/dynamodb.py rename to providers/src/airflow/providers/amazon/aws/hooks/dynamodb.py diff --git a/airflow/providers/amazon/aws/hooks/ec2.py b/providers/src/airflow/providers/amazon/aws/hooks/ec2.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/ec2.py rename to providers/src/airflow/providers/amazon/aws/hooks/ec2.py diff --git a/airflow/providers/amazon/aws/hooks/ecr.py b/providers/src/airflow/providers/amazon/aws/hooks/ecr.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/ecr.py rename to providers/src/airflow/providers/amazon/aws/hooks/ecr.py diff --git a/airflow/providers/amazon/aws/hooks/ecs.py b/providers/src/airflow/providers/amazon/aws/hooks/ecs.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/ecs.py rename to providers/src/airflow/providers/amazon/aws/hooks/ecs.py diff --git a/airflow/providers/amazon/aws/hooks/eks.py b/providers/src/airflow/providers/amazon/aws/hooks/eks.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/eks.py rename to providers/src/airflow/providers/amazon/aws/hooks/eks.py diff --git a/airflow/providers/amazon/aws/hooks/elasticache_replication_group.py b/providers/src/airflow/providers/amazon/aws/hooks/elasticache_replication_group.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/elasticache_replication_group.py rename to providers/src/airflow/providers/amazon/aws/hooks/elasticache_replication_group.py diff --git a/airflow/providers/amazon/aws/hooks/emr.py b/providers/src/airflow/providers/amazon/aws/hooks/emr.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/emr.py rename to providers/src/airflow/providers/amazon/aws/hooks/emr.py diff --git a/airflow/providers/amazon/aws/hooks/eventbridge.py b/providers/src/airflow/providers/amazon/aws/hooks/eventbridge.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/eventbridge.py rename to providers/src/airflow/providers/amazon/aws/hooks/eventbridge.py diff --git a/airflow/providers/amazon/aws/hooks/glacier.py b/providers/src/airflow/providers/amazon/aws/hooks/glacier.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/glacier.py rename to providers/src/airflow/providers/amazon/aws/hooks/glacier.py diff --git a/airflow/providers/amazon/aws/hooks/glue.py b/providers/src/airflow/providers/amazon/aws/hooks/glue.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/glue.py rename to providers/src/airflow/providers/amazon/aws/hooks/glue.py diff --git a/airflow/providers/amazon/aws/hooks/glue_catalog.py b/providers/src/airflow/providers/amazon/aws/hooks/glue_catalog.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/glue_catalog.py rename to providers/src/airflow/providers/amazon/aws/hooks/glue_catalog.py diff --git a/airflow/providers/amazon/aws/hooks/glue_crawler.py b/providers/src/airflow/providers/amazon/aws/hooks/glue_crawler.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/glue_crawler.py rename to providers/src/airflow/providers/amazon/aws/hooks/glue_crawler.py diff --git a/airflow/providers/amazon/aws/hooks/glue_databrew.py b/providers/src/airflow/providers/amazon/aws/hooks/glue_databrew.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/glue_databrew.py rename to providers/src/airflow/providers/amazon/aws/hooks/glue_databrew.py diff --git a/airflow/providers/amazon/aws/hooks/kinesis.py b/providers/src/airflow/providers/amazon/aws/hooks/kinesis.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/kinesis.py rename to providers/src/airflow/providers/amazon/aws/hooks/kinesis.py diff --git a/airflow/providers/amazon/aws/hooks/kinesis_analytics.py b/providers/src/airflow/providers/amazon/aws/hooks/kinesis_analytics.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/kinesis_analytics.py rename to providers/src/airflow/providers/amazon/aws/hooks/kinesis_analytics.py diff --git a/airflow/providers/amazon/aws/hooks/lambda_function.py b/providers/src/airflow/providers/amazon/aws/hooks/lambda_function.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/lambda_function.py rename to providers/src/airflow/providers/amazon/aws/hooks/lambda_function.py diff --git a/airflow/providers/amazon/aws/hooks/logs.py b/providers/src/airflow/providers/amazon/aws/hooks/logs.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/logs.py rename to providers/src/airflow/providers/amazon/aws/hooks/logs.py diff --git a/airflow/providers/amazon/aws/hooks/neptune.py b/providers/src/airflow/providers/amazon/aws/hooks/neptune.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/neptune.py rename to providers/src/airflow/providers/amazon/aws/hooks/neptune.py diff --git a/airflow/providers/amazon/aws/hooks/opensearch_serverless.py b/providers/src/airflow/providers/amazon/aws/hooks/opensearch_serverless.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/opensearch_serverless.py rename to providers/src/airflow/providers/amazon/aws/hooks/opensearch_serverless.py diff --git a/airflow/providers/amazon/aws/hooks/quicksight.py b/providers/src/airflow/providers/amazon/aws/hooks/quicksight.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/quicksight.py rename to providers/src/airflow/providers/amazon/aws/hooks/quicksight.py diff --git a/airflow/providers/amazon/aws/hooks/rds.py b/providers/src/airflow/providers/amazon/aws/hooks/rds.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/rds.py rename to providers/src/airflow/providers/amazon/aws/hooks/rds.py diff --git a/airflow/providers/amazon/aws/hooks/redshift_cluster.py b/providers/src/airflow/providers/amazon/aws/hooks/redshift_cluster.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/redshift_cluster.py rename to providers/src/airflow/providers/amazon/aws/hooks/redshift_cluster.py diff --git a/airflow/providers/amazon/aws/hooks/redshift_data.py b/providers/src/airflow/providers/amazon/aws/hooks/redshift_data.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/redshift_data.py rename to providers/src/airflow/providers/amazon/aws/hooks/redshift_data.py diff --git a/airflow/providers/amazon/aws/hooks/redshift_sql.py b/providers/src/airflow/providers/amazon/aws/hooks/redshift_sql.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/redshift_sql.py rename to providers/src/airflow/providers/amazon/aws/hooks/redshift_sql.py diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/providers/src/airflow/providers/amazon/aws/hooks/s3.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/s3.py rename to providers/src/airflow/providers/amazon/aws/hooks/s3.py diff --git a/airflow/providers/amazon/aws/hooks/sagemaker.py b/providers/src/airflow/providers/amazon/aws/hooks/sagemaker.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/sagemaker.py rename to providers/src/airflow/providers/amazon/aws/hooks/sagemaker.py diff --git a/airflow/providers/amazon/aws/hooks/secrets_manager.py b/providers/src/airflow/providers/amazon/aws/hooks/secrets_manager.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/secrets_manager.py rename to providers/src/airflow/providers/amazon/aws/hooks/secrets_manager.py diff --git a/airflow/providers/amazon/aws/hooks/ses.py b/providers/src/airflow/providers/amazon/aws/hooks/ses.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/ses.py rename to providers/src/airflow/providers/amazon/aws/hooks/ses.py diff --git a/airflow/providers/amazon/aws/hooks/sns.py b/providers/src/airflow/providers/amazon/aws/hooks/sns.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/sns.py rename to providers/src/airflow/providers/amazon/aws/hooks/sns.py diff --git a/airflow/providers/amazon/aws/hooks/sqs.py b/providers/src/airflow/providers/amazon/aws/hooks/sqs.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/sqs.py rename to providers/src/airflow/providers/amazon/aws/hooks/sqs.py diff --git a/airflow/providers/amazon/aws/hooks/ssm.py b/providers/src/airflow/providers/amazon/aws/hooks/ssm.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/ssm.py rename to providers/src/airflow/providers/amazon/aws/hooks/ssm.py diff --git a/airflow/providers/amazon/aws/hooks/step_function.py b/providers/src/airflow/providers/amazon/aws/hooks/step_function.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/step_function.py rename to providers/src/airflow/providers/amazon/aws/hooks/step_function.py diff --git a/airflow/providers/amazon/aws/hooks/sts.py b/providers/src/airflow/providers/amazon/aws/hooks/sts.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/sts.py rename to providers/src/airflow/providers/amazon/aws/hooks/sts.py diff --git a/airflow/providers/amazon/aws/hooks/verified_permissions.py b/providers/src/airflow/providers/amazon/aws/hooks/verified_permissions.py similarity index 100% rename from airflow/providers/amazon/aws/hooks/verified_permissions.py rename to providers/src/airflow/providers/amazon/aws/hooks/verified_permissions.py diff --git a/airflow/providers/amazon/aws/operators/__init__.py b/providers/src/airflow/providers/amazon/aws/links/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/operators/__init__.py rename to providers/src/airflow/providers/amazon/aws/links/__init__.py diff --git a/airflow/providers/amazon/aws/links/athena.py b/providers/src/airflow/providers/amazon/aws/links/athena.py similarity index 100% rename from airflow/providers/amazon/aws/links/athena.py rename to providers/src/airflow/providers/amazon/aws/links/athena.py diff --git a/airflow/providers/amazon/aws/links/base_aws.py b/providers/src/airflow/providers/amazon/aws/links/base_aws.py similarity index 100% rename from airflow/providers/amazon/aws/links/base_aws.py rename to providers/src/airflow/providers/amazon/aws/links/base_aws.py diff --git a/airflow/providers/amazon/aws/links/batch.py b/providers/src/airflow/providers/amazon/aws/links/batch.py similarity index 100% rename from airflow/providers/amazon/aws/links/batch.py rename to providers/src/airflow/providers/amazon/aws/links/batch.py diff --git a/airflow/providers/amazon/aws/links/emr.py b/providers/src/airflow/providers/amazon/aws/links/emr.py similarity index 100% rename from airflow/providers/amazon/aws/links/emr.py rename to providers/src/airflow/providers/amazon/aws/links/emr.py diff --git a/airflow/providers/amazon/aws/links/glue.py b/providers/src/airflow/providers/amazon/aws/links/glue.py similarity index 100% rename from airflow/providers/amazon/aws/links/glue.py rename to providers/src/airflow/providers/amazon/aws/links/glue.py diff --git a/airflow/providers/amazon/aws/links/logs.py b/providers/src/airflow/providers/amazon/aws/links/logs.py similarity index 100% rename from airflow/providers/amazon/aws/links/logs.py rename to providers/src/airflow/providers/amazon/aws/links/logs.py diff --git a/airflow/providers/amazon/aws/links/step_function.py b/providers/src/airflow/providers/amazon/aws/links/step_function.py similarity index 100% rename from airflow/providers/amazon/aws/links/step_function.py rename to providers/src/airflow/providers/amazon/aws/links/step_function.py diff --git a/airflow/providers/amazon/aws/secrets/__init__.py b/providers/src/airflow/providers/amazon/aws/log/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/secrets/__init__.py rename to providers/src/airflow/providers/amazon/aws/log/__init__.py diff --git a/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py b/providers/src/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py similarity index 100% rename from airflow/providers/amazon/aws/log/cloudwatch_task_handler.py rename to providers/src/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py diff --git a/airflow/providers/amazon/aws/log/s3_task_handler.py b/providers/src/airflow/providers/amazon/aws/log/s3_task_handler.py similarity index 100% rename from airflow/providers/amazon/aws/log/s3_task_handler.py rename to providers/src/airflow/providers/amazon/aws/log/s3_task_handler.py diff --git a/airflow/providers/amazon/aws/sensors/__init__.py b/providers/src/airflow/providers/amazon/aws/notifications/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/__init__.py rename to providers/src/airflow/providers/amazon/aws/notifications/__init__.py diff --git a/airflow/providers/amazon/aws/notifications/chime.py b/providers/src/airflow/providers/amazon/aws/notifications/chime.py similarity index 100% rename from airflow/providers/amazon/aws/notifications/chime.py rename to providers/src/airflow/providers/amazon/aws/notifications/chime.py diff --git a/airflow/providers/amazon/aws/notifications/sns.py b/providers/src/airflow/providers/amazon/aws/notifications/sns.py similarity index 100% rename from airflow/providers/amazon/aws/notifications/sns.py rename to providers/src/airflow/providers/amazon/aws/notifications/sns.py diff --git a/airflow/providers/amazon/aws/notifications/sqs.py b/providers/src/airflow/providers/amazon/aws/notifications/sqs.py similarity index 100% rename from airflow/providers/amazon/aws/notifications/sqs.py rename to providers/src/airflow/providers/amazon/aws/notifications/sqs.py diff --git a/airflow/providers/amazon/aws/transfers/__init__.py b/providers/src/airflow/providers/amazon/aws/operators/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/__init__.py rename to providers/src/airflow/providers/amazon/aws/operators/__init__.py diff --git a/airflow/providers/amazon/aws/operators/appflow.py b/providers/src/airflow/providers/amazon/aws/operators/appflow.py similarity index 100% rename from airflow/providers/amazon/aws/operators/appflow.py rename to providers/src/airflow/providers/amazon/aws/operators/appflow.py diff --git a/airflow/providers/amazon/aws/operators/athena.py b/providers/src/airflow/providers/amazon/aws/operators/athena.py similarity index 100% rename from airflow/providers/amazon/aws/operators/athena.py rename to providers/src/airflow/providers/amazon/aws/operators/athena.py diff --git a/airflow/providers/amazon/aws/operators/base_aws.py b/providers/src/airflow/providers/amazon/aws/operators/base_aws.py similarity index 100% rename from airflow/providers/amazon/aws/operators/base_aws.py rename to providers/src/airflow/providers/amazon/aws/operators/base_aws.py diff --git a/airflow/providers/amazon/aws/operators/batch.py b/providers/src/airflow/providers/amazon/aws/operators/batch.py similarity index 100% rename from airflow/providers/amazon/aws/operators/batch.py rename to providers/src/airflow/providers/amazon/aws/operators/batch.py diff --git a/airflow/providers/amazon/aws/operators/bedrock.py b/providers/src/airflow/providers/amazon/aws/operators/bedrock.py similarity index 100% rename from airflow/providers/amazon/aws/operators/bedrock.py rename to providers/src/airflow/providers/amazon/aws/operators/bedrock.py diff --git a/airflow/providers/amazon/aws/operators/cloud_formation.py b/providers/src/airflow/providers/amazon/aws/operators/cloud_formation.py similarity index 100% rename from airflow/providers/amazon/aws/operators/cloud_formation.py rename to providers/src/airflow/providers/amazon/aws/operators/cloud_formation.py diff --git a/airflow/providers/amazon/aws/operators/comprehend.py b/providers/src/airflow/providers/amazon/aws/operators/comprehend.py similarity index 100% rename from airflow/providers/amazon/aws/operators/comprehend.py rename to providers/src/airflow/providers/amazon/aws/operators/comprehend.py diff --git a/airflow/providers/amazon/aws/operators/datasync.py b/providers/src/airflow/providers/amazon/aws/operators/datasync.py similarity index 100% rename from airflow/providers/amazon/aws/operators/datasync.py rename to providers/src/airflow/providers/amazon/aws/operators/datasync.py diff --git a/airflow/providers/amazon/aws/operators/dms.py b/providers/src/airflow/providers/amazon/aws/operators/dms.py similarity index 100% rename from airflow/providers/amazon/aws/operators/dms.py rename to providers/src/airflow/providers/amazon/aws/operators/dms.py diff --git a/airflow/providers/amazon/aws/operators/ec2.py b/providers/src/airflow/providers/amazon/aws/operators/ec2.py similarity index 100% rename from airflow/providers/amazon/aws/operators/ec2.py rename to providers/src/airflow/providers/amazon/aws/operators/ec2.py diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/providers/src/airflow/providers/amazon/aws/operators/ecs.py similarity index 100% rename from airflow/providers/amazon/aws/operators/ecs.py rename to providers/src/airflow/providers/amazon/aws/operators/ecs.py diff --git a/airflow/providers/amazon/aws/operators/eks.py b/providers/src/airflow/providers/amazon/aws/operators/eks.py similarity index 100% rename from airflow/providers/amazon/aws/operators/eks.py rename to providers/src/airflow/providers/amazon/aws/operators/eks.py diff --git a/airflow/providers/amazon/aws/operators/emr.py b/providers/src/airflow/providers/amazon/aws/operators/emr.py similarity index 100% rename from airflow/providers/amazon/aws/operators/emr.py rename to providers/src/airflow/providers/amazon/aws/operators/emr.py diff --git a/airflow/providers/amazon/aws/operators/eventbridge.py b/providers/src/airflow/providers/amazon/aws/operators/eventbridge.py similarity index 100% rename from airflow/providers/amazon/aws/operators/eventbridge.py rename to providers/src/airflow/providers/amazon/aws/operators/eventbridge.py diff --git a/airflow/providers/amazon/aws/operators/glacier.py b/providers/src/airflow/providers/amazon/aws/operators/glacier.py similarity index 100% rename from airflow/providers/amazon/aws/operators/glacier.py rename to providers/src/airflow/providers/amazon/aws/operators/glacier.py diff --git a/airflow/providers/amazon/aws/operators/glue.py b/providers/src/airflow/providers/amazon/aws/operators/glue.py similarity index 100% rename from airflow/providers/amazon/aws/operators/glue.py rename to providers/src/airflow/providers/amazon/aws/operators/glue.py diff --git a/airflow/providers/amazon/aws/operators/glue_crawler.py b/providers/src/airflow/providers/amazon/aws/operators/glue_crawler.py similarity index 100% rename from airflow/providers/amazon/aws/operators/glue_crawler.py rename to providers/src/airflow/providers/amazon/aws/operators/glue_crawler.py diff --git a/airflow/providers/amazon/aws/operators/glue_databrew.py b/providers/src/airflow/providers/amazon/aws/operators/glue_databrew.py similarity index 100% rename from airflow/providers/amazon/aws/operators/glue_databrew.py rename to providers/src/airflow/providers/amazon/aws/operators/glue_databrew.py diff --git a/airflow/providers/amazon/aws/operators/kinesis_analytics.py b/providers/src/airflow/providers/amazon/aws/operators/kinesis_analytics.py similarity index 100% rename from airflow/providers/amazon/aws/operators/kinesis_analytics.py rename to providers/src/airflow/providers/amazon/aws/operators/kinesis_analytics.py diff --git a/airflow/providers/amazon/aws/operators/lambda_function.py b/providers/src/airflow/providers/amazon/aws/operators/lambda_function.py similarity index 100% rename from airflow/providers/amazon/aws/operators/lambda_function.py rename to providers/src/airflow/providers/amazon/aws/operators/lambda_function.py diff --git a/airflow/providers/amazon/aws/operators/neptune.py b/providers/src/airflow/providers/amazon/aws/operators/neptune.py similarity index 100% rename from airflow/providers/amazon/aws/operators/neptune.py rename to providers/src/airflow/providers/amazon/aws/operators/neptune.py diff --git a/airflow/providers/amazon/aws/operators/quicksight.py b/providers/src/airflow/providers/amazon/aws/operators/quicksight.py similarity index 100% rename from airflow/providers/amazon/aws/operators/quicksight.py rename to providers/src/airflow/providers/amazon/aws/operators/quicksight.py diff --git a/airflow/providers/amazon/aws/operators/rds.py b/providers/src/airflow/providers/amazon/aws/operators/rds.py similarity index 100% rename from airflow/providers/amazon/aws/operators/rds.py rename to providers/src/airflow/providers/amazon/aws/operators/rds.py diff --git a/airflow/providers/amazon/aws/operators/redshift_cluster.py b/providers/src/airflow/providers/amazon/aws/operators/redshift_cluster.py similarity index 100% rename from airflow/providers/amazon/aws/operators/redshift_cluster.py rename to providers/src/airflow/providers/amazon/aws/operators/redshift_cluster.py diff --git a/airflow/providers/amazon/aws/operators/redshift_data.py b/providers/src/airflow/providers/amazon/aws/operators/redshift_data.py similarity index 100% rename from airflow/providers/amazon/aws/operators/redshift_data.py rename to providers/src/airflow/providers/amazon/aws/operators/redshift_data.py diff --git a/airflow/providers/amazon/aws/operators/s3.py b/providers/src/airflow/providers/amazon/aws/operators/s3.py similarity index 100% rename from airflow/providers/amazon/aws/operators/s3.py rename to providers/src/airflow/providers/amazon/aws/operators/s3.py diff --git a/airflow/providers/amazon/aws/operators/sagemaker.py b/providers/src/airflow/providers/amazon/aws/operators/sagemaker.py similarity index 100% rename from airflow/providers/amazon/aws/operators/sagemaker.py rename to providers/src/airflow/providers/amazon/aws/operators/sagemaker.py diff --git a/airflow/providers/amazon/aws/operators/sns.py b/providers/src/airflow/providers/amazon/aws/operators/sns.py similarity index 100% rename from airflow/providers/amazon/aws/operators/sns.py rename to providers/src/airflow/providers/amazon/aws/operators/sns.py diff --git a/airflow/providers/amazon/aws/operators/sqs.py b/providers/src/airflow/providers/amazon/aws/operators/sqs.py similarity index 100% rename from airflow/providers/amazon/aws/operators/sqs.py rename to providers/src/airflow/providers/amazon/aws/operators/sqs.py diff --git a/airflow/providers/amazon/aws/operators/step_function.py b/providers/src/airflow/providers/amazon/aws/operators/step_function.py similarity index 100% rename from airflow/providers/amazon/aws/operators/step_function.py rename to providers/src/airflow/providers/amazon/aws/operators/step_function.py diff --git a/airflow/providers/amazon/aws/waiters/__init__.py b/providers/src/airflow/providers/amazon/aws/secrets/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/waiters/__init__.py rename to providers/src/airflow/providers/amazon/aws/secrets/__init__.py diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/providers/src/airflow/providers/amazon/aws/secrets/secrets_manager.py similarity index 100% rename from airflow/providers/amazon/aws/secrets/secrets_manager.py rename to providers/src/airflow/providers/amazon/aws/secrets/secrets_manager.py diff --git a/airflow/providers/amazon/aws/secrets/systems_manager.py b/providers/src/airflow/providers/amazon/aws/secrets/systems_manager.py similarity index 100% rename from airflow/providers/amazon/aws/secrets/systems_manager.py rename to providers/src/airflow/providers/amazon/aws/secrets/systems_manager.py diff --git a/airflow/providers/apache/__init__.py b/providers/src/airflow/providers/amazon/aws/sensors/__init__.py similarity index 100% rename from airflow/providers/apache/__init__.py rename to providers/src/airflow/providers/amazon/aws/sensors/__init__.py diff --git a/airflow/providers/amazon/aws/sensors/athena.py b/providers/src/airflow/providers/amazon/aws/sensors/athena.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/athena.py rename to providers/src/airflow/providers/amazon/aws/sensors/athena.py diff --git a/airflow/providers/amazon/aws/sensors/base_aws.py b/providers/src/airflow/providers/amazon/aws/sensors/base_aws.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/base_aws.py rename to providers/src/airflow/providers/amazon/aws/sensors/base_aws.py diff --git a/airflow/providers/amazon/aws/sensors/batch.py b/providers/src/airflow/providers/amazon/aws/sensors/batch.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/batch.py rename to providers/src/airflow/providers/amazon/aws/sensors/batch.py diff --git a/airflow/providers/amazon/aws/sensors/bedrock.py b/providers/src/airflow/providers/amazon/aws/sensors/bedrock.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/bedrock.py rename to providers/src/airflow/providers/amazon/aws/sensors/bedrock.py diff --git a/airflow/providers/amazon/aws/sensors/cloud_formation.py b/providers/src/airflow/providers/amazon/aws/sensors/cloud_formation.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/cloud_formation.py rename to providers/src/airflow/providers/amazon/aws/sensors/cloud_formation.py diff --git a/airflow/providers/amazon/aws/sensors/comprehend.py b/providers/src/airflow/providers/amazon/aws/sensors/comprehend.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/comprehend.py rename to providers/src/airflow/providers/amazon/aws/sensors/comprehend.py diff --git a/airflow/providers/amazon/aws/sensors/dms.py b/providers/src/airflow/providers/amazon/aws/sensors/dms.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/dms.py rename to providers/src/airflow/providers/amazon/aws/sensors/dms.py diff --git a/airflow/providers/amazon/aws/sensors/dynamodb.py b/providers/src/airflow/providers/amazon/aws/sensors/dynamodb.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/dynamodb.py rename to providers/src/airflow/providers/amazon/aws/sensors/dynamodb.py diff --git a/airflow/providers/amazon/aws/sensors/ec2.py b/providers/src/airflow/providers/amazon/aws/sensors/ec2.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/ec2.py rename to providers/src/airflow/providers/amazon/aws/sensors/ec2.py diff --git a/airflow/providers/amazon/aws/sensors/ecs.py b/providers/src/airflow/providers/amazon/aws/sensors/ecs.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/ecs.py rename to providers/src/airflow/providers/amazon/aws/sensors/ecs.py diff --git a/airflow/providers/amazon/aws/sensors/eks.py b/providers/src/airflow/providers/amazon/aws/sensors/eks.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/eks.py rename to providers/src/airflow/providers/amazon/aws/sensors/eks.py diff --git a/airflow/providers/amazon/aws/sensors/emr.py b/providers/src/airflow/providers/amazon/aws/sensors/emr.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/emr.py rename to providers/src/airflow/providers/amazon/aws/sensors/emr.py diff --git a/airflow/providers/amazon/aws/sensors/glacier.py b/providers/src/airflow/providers/amazon/aws/sensors/glacier.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/glacier.py rename to providers/src/airflow/providers/amazon/aws/sensors/glacier.py diff --git a/airflow/providers/amazon/aws/sensors/glue.py b/providers/src/airflow/providers/amazon/aws/sensors/glue.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/glue.py rename to providers/src/airflow/providers/amazon/aws/sensors/glue.py diff --git a/airflow/providers/amazon/aws/sensors/glue_catalog_partition.py b/providers/src/airflow/providers/amazon/aws/sensors/glue_catalog_partition.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/glue_catalog_partition.py rename to providers/src/airflow/providers/amazon/aws/sensors/glue_catalog_partition.py diff --git a/airflow/providers/amazon/aws/sensors/glue_crawler.py b/providers/src/airflow/providers/amazon/aws/sensors/glue_crawler.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/glue_crawler.py rename to providers/src/airflow/providers/amazon/aws/sensors/glue_crawler.py diff --git a/airflow/providers/amazon/aws/sensors/kinesis_analytics.py b/providers/src/airflow/providers/amazon/aws/sensors/kinesis_analytics.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/kinesis_analytics.py rename to providers/src/airflow/providers/amazon/aws/sensors/kinesis_analytics.py diff --git a/airflow/providers/amazon/aws/sensors/lambda_function.py b/providers/src/airflow/providers/amazon/aws/sensors/lambda_function.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/lambda_function.py rename to providers/src/airflow/providers/amazon/aws/sensors/lambda_function.py diff --git a/airflow/providers/amazon/aws/sensors/opensearch_serverless.py b/providers/src/airflow/providers/amazon/aws/sensors/opensearch_serverless.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/opensearch_serverless.py rename to providers/src/airflow/providers/amazon/aws/sensors/opensearch_serverless.py diff --git a/airflow/providers/amazon/aws/sensors/quicksight.py b/providers/src/airflow/providers/amazon/aws/sensors/quicksight.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/quicksight.py rename to providers/src/airflow/providers/amazon/aws/sensors/quicksight.py diff --git a/airflow/providers/amazon/aws/sensors/rds.py b/providers/src/airflow/providers/amazon/aws/sensors/rds.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/rds.py rename to providers/src/airflow/providers/amazon/aws/sensors/rds.py diff --git a/airflow/providers/amazon/aws/sensors/redshift_cluster.py b/providers/src/airflow/providers/amazon/aws/sensors/redshift_cluster.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/redshift_cluster.py rename to providers/src/airflow/providers/amazon/aws/sensors/redshift_cluster.py diff --git a/airflow/providers/amazon/aws/sensors/s3.py b/providers/src/airflow/providers/amazon/aws/sensors/s3.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/s3.py rename to providers/src/airflow/providers/amazon/aws/sensors/s3.py diff --git a/airflow/providers/amazon/aws/sensors/sagemaker.py b/providers/src/airflow/providers/amazon/aws/sensors/sagemaker.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/sagemaker.py rename to providers/src/airflow/providers/amazon/aws/sensors/sagemaker.py diff --git a/airflow/providers/amazon/aws/sensors/sqs.py b/providers/src/airflow/providers/amazon/aws/sensors/sqs.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/sqs.py rename to providers/src/airflow/providers/amazon/aws/sensors/sqs.py diff --git a/airflow/providers/amazon/aws/sensors/step_function.py b/providers/src/airflow/providers/amazon/aws/sensors/step_function.py similarity index 100% rename from airflow/providers/amazon/aws/sensors/step_function.py rename to providers/src/airflow/providers/amazon/aws/sensors/step_function.py diff --git a/airflow/providers/apache/beam/triggers/__init__.py b/providers/src/airflow/providers/amazon/aws/transfers/__init__.py similarity index 100% rename from airflow/providers/apache/beam/triggers/__init__.py rename to providers/src/airflow/providers/amazon/aws/transfers/__init__.py diff --git a/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/base.py b/providers/src/airflow/providers/amazon/aws/transfers/base.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/base.py rename to providers/src/airflow/providers/amazon/aws/transfers/base.py diff --git a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/exasol_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/exasol_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/exasol_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/exasol_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/ftp_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/ftp_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/ftp_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/ftp_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/gcs_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/glacier_to_gcs.py b/providers/src/airflow/providers/amazon/aws/transfers/glacier_to_gcs.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/glacier_to_gcs.py rename to providers/src/airflow/providers/amazon/aws/transfers/glacier_to_gcs.py diff --git a/airflow/providers/amazon/aws/transfers/google_api_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/google_api_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/google_api_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/google_api_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py b/providers/src/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py rename to providers/src/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py diff --git a/airflow/providers/amazon/aws/transfers/http_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/http_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/http_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/http_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/local_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/local_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/local_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/local_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/mongo_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/mongo_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/mongo_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/mongo_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/redshift_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/redshift_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/redshift_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/redshift_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py b/providers/src/airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py rename to providers/src/airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/providers/src/airflow/providers/amazon/aws/transfers/s3_to_ftp.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/s3_to_ftp.py rename to providers/src/airflow/providers/amazon/aws/transfers/s3_to_ftp.py diff --git a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py b/providers/src/airflow/providers/amazon/aws/transfers/s3_to_redshift.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/s3_to_redshift.py rename to providers/src/airflow/providers/amazon/aws/transfers/s3_to_redshift.py diff --git a/airflow/providers/amazon/aws/transfers/s3_to_sftp.py b/providers/src/airflow/providers/amazon/aws/transfers/s3_to_sftp.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/s3_to_sftp.py rename to providers/src/airflow/providers/amazon/aws/transfers/s3_to_sftp.py diff --git a/airflow/providers/amazon/aws/transfers/s3_to_sql.py b/providers/src/airflow/providers/amazon/aws/transfers/s3_to_sql.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/s3_to_sql.py rename to providers/src/airflow/providers/amazon/aws/transfers/s3_to_sql.py diff --git a/airflow/providers/amazon/aws/transfers/salesforce_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/salesforce_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/salesforce_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/salesforce_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/sftp_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/sftp_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/sftp_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/sftp_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/sql_to_s3.py b/providers/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py similarity index 100% rename from airflow/providers/amazon/aws/transfers/sql_to_s3.py rename to providers/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py diff --git a/airflow/providers/amazon/aws/triggers/README.md b/providers/src/airflow/providers/amazon/aws/triggers/README.md similarity index 100% rename from airflow/providers/amazon/aws/triggers/README.md rename to providers/src/airflow/providers/amazon/aws/triggers/README.md diff --git a/airflow/providers/amazon/aws/triggers/__init__.py b/providers/src/airflow/providers/amazon/aws/triggers/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/__init__.py rename to providers/src/airflow/providers/amazon/aws/triggers/__init__.py diff --git a/airflow/providers/amazon/aws/triggers/athena.py b/providers/src/airflow/providers/amazon/aws/triggers/athena.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/athena.py rename to providers/src/airflow/providers/amazon/aws/triggers/athena.py diff --git a/airflow/providers/amazon/aws/triggers/base.py b/providers/src/airflow/providers/amazon/aws/triggers/base.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/base.py rename to providers/src/airflow/providers/amazon/aws/triggers/base.py diff --git a/airflow/providers/amazon/aws/triggers/batch.py b/providers/src/airflow/providers/amazon/aws/triggers/batch.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/batch.py rename to providers/src/airflow/providers/amazon/aws/triggers/batch.py diff --git a/airflow/providers/amazon/aws/triggers/bedrock.py b/providers/src/airflow/providers/amazon/aws/triggers/bedrock.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/bedrock.py rename to providers/src/airflow/providers/amazon/aws/triggers/bedrock.py diff --git a/airflow/providers/amazon/aws/triggers/comprehend.py b/providers/src/airflow/providers/amazon/aws/triggers/comprehend.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/comprehend.py rename to providers/src/airflow/providers/amazon/aws/triggers/comprehend.py diff --git a/airflow/providers/amazon/aws/triggers/ec2.py b/providers/src/airflow/providers/amazon/aws/triggers/ec2.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/ec2.py rename to providers/src/airflow/providers/amazon/aws/triggers/ec2.py diff --git a/airflow/providers/amazon/aws/triggers/ecs.py b/providers/src/airflow/providers/amazon/aws/triggers/ecs.py similarity index 96% rename from airflow/providers/amazon/aws/triggers/ecs.py rename to providers/src/airflow/providers/amazon/aws/triggers/ecs.py index dd86899f22006..11c9cf18043c3 100644 --- a/airflow/providers/amazon/aws/triggers/ecs.py +++ b/providers/src/airflow/providers/amazon/aws/triggers/ecs.py @@ -18,7 +18,8 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING, Any, AsyncIterator +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING, Any from botocore.exceptions import ClientError, WaiterError @@ -165,11 +166,10 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - async with EcsHook( - aws_conn_id=self.aws_conn_id, region_name=self.region - ).async_conn as ecs_client, AwsLogsHook( - aws_conn_id=self.aws_conn_id, region_name=self.region - ).async_conn as logs_client: + async with ( + EcsHook(aws_conn_id=self.aws_conn_id, region_name=self.region).async_conn as ecs_client, + AwsLogsHook(aws_conn_id=self.aws_conn_id, region_name=self.region).async_conn as logs_client, + ): waiter = ecs_client.get_waiter("tasks_stopped") logs_token = None while self.waiter_max_attempts: diff --git a/airflow/providers/amazon/aws/triggers/eks.py b/providers/src/airflow/providers/amazon/aws/triggers/eks.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/eks.py rename to providers/src/airflow/providers/amazon/aws/triggers/eks.py diff --git a/airflow/providers/amazon/aws/triggers/emr.py b/providers/src/airflow/providers/amazon/aws/triggers/emr.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/emr.py rename to providers/src/airflow/providers/amazon/aws/triggers/emr.py diff --git a/airflow/providers/amazon/aws/triggers/glue.py b/providers/src/airflow/providers/amazon/aws/triggers/glue.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/glue.py rename to providers/src/airflow/providers/amazon/aws/triggers/glue.py diff --git a/airflow/providers/amazon/aws/triggers/glue_crawler.py b/providers/src/airflow/providers/amazon/aws/triggers/glue_crawler.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/glue_crawler.py rename to providers/src/airflow/providers/amazon/aws/triggers/glue_crawler.py diff --git a/airflow/providers/amazon/aws/triggers/glue_databrew.py b/providers/src/airflow/providers/amazon/aws/triggers/glue_databrew.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/glue_databrew.py rename to providers/src/airflow/providers/amazon/aws/triggers/glue_databrew.py diff --git a/airflow/providers/amazon/aws/triggers/kinesis_analytics.py b/providers/src/airflow/providers/amazon/aws/triggers/kinesis_analytics.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/kinesis_analytics.py rename to providers/src/airflow/providers/amazon/aws/triggers/kinesis_analytics.py diff --git a/airflow/providers/amazon/aws/triggers/lambda_function.py b/providers/src/airflow/providers/amazon/aws/triggers/lambda_function.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/lambda_function.py rename to providers/src/airflow/providers/amazon/aws/triggers/lambda_function.py diff --git a/airflow/providers/amazon/aws/triggers/neptune.py b/providers/src/airflow/providers/amazon/aws/triggers/neptune.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/neptune.py rename to providers/src/airflow/providers/amazon/aws/triggers/neptune.py diff --git a/airflow/providers/amazon/aws/triggers/opensearch_serverless.py b/providers/src/airflow/providers/amazon/aws/triggers/opensearch_serverless.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/opensearch_serverless.py rename to providers/src/airflow/providers/amazon/aws/triggers/opensearch_serverless.py diff --git a/airflow/providers/amazon/aws/triggers/rds.py b/providers/src/airflow/providers/amazon/aws/triggers/rds.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/rds.py rename to providers/src/airflow/providers/amazon/aws/triggers/rds.py diff --git a/airflow/providers/amazon/aws/triggers/redshift_cluster.py b/providers/src/airflow/providers/amazon/aws/triggers/redshift_cluster.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/redshift_cluster.py rename to providers/src/airflow/providers/amazon/aws/triggers/redshift_cluster.py diff --git a/airflow/providers/amazon/aws/triggers/redshift_data.py b/providers/src/airflow/providers/amazon/aws/triggers/redshift_data.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/redshift_data.py rename to providers/src/airflow/providers/amazon/aws/triggers/redshift_data.py diff --git a/airflow/providers/amazon/aws/triggers/s3.py b/providers/src/airflow/providers/amazon/aws/triggers/s3.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/s3.py rename to providers/src/airflow/providers/amazon/aws/triggers/s3.py diff --git a/airflow/providers/amazon/aws/triggers/sagemaker.py b/providers/src/airflow/providers/amazon/aws/triggers/sagemaker.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/sagemaker.py rename to providers/src/airflow/providers/amazon/aws/triggers/sagemaker.py diff --git a/airflow/providers/amazon/aws/triggers/sqs.py b/providers/src/airflow/providers/amazon/aws/triggers/sqs.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/sqs.py rename to providers/src/airflow/providers/amazon/aws/triggers/sqs.py diff --git a/airflow/providers/amazon/aws/triggers/step_function.py b/providers/src/airflow/providers/amazon/aws/triggers/step_function.py similarity index 100% rename from airflow/providers/amazon/aws/triggers/step_function.py rename to providers/src/airflow/providers/amazon/aws/triggers/step_function.py diff --git a/airflow/providers/amazon/aws/utils/__init__.py b/providers/src/airflow/providers/amazon/aws/utils/__init__.py similarity index 100% rename from airflow/providers/amazon/aws/utils/__init__.py rename to providers/src/airflow/providers/amazon/aws/utils/__init__.py diff --git a/airflow/providers/amazon/aws/utils/asset_compat_lineage_collector.py b/providers/src/airflow/providers/amazon/aws/utils/asset_compat_lineage_collector.py similarity index 100% rename from airflow/providers/amazon/aws/utils/asset_compat_lineage_collector.py rename to providers/src/airflow/providers/amazon/aws/utils/asset_compat_lineage_collector.py diff --git a/airflow/providers/amazon/aws/utils/connection_wrapper.py b/providers/src/airflow/providers/amazon/aws/utils/connection_wrapper.py similarity index 100% rename from airflow/providers/amazon/aws/utils/connection_wrapper.py rename to providers/src/airflow/providers/amazon/aws/utils/connection_wrapper.py diff --git a/airflow/providers/amazon/aws/utils/eks_get_token.py b/providers/src/airflow/providers/amazon/aws/utils/eks_get_token.py similarity index 100% rename from airflow/providers/amazon/aws/utils/eks_get_token.py rename to providers/src/airflow/providers/amazon/aws/utils/eks_get_token.py diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/providers/src/airflow/providers/amazon/aws/utils/emailer.py similarity index 100% rename from airflow/providers/amazon/aws/utils/emailer.py rename to providers/src/airflow/providers/amazon/aws/utils/emailer.py diff --git a/airflow/providers/amazon/aws/utils/identifiers.py b/providers/src/airflow/providers/amazon/aws/utils/identifiers.py similarity index 100% rename from airflow/providers/amazon/aws/utils/identifiers.py rename to providers/src/airflow/providers/amazon/aws/utils/identifiers.py diff --git a/airflow/providers/amazon/aws/utils/mixins.py b/providers/src/airflow/providers/amazon/aws/utils/mixins.py similarity index 100% rename from airflow/providers/amazon/aws/utils/mixins.py rename to providers/src/airflow/providers/amazon/aws/utils/mixins.py diff --git a/airflow/providers/amazon/aws/utils/openlineage.py b/providers/src/airflow/providers/amazon/aws/utils/openlineage.py similarity index 100% rename from airflow/providers/amazon/aws/utils/openlineage.py rename to providers/src/airflow/providers/amazon/aws/utils/openlineage.py diff --git a/airflow/providers/amazon/aws/utils/rds.py b/providers/src/airflow/providers/amazon/aws/utils/rds.py similarity index 100% rename from airflow/providers/amazon/aws/utils/rds.py rename to providers/src/airflow/providers/amazon/aws/utils/rds.py diff --git a/airflow/providers/amazon/aws/utils/redshift.py b/providers/src/airflow/providers/amazon/aws/utils/redshift.py similarity index 100% rename from airflow/providers/amazon/aws/utils/redshift.py rename to providers/src/airflow/providers/amazon/aws/utils/redshift.py diff --git a/airflow/providers/amazon/aws/utils/sagemaker.py b/providers/src/airflow/providers/amazon/aws/utils/sagemaker.py similarity index 100% rename from airflow/providers/amazon/aws/utils/sagemaker.py rename to providers/src/airflow/providers/amazon/aws/utils/sagemaker.py diff --git a/airflow/providers/amazon/aws/utils/sqs.py b/providers/src/airflow/providers/amazon/aws/utils/sqs.py similarity index 100% rename from airflow/providers/amazon/aws/utils/sqs.py rename to providers/src/airflow/providers/amazon/aws/utils/sqs.py diff --git a/airflow/providers/amazon/aws/utils/suppress.py b/providers/src/airflow/providers/amazon/aws/utils/suppress.py similarity index 100% rename from airflow/providers/amazon/aws/utils/suppress.py rename to providers/src/airflow/providers/amazon/aws/utils/suppress.py diff --git a/airflow/providers/amazon/aws/utils/tags.py b/providers/src/airflow/providers/amazon/aws/utils/tags.py similarity index 100% rename from airflow/providers/amazon/aws/utils/tags.py rename to providers/src/airflow/providers/amazon/aws/utils/tags.py diff --git a/airflow/providers/amazon/aws/utils/task_log_fetcher.py b/providers/src/airflow/providers/amazon/aws/utils/task_log_fetcher.py similarity index 100% rename from airflow/providers/amazon/aws/utils/task_log_fetcher.py rename to providers/src/airflow/providers/amazon/aws/utils/task_log_fetcher.py diff --git a/airflow/providers/amazon/aws/utils/waiter.py b/providers/src/airflow/providers/amazon/aws/utils/waiter.py similarity index 100% rename from airflow/providers/amazon/aws/utils/waiter.py rename to providers/src/airflow/providers/amazon/aws/utils/waiter.py diff --git a/airflow/providers/amazon/aws/utils/waiter_with_logging.py b/providers/src/airflow/providers/amazon/aws/utils/waiter_with_logging.py similarity index 100% rename from airflow/providers/amazon/aws/utils/waiter_with_logging.py rename to providers/src/airflow/providers/amazon/aws/utils/waiter_with_logging.py diff --git a/airflow/providers/amazon/aws/waiters/README.md b/providers/src/airflow/providers/amazon/aws/waiters/README.md similarity index 100% rename from airflow/providers/amazon/aws/waiters/README.md rename to providers/src/airflow/providers/amazon/aws/waiters/README.md diff --git a/airflow/providers/apache/cassandra/hooks/__init__.py b/providers/src/airflow/providers/amazon/aws/waiters/__init__.py similarity index 100% rename from airflow/providers/apache/cassandra/hooks/__init__.py rename to providers/src/airflow/providers/amazon/aws/waiters/__init__.py diff --git a/airflow/providers/amazon/aws/waiters/appflow.json b/providers/src/airflow/providers/amazon/aws/waiters/appflow.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/appflow.json rename to providers/src/airflow/providers/amazon/aws/waiters/appflow.json diff --git a/airflow/providers/amazon/aws/waiters/athena.json b/providers/src/airflow/providers/amazon/aws/waiters/athena.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/athena.json rename to providers/src/airflow/providers/amazon/aws/waiters/athena.json diff --git a/airflow/providers/amazon/aws/waiters/base_waiter.py b/providers/src/airflow/providers/amazon/aws/waiters/base_waiter.py similarity index 100% rename from airflow/providers/amazon/aws/waiters/base_waiter.py rename to providers/src/airflow/providers/amazon/aws/waiters/base_waiter.py diff --git a/airflow/providers/amazon/aws/waiters/batch.json b/providers/src/airflow/providers/amazon/aws/waiters/batch.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/batch.json rename to providers/src/airflow/providers/amazon/aws/waiters/batch.json diff --git a/airflow/providers/amazon/aws/waiters/bedrock-agent.json b/providers/src/airflow/providers/amazon/aws/waiters/bedrock-agent.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/bedrock-agent.json rename to providers/src/airflow/providers/amazon/aws/waiters/bedrock-agent.json diff --git a/airflow/providers/amazon/aws/waiters/bedrock.json b/providers/src/airflow/providers/amazon/aws/waiters/bedrock.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/bedrock.json rename to providers/src/airflow/providers/amazon/aws/waiters/bedrock.json diff --git a/airflow/providers/amazon/aws/waiters/comprehend.json b/providers/src/airflow/providers/amazon/aws/waiters/comprehend.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/comprehend.json rename to providers/src/airflow/providers/amazon/aws/waiters/comprehend.json diff --git a/airflow/providers/amazon/aws/waiters/databrew.json b/providers/src/airflow/providers/amazon/aws/waiters/databrew.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/databrew.json rename to providers/src/airflow/providers/amazon/aws/waiters/databrew.json diff --git a/airflow/providers/amazon/aws/waiters/dynamodb.json b/providers/src/airflow/providers/amazon/aws/waiters/dynamodb.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/dynamodb.json rename to providers/src/airflow/providers/amazon/aws/waiters/dynamodb.json diff --git a/airflow/providers/amazon/aws/waiters/ecs.json b/providers/src/airflow/providers/amazon/aws/waiters/ecs.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/ecs.json rename to providers/src/airflow/providers/amazon/aws/waiters/ecs.json diff --git a/airflow/providers/amazon/aws/waiters/eks.json b/providers/src/airflow/providers/amazon/aws/waiters/eks.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/eks.json rename to providers/src/airflow/providers/amazon/aws/waiters/eks.json diff --git a/airflow/providers/amazon/aws/waiters/emr-containers.json b/providers/src/airflow/providers/amazon/aws/waiters/emr-containers.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/emr-containers.json rename to providers/src/airflow/providers/amazon/aws/waiters/emr-containers.json diff --git a/airflow/providers/amazon/aws/waiters/emr-serverless.json b/providers/src/airflow/providers/amazon/aws/waiters/emr-serverless.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/emr-serverless.json rename to providers/src/airflow/providers/amazon/aws/waiters/emr-serverless.json diff --git a/airflow/providers/amazon/aws/waiters/emr.json b/providers/src/airflow/providers/amazon/aws/waiters/emr.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/emr.json rename to providers/src/airflow/providers/amazon/aws/waiters/emr.json diff --git a/airflow/providers/amazon/aws/waiters/glue.json b/providers/src/airflow/providers/amazon/aws/waiters/glue.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/glue.json rename to providers/src/airflow/providers/amazon/aws/waiters/glue.json diff --git a/airflow/providers/amazon/aws/waiters/kinesisanalyticsv2.json b/providers/src/airflow/providers/amazon/aws/waiters/kinesisanalyticsv2.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/kinesisanalyticsv2.json rename to providers/src/airflow/providers/amazon/aws/waiters/kinesisanalyticsv2.json diff --git a/airflow/providers/amazon/aws/waiters/neptune.json b/providers/src/airflow/providers/amazon/aws/waiters/neptune.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/neptune.json rename to providers/src/airflow/providers/amazon/aws/waiters/neptune.json diff --git a/airflow/providers/amazon/aws/waiters/opensearchserverless.json b/providers/src/airflow/providers/amazon/aws/waiters/opensearchserverless.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/opensearchserverless.json rename to providers/src/airflow/providers/amazon/aws/waiters/opensearchserverless.json diff --git a/airflow/providers/amazon/aws/waiters/rds.json b/providers/src/airflow/providers/amazon/aws/waiters/rds.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/rds.json rename to providers/src/airflow/providers/amazon/aws/waiters/rds.json diff --git a/airflow/providers/amazon/aws/waiters/redshift.json b/providers/src/airflow/providers/amazon/aws/waiters/redshift.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/redshift.json rename to providers/src/airflow/providers/amazon/aws/waiters/redshift.json diff --git a/airflow/providers/amazon/aws/waiters/sagemaker.json b/providers/src/airflow/providers/amazon/aws/waiters/sagemaker.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/sagemaker.json rename to providers/src/airflow/providers/amazon/aws/waiters/sagemaker.json diff --git a/airflow/providers/amazon/aws/waiters/stepfunctions.json b/providers/src/airflow/providers/amazon/aws/waiters/stepfunctions.json similarity index 100% rename from airflow/providers/amazon/aws/waiters/stepfunctions.json rename to providers/src/airflow/providers/amazon/aws/waiters/stepfunctions.json diff --git a/airflow/providers/amazon/provider.yaml b/providers/src/airflow/providers/amazon/provider.yaml similarity index 100% rename from airflow/providers/amazon/provider.yaml rename to providers/src/airflow/providers/amazon/provider.yaml diff --git a/airflow/providers/apache/beam/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/beam/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/beam/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/beam/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/beam/CHANGELOG.rst b/providers/src/airflow/providers/apache/beam/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/beam/CHANGELOG.rst rename to providers/src/airflow/providers/apache/beam/CHANGELOG.rst diff --git a/airflow/providers/apache/beam/README.md b/providers/src/airflow/providers/apache/beam/README.md similarity index 100% rename from airflow/providers/apache/beam/README.md rename to providers/src/airflow/providers/apache/beam/README.md diff --git a/airflow/providers/apache/beam/__init__.py b/providers/src/airflow/providers/apache/beam/__init__.py similarity index 100% rename from airflow/providers/apache/beam/__init__.py rename to providers/src/airflow/providers/apache/beam/__init__.py diff --git a/airflow/providers/apache/beam/hooks/__init__.py b/providers/src/airflow/providers/apache/beam/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/beam/hooks/__init__.py rename to providers/src/airflow/providers/apache/beam/hooks/__init__.py diff --git a/airflow/providers/apache/beam/hooks/beam.py b/providers/src/airflow/providers/apache/beam/hooks/beam.py similarity index 100% rename from airflow/providers/apache/beam/hooks/beam.py rename to providers/src/airflow/providers/apache/beam/hooks/beam.py diff --git a/airflow/providers/apache/beam/operators/__init__.py b/providers/src/airflow/providers/apache/beam/operators/__init__.py similarity index 100% rename from airflow/providers/apache/beam/operators/__init__.py rename to providers/src/airflow/providers/apache/beam/operators/__init__.py diff --git a/airflow/providers/apache/beam/operators/beam.py b/providers/src/airflow/providers/apache/beam/operators/beam.py similarity index 100% rename from airflow/providers/apache/beam/operators/beam.py rename to providers/src/airflow/providers/apache/beam/operators/beam.py diff --git a/airflow/providers/apache/beam/provider.yaml b/providers/src/airflow/providers/apache/beam/provider.yaml similarity index 100% rename from airflow/providers/apache/beam/provider.yaml rename to providers/src/airflow/providers/apache/beam/provider.yaml diff --git a/airflow/providers/apache/cassandra/sensors/__init__.py b/providers/src/airflow/providers/apache/beam/triggers/__init__.py similarity index 100% rename from airflow/providers/apache/cassandra/sensors/__init__.py rename to providers/src/airflow/providers/apache/beam/triggers/__init__.py diff --git a/airflow/providers/apache/beam/triggers/beam.py b/providers/src/airflow/providers/apache/beam/triggers/beam.py similarity index 100% rename from airflow/providers/apache/beam/triggers/beam.py rename to providers/src/airflow/providers/apache/beam/triggers/beam.py diff --git a/airflow/providers/apache/cassandra/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/cassandra/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/cassandra/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/cassandra/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/cassandra/CHANGELOG.rst b/providers/src/airflow/providers/apache/cassandra/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/cassandra/CHANGELOG.rst rename to providers/src/airflow/providers/apache/cassandra/CHANGELOG.rst diff --git a/airflow/providers/apache/cassandra/__init__.py b/providers/src/airflow/providers/apache/cassandra/__init__.py similarity index 100% rename from airflow/providers/apache/cassandra/__init__.py rename to providers/src/airflow/providers/apache/cassandra/__init__.py diff --git a/airflow/providers/apache/druid/transfers/__init__.py b/providers/src/airflow/providers/apache/cassandra/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/druid/transfers/__init__.py rename to providers/src/airflow/providers/apache/cassandra/hooks/__init__.py diff --git a/airflow/providers/apache/cassandra/hooks/cassandra.py b/providers/src/airflow/providers/apache/cassandra/hooks/cassandra.py similarity index 100% rename from airflow/providers/apache/cassandra/hooks/cassandra.py rename to providers/src/airflow/providers/apache/cassandra/hooks/cassandra.py diff --git a/airflow/providers/apache/cassandra/provider.yaml b/providers/src/airflow/providers/apache/cassandra/provider.yaml similarity index 100% rename from airflow/providers/apache/cassandra/provider.yaml rename to providers/src/airflow/providers/apache/cassandra/provider.yaml diff --git a/airflow/providers/apache/hdfs/log/__init__.py b/providers/src/airflow/providers/apache/cassandra/sensors/__init__.py similarity index 100% rename from airflow/providers/apache/hdfs/log/__init__.py rename to providers/src/airflow/providers/apache/cassandra/sensors/__init__.py diff --git a/airflow/providers/apache/cassandra/sensors/record.py b/providers/src/airflow/providers/apache/cassandra/sensors/record.py similarity index 100% rename from airflow/providers/apache/cassandra/sensors/record.py rename to providers/src/airflow/providers/apache/cassandra/sensors/record.py diff --git a/airflow/providers/apache/cassandra/sensors/table.py b/providers/src/airflow/providers/apache/cassandra/sensors/table.py similarity index 100% rename from airflow/providers/apache/cassandra/sensors/table.py rename to providers/src/airflow/providers/apache/cassandra/sensors/table.py diff --git a/airflow/providers/apache/drill/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/drill/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/drill/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/drill/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/drill/CHANGELOG.rst b/providers/src/airflow/providers/apache/drill/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/drill/CHANGELOG.rst rename to providers/src/airflow/providers/apache/drill/CHANGELOG.rst diff --git a/airflow/providers/apache/drill/__init__.py b/providers/src/airflow/providers/apache/drill/__init__.py similarity index 100% rename from airflow/providers/apache/drill/__init__.py rename to providers/src/airflow/providers/apache/drill/__init__.py diff --git a/airflow/providers/apache/drill/hooks/__init__.py b/providers/src/airflow/providers/apache/drill/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/drill/hooks/__init__.py rename to providers/src/airflow/providers/apache/drill/hooks/__init__.py diff --git a/airflow/providers/apache/drill/hooks/drill.py b/providers/src/airflow/providers/apache/drill/hooks/drill.py similarity index 100% rename from airflow/providers/apache/drill/hooks/drill.py rename to providers/src/airflow/providers/apache/drill/hooks/drill.py diff --git a/airflow/providers/apache/drill/operators/__init__.py b/providers/src/airflow/providers/apache/drill/operators/__init__.py similarity index 100% rename from airflow/providers/apache/drill/operators/__init__.py rename to providers/src/airflow/providers/apache/drill/operators/__init__.py diff --git a/airflow/providers/apache/drill/operators/drill.py b/providers/src/airflow/providers/apache/drill/operators/drill.py similarity index 100% rename from airflow/providers/apache/drill/operators/drill.py rename to providers/src/airflow/providers/apache/drill/operators/drill.py diff --git a/airflow/providers/apache/drill/provider.yaml b/providers/src/airflow/providers/apache/drill/provider.yaml similarity index 100% rename from airflow/providers/apache/drill/provider.yaml rename to providers/src/airflow/providers/apache/drill/provider.yaml diff --git a/airflow/providers/apache/druid/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/druid/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/druid/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/druid/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/druid/CHANGELOG.rst b/providers/src/airflow/providers/apache/druid/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/druid/CHANGELOG.rst rename to providers/src/airflow/providers/apache/druid/CHANGELOG.rst diff --git a/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py similarity index 100% rename from airflow/providers/apache/druid/__init__.py rename to providers/src/airflow/providers/apache/druid/__init__.py diff --git a/airflow/providers/apache/druid/hooks/__init__.py b/providers/src/airflow/providers/apache/druid/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/druid/hooks/__init__.py rename to providers/src/airflow/providers/apache/druid/hooks/__init__.py diff --git a/airflow/providers/apache/druid/hooks/druid.py b/providers/src/airflow/providers/apache/druid/hooks/druid.py similarity index 100% rename from airflow/providers/apache/druid/hooks/druid.py rename to providers/src/airflow/providers/apache/druid/hooks/druid.py diff --git a/airflow/providers/apache/druid/operators/__init__.py b/providers/src/airflow/providers/apache/druid/operators/__init__.py similarity index 100% rename from airflow/providers/apache/druid/operators/__init__.py rename to providers/src/airflow/providers/apache/druid/operators/__init__.py diff --git a/airflow/providers/apache/druid/operators/druid.py b/providers/src/airflow/providers/apache/druid/operators/druid.py similarity index 100% rename from airflow/providers/apache/druid/operators/druid.py rename to providers/src/airflow/providers/apache/druid/operators/druid.py diff --git a/airflow/providers/apache/druid/operators/druid_check.py b/providers/src/airflow/providers/apache/druid/operators/druid_check.py similarity index 100% rename from airflow/providers/apache/druid/operators/druid_check.py rename to providers/src/airflow/providers/apache/druid/operators/druid_check.py diff --git a/airflow/providers/apache/druid/provider.yaml b/providers/src/airflow/providers/apache/druid/provider.yaml similarity index 100% rename from airflow/providers/apache/druid/provider.yaml rename to providers/src/airflow/providers/apache/druid/provider.yaml diff --git a/airflow/providers/apache/hive/transfers/__init__.py b/providers/src/airflow/providers/apache/druid/transfers/__init__.py similarity index 100% rename from airflow/providers/apache/hive/transfers/__init__.py rename to providers/src/airflow/providers/apache/druid/transfers/__init__.py diff --git a/airflow/providers/apache/druid/transfers/hive_to_druid.py b/providers/src/airflow/providers/apache/druid/transfers/hive_to_druid.py similarity index 100% rename from airflow/providers/apache/druid/transfers/hive_to_druid.py rename to providers/src/airflow/providers/apache/druid/transfers/hive_to_druid.py diff --git a/airflow/providers/apache/flink/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/flink/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/flink/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/flink/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/flink/CHANGELOG.rst b/providers/src/airflow/providers/apache/flink/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/flink/CHANGELOG.rst rename to providers/src/airflow/providers/apache/flink/CHANGELOG.rst diff --git a/airflow/providers/apache/flink/__init__.py b/providers/src/airflow/providers/apache/flink/__init__.py similarity index 100% rename from airflow/providers/apache/flink/__init__.py rename to providers/src/airflow/providers/apache/flink/__init__.py diff --git a/airflow/providers/apache/flink/hooks/__init__.py b/providers/src/airflow/providers/apache/flink/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/flink/hooks/__init__.py rename to providers/src/airflow/providers/apache/flink/hooks/__init__.py diff --git a/airflow/providers/apache/flink/operators/__init__.py b/providers/src/airflow/providers/apache/flink/operators/__init__.py similarity index 100% rename from airflow/providers/apache/flink/operators/__init__.py rename to providers/src/airflow/providers/apache/flink/operators/__init__.py diff --git a/airflow/providers/apache/flink/operators/flink_kubernetes.py b/providers/src/airflow/providers/apache/flink/operators/flink_kubernetes.py similarity index 100% rename from airflow/providers/apache/flink/operators/flink_kubernetes.py rename to providers/src/airflow/providers/apache/flink/operators/flink_kubernetes.py diff --git a/airflow/providers/apache/flink/provider.yaml b/providers/src/airflow/providers/apache/flink/provider.yaml similarity index 100% rename from airflow/providers/apache/flink/provider.yaml rename to providers/src/airflow/providers/apache/flink/provider.yaml diff --git a/airflow/providers/apache/flink/sensors/__init__.py b/providers/src/airflow/providers/apache/flink/sensors/__init__.py similarity index 100% rename from airflow/providers/apache/flink/sensors/__init__.py rename to providers/src/airflow/providers/apache/flink/sensors/__init__.py diff --git a/airflow/providers/apache/flink/sensors/flink_kubernetes.py b/providers/src/airflow/providers/apache/flink/sensors/flink_kubernetes.py similarity index 100% rename from airflow/providers/apache/flink/sensors/flink_kubernetes.py rename to providers/src/airflow/providers/apache/flink/sensors/flink_kubernetes.py diff --git a/airflow/providers/apache/hdfs/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/hdfs/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/hdfs/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/hdfs/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/hdfs/CHANGELOG.rst b/providers/src/airflow/providers/apache/hdfs/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/hdfs/CHANGELOG.rst rename to providers/src/airflow/providers/apache/hdfs/CHANGELOG.rst diff --git a/airflow/providers/apache/hdfs/__init__.py b/providers/src/airflow/providers/apache/hdfs/__init__.py similarity index 100% rename from airflow/providers/apache/hdfs/__init__.py rename to providers/src/airflow/providers/apache/hdfs/__init__.py diff --git a/airflow/providers/apache/hdfs/hooks/__init__.py b/providers/src/airflow/providers/apache/hdfs/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/hdfs/hooks/__init__.py rename to providers/src/airflow/providers/apache/hdfs/hooks/__init__.py diff --git a/airflow/providers/apache/hdfs/hooks/hdfs.py b/providers/src/airflow/providers/apache/hdfs/hooks/hdfs.py similarity index 100% rename from airflow/providers/apache/hdfs/hooks/hdfs.py rename to providers/src/airflow/providers/apache/hdfs/hooks/hdfs.py diff --git a/airflow/providers/apache/hdfs/hooks/webhdfs.py b/providers/src/airflow/providers/apache/hdfs/hooks/webhdfs.py similarity index 100% rename from airflow/providers/apache/hdfs/hooks/webhdfs.py rename to providers/src/airflow/providers/apache/hdfs/hooks/webhdfs.py diff --git a/airflow/providers/apache/kafka/hooks/__init__.py b/providers/src/airflow/providers/apache/hdfs/log/__init__.py similarity index 100% rename from airflow/providers/apache/kafka/hooks/__init__.py rename to providers/src/airflow/providers/apache/hdfs/log/__init__.py diff --git a/airflow/providers/apache/hdfs/log/hdfs_task_handler.py b/providers/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py similarity index 100% rename from airflow/providers/apache/hdfs/log/hdfs_task_handler.py rename to providers/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py diff --git a/airflow/providers/apache/hdfs/provider.yaml b/providers/src/airflow/providers/apache/hdfs/provider.yaml similarity index 100% rename from airflow/providers/apache/hdfs/provider.yaml rename to providers/src/airflow/providers/apache/hdfs/provider.yaml diff --git a/airflow/providers/apache/hdfs/sensors/__init__.py b/providers/src/airflow/providers/apache/hdfs/sensors/__init__.py similarity index 100% rename from airflow/providers/apache/hdfs/sensors/__init__.py rename to providers/src/airflow/providers/apache/hdfs/sensors/__init__.py diff --git a/airflow/providers/apache/hdfs/sensors/hdfs.py b/providers/src/airflow/providers/apache/hdfs/sensors/hdfs.py similarity index 100% rename from airflow/providers/apache/hdfs/sensors/hdfs.py rename to providers/src/airflow/providers/apache/hdfs/sensors/hdfs.py diff --git a/airflow/providers/apache/hdfs/sensors/web_hdfs.py b/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py similarity index 100% rename from airflow/providers/apache/hdfs/sensors/web_hdfs.py rename to providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py diff --git a/airflow/providers/apache/hive/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/hive/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/hive/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/hive/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/hive/CHANGELOG.rst b/providers/src/airflow/providers/apache/hive/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/hive/CHANGELOG.rst rename to providers/src/airflow/providers/apache/hive/CHANGELOG.rst diff --git a/airflow/providers/apache/hive/__init__.py b/providers/src/airflow/providers/apache/hive/__init__.py similarity index 100% rename from airflow/providers/apache/hive/__init__.py rename to providers/src/airflow/providers/apache/hive/__init__.py diff --git a/airflow/providers/apache/hive/hooks/__init__.py b/providers/src/airflow/providers/apache/hive/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/hive/hooks/__init__.py rename to providers/src/airflow/providers/apache/hive/hooks/__init__.py diff --git a/airflow/providers/apache/hive/hooks/hive.py b/providers/src/airflow/providers/apache/hive/hooks/hive.py similarity index 99% rename from airflow/providers/apache/hive/hooks/hive.py rename to providers/src/airflow/providers/apache/hive/hooks/hive.py index e08c4d6128d81..d768743cd459b 100644 --- a/airflow/providers/apache/hive/hooks/hive.py +++ b/providers/src/airflow/providers/apache/hive/hooks/hive.py @@ -23,8 +23,9 @@ import socket import subprocess import time +from collections.abc import Iterable, Mapping from tempfile import NamedTemporaryFile, TemporaryDirectory -from typing import TYPE_CHECKING, Any, Iterable, Mapping +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: import pandas as pd @@ -411,9 +412,10 @@ def _infer_field_types_from_df(df: pd.DataFrame) -> dict[Any, Any]: if pandas_kwargs is None: pandas_kwargs = {} - with TemporaryDirectory(prefix="airflow_hiveop_") as tmp_dir, NamedTemporaryFile( - dir=tmp_dir, mode="w" - ) as f: + with ( + TemporaryDirectory(prefix="airflow_hiveop_") as tmp_dir, + NamedTemporaryFile(dir=tmp_dir, mode="w") as f, + ): if field_dict is None: field_dict = _infer_field_types_from_df(df) diff --git a/airflow/providers/apache/hive/macros/__init__.py b/providers/src/airflow/providers/apache/hive/macros/__init__.py similarity index 100% rename from airflow/providers/apache/hive/macros/__init__.py rename to providers/src/airflow/providers/apache/hive/macros/__init__.py diff --git a/airflow/providers/apache/hive/macros/hive.py b/providers/src/airflow/providers/apache/hive/macros/hive.py similarity index 100% rename from airflow/providers/apache/hive/macros/hive.py rename to providers/src/airflow/providers/apache/hive/macros/hive.py diff --git a/airflow/providers/apache/hive/operators/__init__.py b/providers/src/airflow/providers/apache/hive/operators/__init__.py similarity index 100% rename from airflow/providers/apache/hive/operators/__init__.py rename to providers/src/airflow/providers/apache/hive/operators/__init__.py diff --git a/airflow/providers/apache/hive/operators/hive.py b/providers/src/airflow/providers/apache/hive/operators/hive.py similarity index 100% rename from airflow/providers/apache/hive/operators/hive.py rename to providers/src/airflow/providers/apache/hive/operators/hive.py diff --git a/airflow/providers/apache/hive/operators/hive_stats.py b/providers/src/airflow/providers/apache/hive/operators/hive_stats.py similarity index 100% rename from airflow/providers/apache/hive/operators/hive_stats.py rename to providers/src/airflow/providers/apache/hive/operators/hive_stats.py diff --git a/airflow/providers/apache/hive/plugins/__init__.py b/providers/src/airflow/providers/apache/hive/plugins/__init__.py similarity index 100% rename from airflow/providers/apache/hive/plugins/__init__.py rename to providers/src/airflow/providers/apache/hive/plugins/__init__.py diff --git a/airflow/providers/apache/hive/plugins/hive.py b/providers/src/airflow/providers/apache/hive/plugins/hive.py similarity index 100% rename from airflow/providers/apache/hive/plugins/hive.py rename to providers/src/airflow/providers/apache/hive/plugins/hive.py diff --git a/airflow/providers/apache/hive/provider.yaml b/providers/src/airflow/providers/apache/hive/provider.yaml similarity index 100% rename from airflow/providers/apache/hive/provider.yaml rename to providers/src/airflow/providers/apache/hive/provider.yaml diff --git a/airflow/providers/apache/hive/sensors/__init__.py b/providers/src/airflow/providers/apache/hive/sensors/__init__.py similarity index 100% rename from airflow/providers/apache/hive/sensors/__init__.py rename to providers/src/airflow/providers/apache/hive/sensors/__init__.py diff --git a/airflow/providers/apache/hive/sensors/hive_partition.py b/providers/src/airflow/providers/apache/hive/sensors/hive_partition.py similarity index 100% rename from airflow/providers/apache/hive/sensors/hive_partition.py rename to providers/src/airflow/providers/apache/hive/sensors/hive_partition.py diff --git a/airflow/providers/apache/hive/sensors/metastore_partition.py b/providers/src/airflow/providers/apache/hive/sensors/metastore_partition.py similarity index 100% rename from airflow/providers/apache/hive/sensors/metastore_partition.py rename to providers/src/airflow/providers/apache/hive/sensors/metastore_partition.py diff --git a/airflow/providers/apache/hive/sensors/named_hive_partition.py b/providers/src/airflow/providers/apache/hive/sensors/named_hive_partition.py similarity index 100% rename from airflow/providers/apache/hive/sensors/named_hive_partition.py rename to providers/src/airflow/providers/apache/hive/sensors/named_hive_partition.py diff --git a/airflow/providers/apache/kafka/operators/__init__.py b/providers/src/airflow/providers/apache/hive/transfers/__init__.py similarity index 100% rename from airflow/providers/apache/kafka/operators/__init__.py rename to providers/src/airflow/providers/apache/hive/transfers/__init__.py diff --git a/airflow/providers/apache/hive/transfers/hive_to_mysql.py b/providers/src/airflow/providers/apache/hive/transfers/hive_to_mysql.py similarity index 100% rename from airflow/providers/apache/hive/transfers/hive_to_mysql.py rename to providers/src/airflow/providers/apache/hive/transfers/hive_to_mysql.py diff --git a/airflow/providers/apache/hive/transfers/hive_to_samba.py b/providers/src/airflow/providers/apache/hive/transfers/hive_to_samba.py similarity index 100% rename from airflow/providers/apache/hive/transfers/hive_to_samba.py rename to providers/src/airflow/providers/apache/hive/transfers/hive_to_samba.py diff --git a/airflow/providers/apache/hive/transfers/mssql_to_hive.py b/providers/src/airflow/providers/apache/hive/transfers/mssql_to_hive.py similarity index 100% rename from airflow/providers/apache/hive/transfers/mssql_to_hive.py rename to providers/src/airflow/providers/apache/hive/transfers/mssql_to_hive.py diff --git a/airflow/providers/apache/hive/transfers/mysql_to_hive.py b/providers/src/airflow/providers/apache/hive/transfers/mysql_to_hive.py similarity index 100% rename from airflow/providers/apache/hive/transfers/mysql_to_hive.py rename to providers/src/airflow/providers/apache/hive/transfers/mysql_to_hive.py diff --git a/airflow/providers/apache/hive/transfers/s3_to_hive.py b/providers/src/airflow/providers/apache/hive/transfers/s3_to_hive.py similarity index 97% rename from airflow/providers/apache/hive/transfers/s3_to_hive.py rename to providers/src/airflow/providers/apache/hive/transfers/s3_to_hive.py index ebc9536897018..6285103d370bd 100644 --- a/airflow/providers/apache/hive/transfers/s3_to_hive.py +++ b/providers/src/airflow/providers/apache/hive/transfers/s3_to_hive.py @@ -23,8 +23,9 @@ import gzip import os import tempfile +from collections.abc import Sequence from tempfile import NamedTemporaryFile, TemporaryDirectory -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -161,9 +162,10 @@ def execute(self, context: Context): if self.select_expression and self.input_compressed and file_ext.lower() != ".gz": raise AirflowException("GZIP is the only compression format Amazon S3 Select supports") - with TemporaryDirectory(prefix="tmps32hive_") as tmp_dir, NamedTemporaryFile( - mode="wb", dir=tmp_dir, suffix=file_ext - ) as f: + with ( + TemporaryDirectory(prefix="tmps32hive_") as tmp_dir, + NamedTemporaryFile(mode="wb", dir=tmp_dir, suffix=file_ext) as f, + ): self.log.info("Dumping S3 key %s contents to local file %s", s3_key_object.key, f.name) if self.select_expression: option = {} diff --git a/airflow/providers/apache/hive/transfers/vertica_to_hive.py b/providers/src/airflow/providers/apache/hive/transfers/vertica_to_hive.py similarity index 100% rename from airflow/providers/apache/hive/transfers/vertica_to_hive.py rename to providers/src/airflow/providers/apache/hive/transfers/vertica_to_hive.py diff --git a/airflow/providers/apache/iceberg/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/iceberg/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/iceberg/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/iceberg/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/iceberg/CHANGELOG.rst b/providers/src/airflow/providers/apache/iceberg/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/iceberg/CHANGELOG.rst rename to providers/src/airflow/providers/apache/iceberg/CHANGELOG.rst diff --git a/airflow/providers/apache/iceberg/__init__.py b/providers/src/airflow/providers/apache/iceberg/__init__.py similarity index 100% rename from airflow/providers/apache/iceberg/__init__.py rename to providers/src/airflow/providers/apache/iceberg/__init__.py diff --git a/airflow/providers/apache/iceberg/hooks/__init__.py b/providers/src/airflow/providers/apache/iceberg/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/iceberg/hooks/__init__.py rename to providers/src/airflow/providers/apache/iceberg/hooks/__init__.py diff --git a/airflow/providers/apache/iceberg/hooks/iceberg.py b/providers/src/airflow/providers/apache/iceberg/hooks/iceberg.py similarity index 100% rename from airflow/providers/apache/iceberg/hooks/iceberg.py rename to providers/src/airflow/providers/apache/iceberg/hooks/iceberg.py diff --git a/airflow/providers/apache/iceberg/provider.yaml b/providers/src/airflow/providers/apache/iceberg/provider.yaml similarity index 100% rename from airflow/providers/apache/iceberg/provider.yaml rename to providers/src/airflow/providers/apache/iceberg/provider.yaml diff --git a/airflow/providers/apache/impala/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/impala/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/impala/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/impala/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/impala/CHANGELOG.rst b/providers/src/airflow/providers/apache/impala/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/impala/CHANGELOG.rst rename to providers/src/airflow/providers/apache/impala/CHANGELOG.rst diff --git a/airflow/providers/apache/impala/__init__.py b/providers/src/airflow/providers/apache/impala/__init__.py similarity index 100% rename from airflow/providers/apache/impala/__init__.py rename to providers/src/airflow/providers/apache/impala/__init__.py diff --git a/airflow/providers/apache/impala/hooks/__init__.py b/providers/src/airflow/providers/apache/impala/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/impala/hooks/__init__.py rename to providers/src/airflow/providers/apache/impala/hooks/__init__.py diff --git a/airflow/providers/apache/impala/hooks/impala.py b/providers/src/airflow/providers/apache/impala/hooks/impala.py similarity index 100% rename from airflow/providers/apache/impala/hooks/impala.py rename to providers/src/airflow/providers/apache/impala/hooks/impala.py diff --git a/airflow/providers/apache/impala/provider.yaml b/providers/src/airflow/providers/apache/impala/provider.yaml similarity index 100% rename from airflow/providers/apache/impala/provider.yaml rename to providers/src/airflow/providers/apache/impala/provider.yaml diff --git a/airflow/providers/apache/kafka/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/kafka/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/kafka/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/kafka/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/kafka/CHANGELOG.rst b/providers/src/airflow/providers/apache/kafka/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/kafka/CHANGELOG.rst rename to providers/src/airflow/providers/apache/kafka/CHANGELOG.rst diff --git a/airflow/providers/apache/kafka/__init__.py b/providers/src/airflow/providers/apache/kafka/__init__.py similarity index 100% rename from airflow/providers/apache/kafka/__init__.py rename to providers/src/airflow/providers/apache/kafka/__init__.py diff --git a/airflow/providers/apache/kafka/sensors/__init__.py b/providers/src/airflow/providers/apache/kafka/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/kafka/sensors/__init__.py rename to providers/src/airflow/providers/apache/kafka/hooks/__init__.py diff --git a/airflow/providers/apache/kafka/hooks/base.py b/providers/src/airflow/providers/apache/kafka/hooks/base.py similarity index 100% rename from airflow/providers/apache/kafka/hooks/base.py rename to providers/src/airflow/providers/apache/kafka/hooks/base.py diff --git a/airflow/providers/apache/kafka/hooks/client.py b/providers/src/airflow/providers/apache/kafka/hooks/client.py similarity index 100% rename from airflow/providers/apache/kafka/hooks/client.py rename to providers/src/airflow/providers/apache/kafka/hooks/client.py diff --git a/airflow/providers/apache/kafka/hooks/consume.py b/providers/src/airflow/providers/apache/kafka/hooks/consume.py similarity index 100% rename from airflow/providers/apache/kafka/hooks/consume.py rename to providers/src/airflow/providers/apache/kafka/hooks/consume.py diff --git a/airflow/providers/apache/kafka/hooks/produce.py b/providers/src/airflow/providers/apache/kafka/hooks/produce.py similarity index 100% rename from airflow/providers/apache/kafka/hooks/produce.py rename to providers/src/airflow/providers/apache/kafka/hooks/produce.py diff --git a/airflow/providers/apache/kafka/triggers/__init__.py b/providers/src/airflow/providers/apache/kafka/operators/__init__.py similarity index 100% rename from airflow/providers/apache/kafka/triggers/__init__.py rename to providers/src/airflow/providers/apache/kafka/operators/__init__.py diff --git a/airflow/providers/apache/kafka/operators/consume.py b/providers/src/airflow/providers/apache/kafka/operators/consume.py similarity index 100% rename from airflow/providers/apache/kafka/operators/consume.py rename to providers/src/airflow/providers/apache/kafka/operators/consume.py diff --git a/airflow/providers/apache/kafka/operators/produce.py b/providers/src/airflow/providers/apache/kafka/operators/produce.py similarity index 100% rename from airflow/providers/apache/kafka/operators/produce.py rename to providers/src/airflow/providers/apache/kafka/operators/produce.py diff --git a/airflow/providers/apache/kafka/provider.yaml b/providers/src/airflow/providers/apache/kafka/provider.yaml similarity index 100% rename from airflow/providers/apache/kafka/provider.yaml rename to providers/src/airflow/providers/apache/kafka/provider.yaml diff --git a/airflow/providers/apache/kylin/hooks/__init__.py b/providers/src/airflow/providers/apache/kafka/sensors/__init__.py similarity index 100% rename from airflow/providers/apache/kylin/hooks/__init__.py rename to providers/src/airflow/providers/apache/kafka/sensors/__init__.py diff --git a/airflow/providers/apache/kafka/sensors/kafka.py b/providers/src/airflow/providers/apache/kafka/sensors/kafka.py similarity index 100% rename from airflow/providers/apache/kafka/sensors/kafka.py rename to providers/src/airflow/providers/apache/kafka/sensors/kafka.py diff --git a/airflow/providers/apache/kylin/operators/__init__.py b/providers/src/airflow/providers/apache/kafka/triggers/__init__.py similarity index 100% rename from airflow/providers/apache/kylin/operators/__init__.py rename to providers/src/airflow/providers/apache/kafka/triggers/__init__.py diff --git a/airflow/providers/apache/kafka/triggers/await_message.py b/providers/src/airflow/providers/apache/kafka/triggers/await_message.py similarity index 100% rename from airflow/providers/apache/kafka/triggers/await_message.py rename to providers/src/airflow/providers/apache/kafka/triggers/await_message.py diff --git a/airflow/providers/apache/kylin/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/kylin/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/kylin/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/kylin/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/kylin/CHANGELOG.rst b/providers/src/airflow/providers/apache/kylin/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/kylin/CHANGELOG.rst rename to providers/src/airflow/providers/apache/kylin/CHANGELOG.rst diff --git a/airflow/providers/apache/kylin/__init__.py b/providers/src/airflow/providers/apache/kylin/__init__.py similarity index 100% rename from airflow/providers/apache/kylin/__init__.py rename to providers/src/airflow/providers/apache/kylin/__init__.py diff --git a/airflow/providers/apache/livy/hooks/__init__.py b/providers/src/airflow/providers/apache/kylin/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/livy/hooks/__init__.py rename to providers/src/airflow/providers/apache/kylin/hooks/__init__.py diff --git a/airflow/providers/apache/kylin/hooks/kylin.py b/providers/src/airflow/providers/apache/kylin/hooks/kylin.py similarity index 100% rename from airflow/providers/apache/kylin/hooks/kylin.py rename to providers/src/airflow/providers/apache/kylin/hooks/kylin.py diff --git a/airflow/providers/apache/livy/operators/__init__.py b/providers/src/airflow/providers/apache/kylin/operators/__init__.py similarity index 100% rename from airflow/providers/apache/livy/operators/__init__.py rename to providers/src/airflow/providers/apache/kylin/operators/__init__.py diff --git a/airflow/providers/apache/kylin/operators/kylin_cube.py b/providers/src/airflow/providers/apache/kylin/operators/kylin_cube.py similarity index 100% rename from airflow/providers/apache/kylin/operators/kylin_cube.py rename to providers/src/airflow/providers/apache/kylin/operators/kylin_cube.py diff --git a/airflow/providers/apache/kylin/provider.yaml b/providers/src/airflow/providers/apache/kylin/provider.yaml similarity index 100% rename from airflow/providers/apache/kylin/provider.yaml rename to providers/src/airflow/providers/apache/kylin/provider.yaml diff --git a/airflow/providers/apache/livy/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/livy/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/livy/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/livy/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/livy/CHANGELOG.rst b/providers/src/airflow/providers/apache/livy/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/livy/CHANGELOG.rst rename to providers/src/airflow/providers/apache/livy/CHANGELOG.rst diff --git a/airflow/providers/apache/livy/__init__.py b/providers/src/airflow/providers/apache/livy/__init__.py similarity index 100% rename from airflow/providers/apache/livy/__init__.py rename to providers/src/airflow/providers/apache/livy/__init__.py diff --git a/airflow/providers/apache/livy/sensors/__init__.py b/providers/src/airflow/providers/apache/livy/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/livy/sensors/__init__.py rename to providers/src/airflow/providers/apache/livy/hooks/__init__.py diff --git a/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py similarity index 100% rename from airflow/providers/apache/livy/hooks/livy.py rename to providers/src/airflow/providers/apache/livy/hooks/livy.py diff --git a/airflow/providers/apache/livy/triggers/__init__.py b/providers/src/airflow/providers/apache/livy/operators/__init__.py similarity index 100% rename from airflow/providers/apache/livy/triggers/__init__.py rename to providers/src/airflow/providers/apache/livy/operators/__init__.py diff --git a/airflow/providers/apache/livy/operators/livy.py b/providers/src/airflow/providers/apache/livy/operators/livy.py similarity index 100% rename from airflow/providers/apache/livy/operators/livy.py rename to providers/src/airflow/providers/apache/livy/operators/livy.py diff --git a/airflow/providers/apache/livy/provider.yaml b/providers/src/airflow/providers/apache/livy/provider.yaml similarity index 100% rename from airflow/providers/apache/livy/provider.yaml rename to providers/src/airflow/providers/apache/livy/provider.yaml diff --git a/airflow/providers/arangodb/example_dags/__init__.py b/providers/src/airflow/providers/apache/livy/sensors/__init__.py similarity index 100% rename from airflow/providers/arangodb/example_dags/__init__.py rename to providers/src/airflow/providers/apache/livy/sensors/__init__.py diff --git a/airflow/providers/apache/livy/sensors/livy.py b/providers/src/airflow/providers/apache/livy/sensors/livy.py similarity index 100% rename from airflow/providers/apache/livy/sensors/livy.py rename to providers/src/airflow/providers/apache/livy/sensors/livy.py diff --git a/airflow/providers/arangodb/hooks/__init__.py b/providers/src/airflow/providers/apache/livy/triggers/__init__.py similarity index 100% rename from airflow/providers/arangodb/hooks/__init__.py rename to providers/src/airflow/providers/apache/livy/triggers/__init__.py diff --git a/airflow/providers/apache/livy/triggers/livy.py b/providers/src/airflow/providers/apache/livy/triggers/livy.py similarity index 100% rename from airflow/providers/apache/livy/triggers/livy.py rename to providers/src/airflow/providers/apache/livy/triggers/livy.py diff --git a/airflow/providers/apache/pig/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/pig/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/pig/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/pig/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/pig/CHANGELOG.rst b/providers/src/airflow/providers/apache/pig/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/pig/CHANGELOG.rst rename to providers/src/airflow/providers/apache/pig/CHANGELOG.rst diff --git a/airflow/providers/apache/pig/__init__.py b/providers/src/airflow/providers/apache/pig/__init__.py similarity index 100% rename from airflow/providers/apache/pig/__init__.py rename to providers/src/airflow/providers/apache/pig/__init__.py diff --git a/airflow/providers/apache/pig/hooks/__init__.py b/providers/src/airflow/providers/apache/pig/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/pig/hooks/__init__.py rename to providers/src/airflow/providers/apache/pig/hooks/__init__.py diff --git a/airflow/providers/apache/pig/hooks/pig.py b/providers/src/airflow/providers/apache/pig/hooks/pig.py similarity index 100% rename from airflow/providers/apache/pig/hooks/pig.py rename to providers/src/airflow/providers/apache/pig/hooks/pig.py diff --git a/airflow/providers/apache/pig/operators/__init__.py b/providers/src/airflow/providers/apache/pig/operators/__init__.py similarity index 100% rename from airflow/providers/apache/pig/operators/__init__.py rename to providers/src/airflow/providers/apache/pig/operators/__init__.py diff --git a/airflow/providers/apache/pig/operators/pig.py b/providers/src/airflow/providers/apache/pig/operators/pig.py similarity index 100% rename from airflow/providers/apache/pig/operators/pig.py rename to providers/src/airflow/providers/apache/pig/operators/pig.py diff --git a/airflow/providers/apache/pig/provider.yaml b/providers/src/airflow/providers/apache/pig/provider.yaml similarity index 100% rename from airflow/providers/apache/pig/provider.yaml rename to providers/src/airflow/providers/apache/pig/provider.yaml diff --git a/airflow/providers/apache/pinot/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/pinot/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/pinot/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/pinot/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/pinot/CHANGELOG.rst b/providers/src/airflow/providers/apache/pinot/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/pinot/CHANGELOG.rst rename to providers/src/airflow/providers/apache/pinot/CHANGELOG.rst diff --git a/airflow/providers/apache/pinot/__init__.py b/providers/src/airflow/providers/apache/pinot/__init__.py similarity index 100% rename from airflow/providers/apache/pinot/__init__.py rename to providers/src/airflow/providers/apache/pinot/__init__.py diff --git a/airflow/providers/apache/pinot/hooks/__init__.py b/providers/src/airflow/providers/apache/pinot/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/pinot/hooks/__init__.py rename to providers/src/airflow/providers/apache/pinot/hooks/__init__.py diff --git a/airflow/providers/apache/pinot/hooks/pinot.py b/providers/src/airflow/providers/apache/pinot/hooks/pinot.py similarity index 100% rename from airflow/providers/apache/pinot/hooks/pinot.py rename to providers/src/airflow/providers/apache/pinot/hooks/pinot.py diff --git a/airflow/providers/apache/pinot/provider.yaml b/providers/src/airflow/providers/apache/pinot/provider.yaml similarity index 100% rename from airflow/providers/apache/pinot/provider.yaml rename to providers/src/airflow/providers/apache/pinot/provider.yaml diff --git a/airflow/providers/apache/spark/.latest-doc-only-change.txt b/providers/src/airflow/providers/apache/spark/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apache/spark/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apache/spark/.latest-doc-only-change.txt diff --git a/airflow/providers/apache/spark/CHANGELOG.rst b/providers/src/airflow/providers/apache/spark/CHANGELOG.rst similarity index 100% rename from airflow/providers/apache/spark/CHANGELOG.rst rename to providers/src/airflow/providers/apache/spark/CHANGELOG.rst diff --git a/airflow/providers/apache/spark/__init__.py b/providers/src/airflow/providers/apache/spark/__init__.py similarity index 100% rename from airflow/providers/apache/spark/__init__.py rename to providers/src/airflow/providers/apache/spark/__init__.py diff --git a/airflow/providers/apache/spark/decorators/__init__.py b/providers/src/airflow/providers/apache/spark/decorators/__init__.py similarity index 100% rename from airflow/providers/apache/spark/decorators/__init__.py rename to providers/src/airflow/providers/apache/spark/decorators/__init__.py diff --git a/airflow/providers/apache/spark/decorators/pyspark.py b/providers/src/airflow/providers/apache/spark/decorators/pyspark.py similarity index 100% rename from airflow/providers/apache/spark/decorators/pyspark.py rename to providers/src/airflow/providers/apache/spark/decorators/pyspark.py diff --git a/airflow/providers/apache/spark/hooks/__init__.py b/providers/src/airflow/providers/apache/spark/hooks/__init__.py similarity index 100% rename from airflow/providers/apache/spark/hooks/__init__.py rename to providers/src/airflow/providers/apache/spark/hooks/__init__.py diff --git a/airflow/providers/apache/spark/hooks/spark_connect.py b/providers/src/airflow/providers/apache/spark/hooks/spark_connect.py similarity index 100% rename from airflow/providers/apache/spark/hooks/spark_connect.py rename to providers/src/airflow/providers/apache/spark/hooks/spark_connect.py diff --git a/airflow/providers/apache/spark/hooks/spark_jdbc.py b/providers/src/airflow/providers/apache/spark/hooks/spark_jdbc.py similarity index 100% rename from airflow/providers/apache/spark/hooks/spark_jdbc.py rename to providers/src/airflow/providers/apache/spark/hooks/spark_jdbc.py diff --git a/airflow/providers/apache/spark/hooks/spark_jdbc_script.py b/providers/src/airflow/providers/apache/spark/hooks/spark_jdbc_script.py similarity index 100% rename from airflow/providers/apache/spark/hooks/spark_jdbc_script.py rename to providers/src/airflow/providers/apache/spark/hooks/spark_jdbc_script.py diff --git a/airflow/providers/apache/spark/hooks/spark_sql.py b/providers/src/airflow/providers/apache/spark/hooks/spark_sql.py similarity index 100% rename from airflow/providers/apache/spark/hooks/spark_sql.py rename to providers/src/airflow/providers/apache/spark/hooks/spark_sql.py diff --git a/airflow/providers/apache/spark/hooks/spark_submit.py b/providers/src/airflow/providers/apache/spark/hooks/spark_submit.py similarity index 100% rename from airflow/providers/apache/spark/hooks/spark_submit.py rename to providers/src/airflow/providers/apache/spark/hooks/spark_submit.py diff --git a/airflow/providers/apache/spark/operators/__init__.py b/providers/src/airflow/providers/apache/spark/operators/__init__.py similarity index 100% rename from airflow/providers/apache/spark/operators/__init__.py rename to providers/src/airflow/providers/apache/spark/operators/__init__.py diff --git a/airflow/providers/apache/spark/operators/spark_jdbc.py b/providers/src/airflow/providers/apache/spark/operators/spark_jdbc.py similarity index 100% rename from airflow/providers/apache/spark/operators/spark_jdbc.py rename to providers/src/airflow/providers/apache/spark/operators/spark_jdbc.py diff --git a/airflow/providers/apache/spark/operators/spark_sql.py b/providers/src/airflow/providers/apache/spark/operators/spark_sql.py similarity index 100% rename from airflow/providers/apache/spark/operators/spark_sql.py rename to providers/src/airflow/providers/apache/spark/operators/spark_sql.py diff --git a/airflow/providers/apache/spark/operators/spark_submit.py b/providers/src/airflow/providers/apache/spark/operators/spark_submit.py similarity index 100% rename from airflow/providers/apache/spark/operators/spark_submit.py rename to providers/src/airflow/providers/apache/spark/operators/spark_submit.py diff --git a/airflow/providers/apache/spark/provider.yaml b/providers/src/airflow/providers/apache/spark/provider.yaml similarity index 100% rename from airflow/providers/apache/spark/provider.yaml rename to providers/src/airflow/providers/apache/spark/provider.yaml diff --git a/airflow/providers/apprise/.latest-doc-only-change.txt b/providers/src/airflow/providers/apprise/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/apprise/.latest-doc-only-change.txt rename to providers/src/airflow/providers/apprise/.latest-doc-only-change.txt diff --git a/airflow/providers/apprise/CHANGELOG.rst b/providers/src/airflow/providers/apprise/CHANGELOG.rst similarity index 100% rename from airflow/providers/apprise/CHANGELOG.rst rename to providers/src/airflow/providers/apprise/CHANGELOG.rst diff --git a/airflow/providers/apprise/__init__.py b/providers/src/airflow/providers/apprise/__init__.py similarity index 100% rename from airflow/providers/apprise/__init__.py rename to providers/src/airflow/providers/apprise/__init__.py diff --git a/airflow/providers/apprise/hooks/__init__.py b/providers/src/airflow/providers/apprise/hooks/__init__.py similarity index 100% rename from airflow/providers/apprise/hooks/__init__.py rename to providers/src/airflow/providers/apprise/hooks/__init__.py diff --git a/airflow/providers/apprise/hooks/apprise.py b/providers/src/airflow/providers/apprise/hooks/apprise.py similarity index 100% rename from airflow/providers/apprise/hooks/apprise.py rename to providers/src/airflow/providers/apprise/hooks/apprise.py diff --git a/airflow/providers/apprise/notifications/__init__.py b/providers/src/airflow/providers/apprise/notifications/__init__.py similarity index 100% rename from airflow/providers/apprise/notifications/__init__.py rename to providers/src/airflow/providers/apprise/notifications/__init__.py diff --git a/airflow/providers/apprise/notifications/apprise.py b/providers/src/airflow/providers/apprise/notifications/apprise.py similarity index 100% rename from airflow/providers/apprise/notifications/apprise.py rename to providers/src/airflow/providers/apprise/notifications/apprise.py diff --git a/airflow/providers/apprise/provider.yaml b/providers/src/airflow/providers/apprise/provider.yaml similarity index 100% rename from airflow/providers/apprise/provider.yaml rename to providers/src/airflow/providers/apprise/provider.yaml diff --git a/airflow/providers/arangodb/.latest-doc-only-change.txt b/providers/src/airflow/providers/arangodb/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/arangodb/.latest-doc-only-change.txt rename to providers/src/airflow/providers/arangodb/.latest-doc-only-change.txt diff --git a/airflow/providers/arangodb/CHANGELOG.rst b/providers/src/airflow/providers/arangodb/CHANGELOG.rst similarity index 100% rename from airflow/providers/arangodb/CHANGELOG.rst rename to providers/src/airflow/providers/arangodb/CHANGELOG.rst diff --git a/airflow/providers/arangodb/__init__.py b/providers/src/airflow/providers/arangodb/__init__.py similarity index 100% rename from airflow/providers/arangodb/__init__.py rename to providers/src/airflow/providers/arangodb/__init__.py diff --git a/airflow/providers/arangodb/operators/__init__.py b/providers/src/airflow/providers/arangodb/example_dags/__init__.py similarity index 100% rename from airflow/providers/arangodb/operators/__init__.py rename to providers/src/airflow/providers/arangodb/example_dags/__init__.py diff --git a/airflow/providers/arangodb/example_dags/example_arangodb.py b/providers/src/airflow/providers/arangodb/example_dags/example_arangodb.py similarity index 100% rename from airflow/providers/arangodb/example_dags/example_arangodb.py rename to providers/src/airflow/providers/arangodb/example_dags/example_arangodb.py diff --git a/airflow/providers/arangodb/sensors/__init__.py b/providers/src/airflow/providers/arangodb/hooks/__init__.py similarity index 100% rename from airflow/providers/arangodb/sensors/__init__.py rename to providers/src/airflow/providers/arangodb/hooks/__init__.py diff --git a/airflow/providers/arangodb/hooks/arangodb.py b/providers/src/airflow/providers/arangodb/hooks/arangodb.py similarity index 100% rename from airflow/providers/arangodb/hooks/arangodb.py rename to providers/src/airflow/providers/arangodb/hooks/arangodb.py diff --git a/airflow/providers/atlassian/__init__.py b/providers/src/airflow/providers/arangodb/operators/__init__.py similarity index 100% rename from airflow/providers/atlassian/__init__.py rename to providers/src/airflow/providers/arangodb/operators/__init__.py diff --git a/airflow/providers/arangodb/operators/arangodb.py b/providers/src/airflow/providers/arangodb/operators/arangodb.py similarity index 100% rename from airflow/providers/arangodb/operators/arangodb.py rename to providers/src/airflow/providers/arangodb/operators/arangodb.py diff --git a/airflow/providers/arangodb/provider.yaml b/providers/src/airflow/providers/arangodb/provider.yaml similarity index 100% rename from airflow/providers/arangodb/provider.yaml rename to providers/src/airflow/providers/arangodb/provider.yaml diff --git a/airflow/providers/atlassian/jira/hooks/__init__.py b/providers/src/airflow/providers/arangodb/sensors/__init__.py similarity index 100% rename from airflow/providers/atlassian/jira/hooks/__init__.py rename to providers/src/airflow/providers/arangodb/sensors/__init__.py diff --git a/airflow/providers/arangodb/sensors/arangodb.py b/providers/src/airflow/providers/arangodb/sensors/arangodb.py similarity index 100% rename from airflow/providers/arangodb/sensors/arangodb.py rename to providers/src/airflow/providers/arangodb/sensors/arangodb.py diff --git a/airflow/providers/asana/.latest-doc-only-change.txt b/providers/src/airflow/providers/asana/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/asana/.latest-doc-only-change.txt rename to providers/src/airflow/providers/asana/.latest-doc-only-change.txt diff --git a/airflow/providers/asana/CHANGELOG.rst b/providers/src/airflow/providers/asana/CHANGELOG.rst similarity index 100% rename from airflow/providers/asana/CHANGELOG.rst rename to providers/src/airflow/providers/asana/CHANGELOG.rst diff --git a/airflow/providers/asana/README.md b/providers/src/airflow/providers/asana/README.md similarity index 100% rename from airflow/providers/asana/README.md rename to providers/src/airflow/providers/asana/README.md diff --git a/airflow/providers/asana/__init__.py b/providers/src/airflow/providers/asana/__init__.py similarity index 100% rename from airflow/providers/asana/__init__.py rename to providers/src/airflow/providers/asana/__init__.py diff --git a/airflow/providers/asana/hooks/__init__.py b/providers/src/airflow/providers/asana/hooks/__init__.py similarity index 100% rename from airflow/providers/asana/hooks/__init__.py rename to providers/src/airflow/providers/asana/hooks/__init__.py diff --git a/airflow/providers/asana/hooks/asana.py b/providers/src/airflow/providers/asana/hooks/asana.py similarity index 100% rename from airflow/providers/asana/hooks/asana.py rename to providers/src/airflow/providers/asana/hooks/asana.py diff --git a/airflow/providers/asana/operators/__init__.py b/providers/src/airflow/providers/asana/operators/__init__.py similarity index 100% rename from airflow/providers/asana/operators/__init__.py rename to providers/src/airflow/providers/asana/operators/__init__.py diff --git a/airflow/providers/asana/operators/asana_tasks.py b/providers/src/airflow/providers/asana/operators/asana_tasks.py similarity index 100% rename from airflow/providers/asana/operators/asana_tasks.py rename to providers/src/airflow/providers/asana/operators/asana_tasks.py diff --git a/airflow/providers/asana/provider.yaml b/providers/src/airflow/providers/asana/provider.yaml similarity index 100% rename from airflow/providers/asana/provider.yaml rename to providers/src/airflow/providers/asana/provider.yaml diff --git a/airflow/providers/atlassian/jira/.latest-doc-only-change.txt b/providers/src/airflow/providers/atlassian/jira/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/atlassian/jira/.latest-doc-only-change.txt rename to providers/src/airflow/providers/atlassian/jira/.latest-doc-only-change.txt diff --git a/airflow/providers/atlassian/jira/CHANGELOG.rst b/providers/src/airflow/providers/atlassian/jira/CHANGELOG.rst similarity index 100% rename from airflow/providers/atlassian/jira/CHANGELOG.rst rename to providers/src/airflow/providers/atlassian/jira/CHANGELOG.rst diff --git a/airflow/providers/atlassian/jira/__init__.py b/providers/src/airflow/providers/atlassian/jira/__init__.py similarity index 100% rename from airflow/providers/atlassian/jira/__init__.py rename to providers/src/airflow/providers/atlassian/jira/__init__.py diff --git a/airflow/providers/atlassian/jira/notifications/__init__.py b/providers/src/airflow/providers/atlassian/jira/hooks/__init__.py similarity index 100% rename from airflow/providers/atlassian/jira/notifications/__init__.py rename to providers/src/airflow/providers/atlassian/jira/hooks/__init__.py diff --git a/airflow/providers/atlassian/jira/hooks/jira.py b/providers/src/airflow/providers/atlassian/jira/hooks/jira.py similarity index 100% rename from airflow/providers/atlassian/jira/hooks/jira.py rename to providers/src/airflow/providers/atlassian/jira/hooks/jira.py diff --git a/airflow/providers/atlassian/jira/operators/__init__.py b/providers/src/airflow/providers/atlassian/jira/notifications/__init__.py similarity index 100% rename from airflow/providers/atlassian/jira/operators/__init__.py rename to providers/src/airflow/providers/atlassian/jira/notifications/__init__.py diff --git a/airflow/providers/atlassian/jira/notifications/jira.py b/providers/src/airflow/providers/atlassian/jira/notifications/jira.py similarity index 100% rename from airflow/providers/atlassian/jira/notifications/jira.py rename to providers/src/airflow/providers/atlassian/jira/notifications/jira.py diff --git a/airflow/providers/atlassian/jira/sensors/__init__.py b/providers/src/airflow/providers/atlassian/jira/operators/__init__.py similarity index 100% rename from airflow/providers/atlassian/jira/sensors/__init__.py rename to providers/src/airflow/providers/atlassian/jira/operators/__init__.py diff --git a/airflow/providers/atlassian/jira/operators/jira.py b/providers/src/airflow/providers/atlassian/jira/operators/jira.py similarity index 100% rename from airflow/providers/atlassian/jira/operators/jira.py rename to providers/src/airflow/providers/atlassian/jira/operators/jira.py diff --git a/airflow/providers/atlassian/jira/provider.yaml b/providers/src/airflow/providers/atlassian/jira/provider.yaml similarity index 100% rename from airflow/providers/atlassian/jira/provider.yaml rename to providers/src/airflow/providers/atlassian/jira/provider.yaml diff --git a/airflow/providers/celery/cli/__init__.py b/providers/src/airflow/providers/atlassian/jira/sensors/__init__.py similarity index 100% rename from airflow/providers/celery/cli/__init__.py rename to providers/src/airflow/providers/atlassian/jira/sensors/__init__.py diff --git a/airflow/providers/atlassian/jira/sensors/jira.py b/providers/src/airflow/providers/atlassian/jira/sensors/jira.py similarity index 100% rename from airflow/providers/atlassian/jira/sensors/jira.py rename to providers/src/airflow/providers/atlassian/jira/sensors/jira.py diff --git a/airflow/providers/celery/.latest-doc-only-change.txt b/providers/src/airflow/providers/celery/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/celery/.latest-doc-only-change.txt rename to providers/src/airflow/providers/celery/.latest-doc-only-change.txt diff --git a/airflow/providers/celery/CHANGELOG.rst b/providers/src/airflow/providers/celery/CHANGELOG.rst similarity index 100% rename from airflow/providers/celery/CHANGELOG.rst rename to providers/src/airflow/providers/celery/CHANGELOG.rst diff --git a/airflow/providers/celery/__init__.py b/providers/src/airflow/providers/celery/__init__.py similarity index 100% rename from airflow/providers/celery/__init__.py rename to providers/src/airflow/providers/celery/__init__.py diff --git a/airflow/providers/cncf/kubernetes/backcompat/__init__.py b/providers/src/airflow/providers/celery/cli/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/backcompat/__init__.py rename to providers/src/airflow/providers/celery/cli/__init__.py diff --git a/airflow/providers/celery/cli/celery_command.py b/providers/src/airflow/providers/celery/cli/celery_command.py similarity index 100% rename from airflow/providers/celery/cli/celery_command.py rename to providers/src/airflow/providers/celery/cli/celery_command.py diff --git a/airflow/providers/celery/executors/__init__.py b/providers/src/airflow/providers/celery/executors/__init__.py similarity index 100% rename from airflow/providers/celery/executors/__init__.py rename to providers/src/airflow/providers/celery/executors/__init__.py diff --git a/airflow/providers/celery/executors/celery_executor.py b/providers/src/airflow/providers/celery/executors/celery_executor.py similarity index 100% rename from airflow/providers/celery/executors/celery_executor.py rename to providers/src/airflow/providers/celery/executors/celery_executor.py diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/providers/src/airflow/providers/celery/executors/celery_executor_utils.py similarity index 100% rename from airflow/providers/celery/executors/celery_executor_utils.py rename to providers/src/airflow/providers/celery/executors/celery_executor_utils.py diff --git a/airflow/providers/celery/executors/celery_kubernetes_executor.py b/providers/src/airflow/providers/celery/executors/celery_kubernetes_executor.py similarity index 100% rename from airflow/providers/celery/executors/celery_kubernetes_executor.py rename to providers/src/airflow/providers/celery/executors/celery_kubernetes_executor.py diff --git a/airflow/providers/celery/executors/default_celery.py b/providers/src/airflow/providers/celery/executors/default_celery.py similarity index 100% rename from airflow/providers/celery/executors/default_celery.py rename to providers/src/airflow/providers/celery/executors/default_celery.py diff --git a/airflow/providers/celery/provider.yaml b/providers/src/airflow/providers/celery/provider.yaml similarity index 100% rename from airflow/providers/celery/provider.yaml rename to providers/src/airflow/providers/celery/provider.yaml diff --git a/airflow/providers/celery/sensors/__init__.py b/providers/src/airflow/providers/celery/sensors/__init__.py similarity index 100% rename from airflow/providers/celery/sensors/__init__.py rename to providers/src/airflow/providers/celery/sensors/__init__.py diff --git a/airflow/providers/celery/sensors/celery_queue.py b/providers/src/airflow/providers/celery/sensors/celery_queue.py similarity index 100% rename from airflow/providers/celery/sensors/celery_queue.py rename to providers/src/airflow/providers/celery/sensors/celery_queue.py diff --git a/airflow/providers/cloudant/.latest-doc-only-change.txt b/providers/src/airflow/providers/cloudant/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/cloudant/.latest-doc-only-change.txt rename to providers/src/airflow/providers/cloudant/.latest-doc-only-change.txt diff --git a/airflow/providers/cloudant/CHANGELOG.rst b/providers/src/airflow/providers/cloudant/CHANGELOG.rst similarity index 100% rename from airflow/providers/cloudant/CHANGELOG.rst rename to providers/src/airflow/providers/cloudant/CHANGELOG.rst diff --git a/airflow/providers/cloudant/__init__.py b/providers/src/airflow/providers/cloudant/__init__.py similarity index 100% rename from airflow/providers/cloudant/__init__.py rename to providers/src/airflow/providers/cloudant/__init__.py diff --git a/airflow/providers/cloudant/cloudant_fake.py b/providers/src/airflow/providers/cloudant/cloudant_fake.py similarity index 100% rename from airflow/providers/cloudant/cloudant_fake.py rename to providers/src/airflow/providers/cloudant/cloudant_fake.py diff --git a/airflow/providers/cloudant/hooks/__init__.py b/providers/src/airflow/providers/cloudant/hooks/__init__.py similarity index 100% rename from airflow/providers/cloudant/hooks/__init__.py rename to providers/src/airflow/providers/cloudant/hooks/__init__.py diff --git a/airflow/providers/cloudant/hooks/cloudant.py b/providers/src/airflow/providers/cloudant/hooks/cloudant.py similarity index 100% rename from airflow/providers/cloudant/hooks/cloudant.py rename to providers/src/airflow/providers/cloudant/hooks/cloudant.py diff --git a/airflow/providers/cloudant/provider.yaml b/providers/src/airflow/providers/cloudant/provider.yaml similarity index 100% rename from airflow/providers/cloudant/provider.yaml rename to providers/src/airflow/providers/cloudant/provider.yaml diff --git a/airflow/providers/cncf/kubernetes/.latest-doc-only-change.txt b/providers/src/airflow/providers/cncf/kubernetes/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/cncf/kubernetes/.latest-doc-only-change.txt rename to providers/src/airflow/providers/cncf/kubernetes/.latest-doc-only-change.txt diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/providers/src/airflow/providers/cncf/kubernetes/CHANGELOG.rst similarity index 100% rename from airflow/providers/cncf/kubernetes/CHANGELOG.rst rename to providers/src/airflow/providers/cncf/kubernetes/CHANGELOG.rst diff --git a/airflow/providers/cncf/kubernetes/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/__init__.py diff --git a/airflow/providers/cncf/kubernetes/cli/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/cli/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py diff --git a/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py b/providers/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py similarity index 100% rename from airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py rename to providers/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py diff --git a/airflow/providers/cncf/kubernetes/callbacks.py b/providers/src/airflow/providers/cncf/kubernetes/callbacks.py similarity index 100% rename from airflow/providers/cncf/kubernetes/callbacks.py rename to providers/src/airflow/providers/cncf/kubernetes/callbacks.py diff --git a/airflow/providers/cncf/kubernetes/hooks/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/cli/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/hooks/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/cli/__init__.py diff --git a/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py b/providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py similarity index 100% rename from airflow/providers/cncf/kubernetes/cli/kubernetes_command.py rename to providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py diff --git a/airflow/providers/cncf/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/decorators/__init__.py similarity index 100% rename from airflow/providers/cncf/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/decorators/__init__.py diff --git a/airflow/providers/cncf/kubernetes/decorators/kubernetes.py b/providers/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py similarity index 100% rename from airflow/providers/cncf/kubernetes/decorators/kubernetes.py rename to providers/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py diff --git a/airflow/providers/cncf/kubernetes/decorators/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/executors/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/decorators/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/executors/__init__.py diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py similarity index 100% rename from airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py rename to providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py similarity index 100% rename from airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py rename to providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py similarity index 100% rename from airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py rename to providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py diff --git a/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py b/providers/src/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py similarity index 100% rename from airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py rename to providers/src/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py diff --git a/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/hooks/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/hooks/__init__.py diff --git a/airflow/providers/cncf/kubernetes/hooks/kubernetes.py b/providers/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py similarity index 100% rename from airflow/providers/cncf/kubernetes/hooks/kubernetes.py rename to providers/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py diff --git a/airflow/providers/cncf/kubernetes/k8s_model.py b/providers/src/airflow/providers/cncf/kubernetes/k8s_model.py similarity index 100% rename from airflow/providers/cncf/kubernetes/k8s_model.py rename to providers/src/airflow/providers/cncf/kubernetes/k8s_model.py diff --git a/airflow/providers/cncf/kubernetes/kube_client.py b/providers/src/airflow/providers/cncf/kubernetes/kube_client.py similarity index 100% rename from airflow/providers/cncf/kubernetes/kube_client.py rename to providers/src/airflow/providers/cncf/kubernetes/kube_client.py diff --git a/airflow/providers/cncf/kubernetes/kube_config.py b/providers/src/airflow/providers/cncf/kubernetes/kube_config.py similarity index 100% rename from airflow/providers/cncf/kubernetes/kube_config.py rename to providers/src/airflow/providers/cncf/kubernetes/kube_config.py diff --git a/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py diff --git a/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml b/providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml similarity index 100% rename from airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml rename to providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml diff --git a/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py b/providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py similarity index 100% rename from airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py rename to providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py diff --git a/airflow/providers/cncf/kubernetes/executors/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/operators/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/executors/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/operators/__init__.py diff --git a/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py b/providers/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py similarity index 100% rename from airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py rename to providers/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py diff --git a/airflow/providers/cncf/kubernetes/operators/job.py b/providers/src/airflow/providers/cncf/kubernetes/operators/job.py similarity index 100% rename from airflow/providers/cncf/kubernetes/operators/job.py rename to providers/src/airflow/providers/cncf/kubernetes/operators/job.py diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/providers/src/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py similarity index 100% rename from airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py rename to providers/src/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py similarity index 100% rename from airflow/providers/cncf/kubernetes/operators/pod.py rename to providers/src/airflow/providers/cncf/kubernetes/operators/pod.py diff --git a/airflow/providers/cncf/kubernetes/operators/resource.py b/providers/src/airflow/providers/cncf/kubernetes/operators/resource.py similarity index 100% rename from airflow/providers/cncf/kubernetes/operators/resource.py rename to providers/src/airflow/providers/cncf/kubernetes/operators/resource.py diff --git a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py similarity index 100% rename from airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py rename to providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py diff --git a/airflow/providers/cncf/kubernetes/pod_generator.py b/providers/src/airflow/providers/cncf/kubernetes/pod_generator.py similarity index 100% rename from airflow/providers/cncf/kubernetes/pod_generator.py rename to providers/src/airflow/providers/cncf/kubernetes/pod_generator.py diff --git a/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py b/providers/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py similarity index 100% rename from airflow/providers/cncf/kubernetes/pod_generator_deprecated.py rename to providers/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py diff --git a/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py b/providers/src/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py similarity index 100% rename from airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py rename to providers/src/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py diff --git a/airflow/providers/cncf/kubernetes/resource_convert/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/resource_convert/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py diff --git a/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml b/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml similarity index 100% rename from airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml rename to providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml diff --git a/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml b/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml similarity index 100% rename from airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml rename to providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml diff --git a/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml b/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml similarity index 100% rename from airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml rename to providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/providers/src/airflow/providers/cncf/kubernetes/provider.yaml similarity index 100% rename from airflow/providers/cncf/kubernetes/provider.yaml rename to providers/src/airflow/providers/cncf/kubernetes/provider.yaml diff --git a/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 b/providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 similarity index 100% rename from airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 rename to providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 diff --git a/airflow/providers/cncf/kubernetes/python_kubernetes_script.py b/providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.py similarity index 100% rename from airflow/providers/cncf/kubernetes/python_kubernetes_script.py rename to providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.py diff --git a/airflow/providers/cncf/kubernetes/sensors/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/resource_convert/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/sensors/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/resource_convert/__init__.py diff --git a/airflow/providers/cncf/kubernetes/resource_convert/configmap.py b/providers/src/airflow/providers/cncf/kubernetes/resource_convert/configmap.py similarity index 100% rename from airflow/providers/cncf/kubernetes/resource_convert/configmap.py rename to providers/src/airflow/providers/cncf/kubernetes/resource_convert/configmap.py diff --git a/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py b/providers/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py similarity index 100% rename from airflow/providers/cncf/kubernetes/resource_convert/env_variable.py rename to providers/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py diff --git a/airflow/providers/cncf/kubernetes/resource_convert/secret.py b/providers/src/airflow/providers/cncf/kubernetes/resource_convert/secret.py similarity index 100% rename from airflow/providers/cncf/kubernetes/resource_convert/secret.py rename to providers/src/airflow/providers/cncf/kubernetes/resource_convert/secret.py diff --git a/airflow/providers/cncf/kubernetes/secret.py b/providers/src/airflow/providers/cncf/kubernetes/secret.py similarity index 100% rename from airflow/providers/cncf/kubernetes/secret.py rename to providers/src/airflow/providers/cncf/kubernetes/secret.py diff --git a/airflow/providers/cncf/kubernetes/triggers/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/sensors/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/triggers/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/sensors/__init__.py diff --git a/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py b/providers/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py similarity index 100% rename from airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py rename to providers/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py diff --git a/airflow/providers/cncf/kubernetes/template_rendering.py b/providers/src/airflow/providers/cncf/kubernetes/template_rendering.py similarity index 100% rename from airflow/providers/cncf/kubernetes/template_rendering.py rename to providers/src/airflow/providers/cncf/kubernetes/template_rendering.py diff --git a/airflow/providers/cohere/operators/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/triggers/__init__.py similarity index 100% rename from airflow/providers/cohere/operators/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/triggers/__init__.py diff --git a/airflow/providers/cncf/kubernetes/triggers/job.py b/providers/src/airflow/providers/cncf/kubernetes/triggers/job.py similarity index 100% rename from airflow/providers/cncf/kubernetes/triggers/job.py rename to providers/src/airflow/providers/cncf/kubernetes/triggers/job.py diff --git a/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py b/providers/src/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py similarity index 100% rename from airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py rename to providers/src/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py diff --git a/airflow/providers/cncf/kubernetes/triggers/pod.py b/providers/src/airflow/providers/cncf/kubernetes/triggers/pod.py similarity index 100% rename from airflow/providers/cncf/kubernetes/triggers/pod.py rename to providers/src/airflow/providers/cncf/kubernetes/triggers/pod.py diff --git a/airflow/providers/cncf/kubernetes/utils/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/utils/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/utils/__init__.py rename to providers/src/airflow/providers/cncf/kubernetes/utils/__init__.py diff --git a/airflow/providers/cncf/kubernetes/utils/delete_from.py b/providers/src/airflow/providers/cncf/kubernetes/utils/delete_from.py similarity index 100% rename from airflow/providers/cncf/kubernetes/utils/delete_from.py rename to providers/src/airflow/providers/cncf/kubernetes/utils/delete_from.py diff --git a/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py b/providers/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py similarity index 100% rename from airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py rename to providers/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/providers/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py similarity index 100% rename from airflow/providers/cncf/kubernetes/utils/pod_manager.py rename to providers/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py diff --git a/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py b/providers/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py similarity index 100% rename from airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py rename to providers/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py diff --git a/airflow/providers/cohere/.latest-doc-only-change.txt b/providers/src/airflow/providers/cohere/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/cohere/.latest-doc-only-change.txt rename to providers/src/airflow/providers/cohere/.latest-doc-only-change.txt diff --git a/airflow/providers/cohere/CHANGELOG.rst b/providers/src/airflow/providers/cohere/CHANGELOG.rst similarity index 100% rename from airflow/providers/cohere/CHANGELOG.rst rename to providers/src/airflow/providers/cohere/CHANGELOG.rst diff --git a/airflow/providers/cohere/__init__.py b/providers/src/airflow/providers/cohere/__init__.py similarity index 100% rename from airflow/providers/cohere/__init__.py rename to providers/src/airflow/providers/cohere/__init__.py diff --git a/airflow/providers/cohere/hooks/__init__.py b/providers/src/airflow/providers/cohere/hooks/__init__.py similarity index 100% rename from airflow/providers/cohere/hooks/__init__.py rename to providers/src/airflow/providers/cohere/hooks/__init__.py diff --git a/airflow/providers/cohere/hooks/cohere.py b/providers/src/airflow/providers/cohere/hooks/cohere.py similarity index 100% rename from airflow/providers/cohere/hooks/cohere.py rename to providers/src/airflow/providers/cohere/hooks/cohere.py diff --git a/airflow/providers/common/__init__.py b/providers/src/airflow/providers/cohere/operators/__init__.py similarity index 100% rename from airflow/providers/common/__init__.py rename to providers/src/airflow/providers/cohere/operators/__init__.py diff --git a/airflow/providers/cohere/operators/embedding.py b/providers/src/airflow/providers/cohere/operators/embedding.py similarity index 100% rename from airflow/providers/cohere/operators/embedding.py rename to providers/src/airflow/providers/cohere/operators/embedding.py diff --git a/airflow/providers/cohere/provider.yaml b/providers/src/airflow/providers/cohere/provider.yaml similarity index 100% rename from airflow/providers/cohere/provider.yaml rename to providers/src/airflow/providers/cohere/provider.yaml diff --git a/airflow/providers/common/compat/.latest-doc-only-change.txt b/providers/src/airflow/providers/common/compat/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/common/compat/.latest-doc-only-change.txt rename to providers/src/airflow/providers/common/compat/.latest-doc-only-change.txt diff --git a/airflow/providers/common/compat/CHANGELOG.rst b/providers/src/airflow/providers/common/compat/CHANGELOG.rst similarity index 100% rename from airflow/providers/common/compat/CHANGELOG.rst rename to providers/src/airflow/providers/common/compat/CHANGELOG.rst diff --git a/airflow/providers/common/compat/__init__.py b/providers/src/airflow/providers/common/compat/__init__.py similarity index 100% rename from airflow/providers/common/compat/__init__.py rename to providers/src/airflow/providers/common/compat/__init__.py diff --git a/airflow/providers/common/compat/assets/__init__.py b/providers/src/airflow/providers/common/compat/assets/__init__.py similarity index 100% rename from airflow/providers/common/compat/assets/__init__.py rename to providers/src/airflow/providers/common/compat/assets/__init__.py diff --git a/airflow/providers/common/compat/lineage/__init__.py b/providers/src/airflow/providers/common/compat/lineage/__init__.py similarity index 100% rename from airflow/providers/common/compat/lineage/__init__.py rename to providers/src/airflow/providers/common/compat/lineage/__init__.py diff --git a/airflow/providers/common/compat/lineage/hook.py b/providers/src/airflow/providers/common/compat/lineage/hook.py similarity index 100% rename from airflow/providers/common/compat/lineage/hook.py rename to providers/src/airflow/providers/common/compat/lineage/hook.py diff --git a/airflow/providers/common/compat/openlineage/__init__.py b/providers/src/airflow/providers/common/compat/openlineage/__init__.py similarity index 100% rename from airflow/providers/common/compat/openlineage/__init__.py rename to providers/src/airflow/providers/common/compat/openlineage/__init__.py diff --git a/airflow/providers/common/compat/openlineage/facet.py b/providers/src/airflow/providers/common/compat/openlineage/facet.py similarity index 100% rename from airflow/providers/common/compat/openlineage/facet.py rename to providers/src/airflow/providers/common/compat/openlineage/facet.py diff --git a/airflow/providers/common/compat/openlineage/utils/__init__.py b/providers/src/airflow/providers/common/compat/openlineage/utils/__init__.py similarity index 100% rename from airflow/providers/common/compat/openlineage/utils/__init__.py rename to providers/src/airflow/providers/common/compat/openlineage/utils/__init__.py diff --git a/airflow/providers/common/compat/openlineage/utils/utils.py b/providers/src/airflow/providers/common/compat/openlineage/utils/utils.py similarity index 100% rename from airflow/providers/common/compat/openlineage/utils/utils.py rename to providers/src/airflow/providers/common/compat/openlineage/utils/utils.py diff --git a/airflow/providers/common/compat/provider.yaml b/providers/src/airflow/providers/common/compat/provider.yaml similarity index 100% rename from airflow/providers/common/compat/provider.yaml rename to providers/src/airflow/providers/common/compat/provider.yaml diff --git a/airflow/providers/common/compat/security/__init__.py b/providers/src/airflow/providers/common/compat/security/__init__.py similarity index 100% rename from airflow/providers/common/compat/security/__init__.py rename to providers/src/airflow/providers/common/compat/security/__init__.py diff --git a/airflow/providers/common/compat/security/permissions.py b/providers/src/airflow/providers/common/compat/security/permissions.py similarity index 100% rename from airflow/providers/common/compat/security/permissions.py rename to providers/src/airflow/providers/common/compat/security/permissions.py diff --git a/airflow/providers/common/io/.latest-doc-only-change.txt b/providers/src/airflow/providers/common/io/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/common/io/.latest-doc-only-change.txt rename to providers/src/airflow/providers/common/io/.latest-doc-only-change.txt diff --git a/airflow/providers/common/io/CHANGELOG.rst b/providers/src/airflow/providers/common/io/CHANGELOG.rst similarity index 100% rename from airflow/providers/common/io/CHANGELOG.rst rename to providers/src/airflow/providers/common/io/CHANGELOG.rst diff --git a/airflow/providers/common/io/__init__.py b/providers/src/airflow/providers/common/io/__init__.py similarity index 100% rename from airflow/providers/common/io/__init__.py rename to providers/src/airflow/providers/common/io/__init__.py diff --git a/airflow/providers/common/io/assets/__init__.py b/providers/src/airflow/providers/common/io/assets/__init__.py similarity index 100% rename from airflow/providers/common/io/assets/__init__.py rename to providers/src/airflow/providers/common/io/assets/__init__.py diff --git a/airflow/providers/common/io/operators/__init__.py b/providers/src/airflow/providers/common/io/assets/assets/__init__.py similarity index 100% rename from airflow/providers/common/io/operators/__init__.py rename to providers/src/airflow/providers/common/io/assets/assets/__init__.py diff --git a/airflow/providers/common/io/assets/file.py b/providers/src/airflow/providers/common/io/assets/file.py similarity index 100% rename from airflow/providers/common/io/assets/file.py rename to providers/src/airflow/providers/common/io/assets/file.py diff --git a/airflow/providers/common/sql/hooks/__init__.py b/providers/src/airflow/providers/common/io/operators/__init__.py similarity index 100% rename from airflow/providers/common/sql/hooks/__init__.py rename to providers/src/airflow/providers/common/io/operators/__init__.py diff --git a/airflow/providers/common/io/operators/file_transfer.py b/providers/src/airflow/providers/common/io/operators/file_transfer.py similarity index 100% rename from airflow/providers/common/io/operators/file_transfer.py rename to providers/src/airflow/providers/common/io/operators/file_transfer.py diff --git a/airflow/providers/common/io/provider.yaml b/providers/src/airflow/providers/common/io/provider.yaml similarity index 100% rename from airflow/providers/common/io/provider.yaml rename to providers/src/airflow/providers/common/io/provider.yaml diff --git a/airflow/providers/common/io/xcom/__init__.py b/providers/src/airflow/providers/common/io/xcom/__init__.py similarity index 100% rename from airflow/providers/common/io/xcom/__init__.py rename to providers/src/airflow/providers/common/io/xcom/__init__.py diff --git a/airflow/providers/common/io/xcom/backend.py b/providers/src/airflow/providers/common/io/xcom/backend.py similarity index 100% rename from airflow/providers/common/io/xcom/backend.py rename to providers/src/airflow/providers/common/io/xcom/backend.py diff --git a/airflow/providers/common/sql/.latest-doc-only-change.txt b/providers/src/airflow/providers/common/sql/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/common/sql/.latest-doc-only-change.txt rename to providers/src/airflow/providers/common/sql/.latest-doc-only-change.txt diff --git a/airflow/providers/common/sql/CHANGELOG.rst b/providers/src/airflow/providers/common/sql/CHANGELOG.rst similarity index 100% rename from airflow/providers/common/sql/CHANGELOG.rst rename to providers/src/airflow/providers/common/sql/CHANGELOG.rst diff --git a/airflow/providers/common/sql/README_API.md b/providers/src/airflow/providers/common/sql/README_API.md similarity index 100% rename from airflow/providers/common/sql/README_API.md rename to providers/src/airflow/providers/common/sql/README_API.md diff --git a/airflow/providers/common/sql/__init__.py b/providers/src/airflow/providers/common/sql/__init__.py similarity index 100% rename from airflow/providers/common/sql/__init__.py rename to providers/src/airflow/providers/common/sql/__init__.py diff --git a/airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md b/providers/src/airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md similarity index 100% rename from airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md rename to providers/src/airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md diff --git a/airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md b/providers/src/airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md similarity index 100% rename from airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md rename to providers/src/airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md diff --git a/airflow/providers/common/sql/operators/__init__.py b/providers/src/airflow/providers/common/sql/hooks/__init__.py similarity index 100% rename from airflow/providers/common/sql/operators/__init__.py rename to providers/src/airflow/providers/common/sql/hooks/__init__.py diff --git a/airflow/providers/common/sql/hooks/sql.py b/providers/src/airflow/providers/common/sql/hooks/sql.py similarity index 99% rename from airflow/providers/common/sql/hooks/sql.py rename to providers/src/airflow/providers/common/sql/hooks/sql.py index 7983808d0d579..afb66ddd13a0b 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/providers/src/airflow/providers/common/sql/hooks/sql.py @@ -190,7 +190,7 @@ def get_conn_id(self) -> str: return getattr(self, self.conn_name_attr) @cached_property - def placeholder(self): + def placeholder(self) -> str: placeholder = self.connection_extra.get("placeholder") if placeholder: if placeholder in SQL_PLACEHOLDERS: @@ -235,9 +235,11 @@ def connection_extra_lower(self) -> dict: """ return {k.lower(): v for k, v in self.connection_extra.items()} - def get_conn(self): + def get_conn(self) -> Any: """Return a connection object.""" db = self.connection + if self.connector is None: + raise RuntimeError(f"{type(self).__name__} didn't have `self.connector` set!") return self.connector.connect(host=db.host, port=db.port, username=db.login, schema=db.schema) def get_uri(self) -> str: @@ -570,7 +572,7 @@ def get_autocommit(self, conn) -> bool: """ return getattr(conn, "autocommit", False) and self.supports_autocommit - def get_cursor(self): + def get_cursor(self) -> Any: """Return a cursor.""" return self.get_conn().cursor() diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/providers/src/airflow/providers/common/sql/hooks/sql.pyi similarity index 92% rename from airflow/providers/common/sql/hooks/sql.pyi rename to providers/src/airflow/providers/common/sql/hooks/sql.pyi index e54b033991412..0039733d966ab 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/providers/src/airflow/providers/common/sql/hooks/sql.pyi @@ -31,19 +31,14 @@ Definition of the public interface for airflow.providers.common.sql.hooks.sql isort:skip_file """ -from _typeshed import Incomplete -from airflow.exceptions import ( - AirflowException as AirflowException, - AirflowOptionalProviderFeatureException as AirflowOptionalProviderFeatureException, - AirflowProviderDeprecationWarning as AirflowProviderDeprecationWarning, -) +from _typeshed import Incomplete as Incomplete from airflow.hooks.base import BaseHook as BaseHook from airflow.models import Connection as Connection from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage from airflow.providers.openlineage.sqlparser import DatabaseInfo as DatabaseInfo from functools import cached_property as cached_property from pandas import DataFrame as DataFrame -from sqlalchemy.engine import Inspector, URL as URL +from sqlalchemy.engine import Inspector as Inspector, URL as URL from typing import Any, Callable, Generator, Iterable, Mapping, Protocol, Sequence, TypeVar, overload T = TypeVar("T") @@ -67,7 +62,7 @@ class DbApiHook(BaseHook): def __init__(self, *args, schema: str | None = None, log_sql: bool = True, **kwargs) -> None: ... def get_conn_id(self) -> str: ... @cached_property - def placeholder(self): ... + def placeholder(self) -> str: ... @property def connection(self) -> Connection: ... @connection.setter @@ -76,7 +71,7 @@ class DbApiHook(BaseHook): def connection_extra(self) -> dict: ... @cached_property def connection_extra_lower(self) -> dict: ... - def get_conn(self): ... + def get_conn(self) -> Any: ... def get_uri(self) -> str: ... @property def sqlalchemy_url(self) -> URL: ... @@ -123,7 +118,7 @@ class DbApiHook(BaseHook): ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: ... def set_autocommit(self, conn, autocommit) -> None: ... def get_autocommit(self, conn) -> bool: ... - def get_cursor(self): ... + def get_cursor(self) -> Any: ... def insert_rows( self, table, @@ -138,7 +133,7 @@ class DbApiHook(BaseHook): ): ... def bulk_dump(self, table, tmp_file) -> None: ... def bulk_load(self, table, tmp_file) -> None: ... - def test_connection(self): ... + def test_connection(self) -> None: ... def get_openlineage_database_info(self, connection) -> DatabaseInfo | None: ... def get_openlineage_database_dialect(self, connection) -> str: ... def get_openlineage_default_schema(self) -> str | None: ... diff --git a/airflow/providers/common/sql/sensors/__init__.py b/providers/src/airflow/providers/common/sql/operators/__init__.py similarity index 100% rename from airflow/providers/common/sql/sensors/__init__.py rename to providers/src/airflow/providers/common/sql/operators/__init__.py diff --git a/airflow/providers/common/sql/operators/sql.py b/providers/src/airflow/providers/common/sql/operators/sql.py similarity index 100% rename from airflow/providers/common/sql/operators/sql.py rename to providers/src/airflow/providers/common/sql/operators/sql.py diff --git a/airflow/providers/common/sql/operators/sql.pyi b/providers/src/airflow/providers/common/sql/operators/sql.pyi similarity index 92% rename from airflow/providers/common/sql/operators/sql.pyi rename to providers/src/airflow/providers/common/sql/operators/sql.pyi index 0a63ccaa7cc38..1b97cec5023cf 100644 --- a/airflow/providers/common/sql/operators/sql.pyi +++ b/providers/src/airflow/providers/common/sql/operators/sql.pyi @@ -31,25 +31,13 @@ Definition of the public interface for airflow.providers.common.sql.operators.sql isort:skip_file """ -from _typeshed import Incomplete -from airflow.exceptions import ( - AirflowException as AirflowException, - AirflowFailException as AirflowFailException, -) -from airflow.hooks.base import BaseHook as BaseHook +from _typeshed import Incomplete as Incomplete from airflow.models import BaseOperator as BaseOperator, SkipMixin as SkipMixin -from airflow.providers.common.sql.hooks.sql import ( - DbApiHook as DbApiHook, - fetch_all_handler as fetch_all_handler, - return_single_query_results as return_single_query_results, -) +from airflow.providers.common.sql.hooks.sql import DbApiHook as DbApiHook from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage from airflow.utils.context import Context as Context -from airflow.utils.helpers import merge_dicts as merge_dicts -from functools import cached_property as cached_property from typing import Any, Callable, Iterable, Mapping, Sequence, SupportsAbs -def _parse_boolean(val: str) -> str | bool: ... def parse_boolean(val: str) -> str | bool: ... class BaseSQLOperator(BaseOperator): @@ -97,7 +85,7 @@ class SQLExecuteQueryOperator(BaseSQLOperator): show_return_value_in_logs: bool = False, **kwargs, ) -> None: ... - def execute(self, context): ... + def execute(self, context) -> None: ... def prepare_template(self) -> None: ... def get_openlineage_facets_on_start(self) -> OperatorLineage | None: ... def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage | None: ... diff --git a/airflow/providers/common/sql/provider.yaml b/providers/src/airflow/providers/common/sql/provider.yaml similarity index 100% rename from airflow/providers/common/sql/provider.yaml rename to providers/src/airflow/providers/common/sql/provider.yaml diff --git a/airflow/providers/databricks/plugins/__init__.py b/providers/src/airflow/providers/common/sql/sensors/__init__.py similarity index 100% rename from airflow/providers/databricks/plugins/__init__.py rename to providers/src/airflow/providers/common/sql/sensors/__init__.py diff --git a/airflow/providers/common/sql/sensors/sql.py b/providers/src/airflow/providers/common/sql/sensors/sql.py similarity index 100% rename from airflow/providers/common/sql/sensors/sql.py rename to providers/src/airflow/providers/common/sql/sensors/sql.py diff --git a/airflow/providers/common/sql/sensors/sql.pyi b/providers/src/airflow/providers/common/sql/sensors/sql.pyi similarity index 91% rename from airflow/providers/common/sql/sensors/sql.pyi rename to providers/src/airflow/providers/common/sql/sensors/sql.pyi index db92f6d6e02a8..4dcbbd0edbc9e 100644 --- a/airflow/providers/common/sql/sensors/sql.pyi +++ b/providers/src/airflow/providers/common/sql/sensors/sql.pyi @@ -31,10 +31,7 @@ Definition of the public interface for airflow.providers.common.sql.sensors.sql isort:skip_file """ -from _typeshed import Incomplete -from airflow.exceptions import AirflowException as AirflowException -from airflow.hooks.base import BaseHook as BaseHook -from airflow.providers.common.sql.hooks.sql import DbApiHook as DbApiHook +from _typeshed import Incomplete as Incomplete from airflow.sensors.base import BaseSensorOperator as BaseSensorOperator from airflow.utils.context import Context as Context from typing import Any, Callable, Mapping, Sequence diff --git a/airflow/providers/databricks/.latest-doc-only-change.txt b/providers/src/airflow/providers/databricks/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/databricks/.latest-doc-only-change.txt rename to providers/src/airflow/providers/databricks/.latest-doc-only-change.txt diff --git a/airflow/providers/databricks/CHANGELOG.rst b/providers/src/airflow/providers/databricks/CHANGELOG.rst similarity index 100% rename from airflow/providers/databricks/CHANGELOG.rst rename to providers/src/airflow/providers/databricks/CHANGELOG.rst diff --git a/airflow/providers/databricks/__init__.py b/providers/src/airflow/providers/databricks/__init__.py similarity index 100% rename from airflow/providers/databricks/__init__.py rename to providers/src/airflow/providers/databricks/__init__.py diff --git a/airflow/providers/databricks/exceptions.py b/providers/src/airflow/providers/databricks/exceptions.py similarity index 100% rename from airflow/providers/databricks/exceptions.py rename to providers/src/airflow/providers/databricks/exceptions.py diff --git a/airflow/providers/cncf/kubernetes/operators/__init__.py b/providers/src/airflow/providers/databricks/hooks/__init__.py similarity index 100% rename from airflow/providers/cncf/kubernetes/operators/__init__.py rename to providers/src/airflow/providers/databricks/hooks/__init__.py diff --git a/airflow/providers/databricks/hooks/databricks.py b/providers/src/airflow/providers/databricks/hooks/databricks.py similarity index 100% rename from airflow/providers/databricks/hooks/databricks.py rename to providers/src/airflow/providers/databricks/hooks/databricks.py diff --git a/airflow/providers/databricks/hooks/databricks_base.py b/providers/src/airflow/providers/databricks/hooks/databricks_base.py similarity index 100% rename from airflow/providers/databricks/hooks/databricks_base.py rename to providers/src/airflow/providers/databricks/hooks/databricks_base.py diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py similarity index 100% rename from airflow/providers/databricks/hooks/databricks_sql.py rename to providers/src/airflow/providers/databricks/hooks/databricks_sql.py diff --git a/airflow/providers/databricks/hooks/__init__.py b/providers/src/airflow/providers/databricks/operators/__init__.py similarity index 100% rename from airflow/providers/databricks/hooks/__init__.py rename to providers/src/airflow/providers/databricks/operators/__init__.py diff --git a/airflow/providers/databricks/operators/databricks.py b/providers/src/airflow/providers/databricks/operators/databricks.py similarity index 100% rename from airflow/providers/databricks/operators/databricks.py rename to providers/src/airflow/providers/databricks/operators/databricks.py diff --git a/airflow/providers/databricks/operators/databricks_repos.py b/providers/src/airflow/providers/databricks/operators/databricks_repos.py similarity index 100% rename from airflow/providers/databricks/operators/databricks_repos.py rename to providers/src/airflow/providers/databricks/operators/databricks_repos.py diff --git a/airflow/providers/databricks/operators/databricks_sql.py b/providers/src/airflow/providers/databricks/operators/databricks_sql.py similarity index 100% rename from airflow/providers/databricks/operators/databricks_sql.py rename to providers/src/airflow/providers/databricks/operators/databricks_sql.py diff --git a/airflow/providers/databricks/operators/databricks_workflow.py b/providers/src/airflow/providers/databricks/operators/databricks_workflow.py similarity index 100% rename from airflow/providers/databricks/operators/databricks_workflow.py rename to providers/src/airflow/providers/databricks/operators/databricks_workflow.py diff --git a/airflow/providers/databricks/sensors/__init__.py b/providers/src/airflow/providers/databricks/plugins/__init__.py similarity index 100% rename from airflow/providers/databricks/sensors/__init__.py rename to providers/src/airflow/providers/databricks/plugins/__init__.py diff --git a/airflow/providers/databricks/plugins/databricks_workflow.py b/providers/src/airflow/providers/databricks/plugins/databricks_workflow.py similarity index 100% rename from airflow/providers/databricks/plugins/databricks_workflow.py rename to providers/src/airflow/providers/databricks/plugins/databricks_workflow.py diff --git a/airflow/providers/databricks/provider.yaml b/providers/src/airflow/providers/databricks/provider.yaml similarity index 100% rename from airflow/providers/databricks/provider.yaml rename to providers/src/airflow/providers/databricks/provider.yaml diff --git a/airflow/providers/databricks/utils/__init__.py b/providers/src/airflow/providers/databricks/sensors/__init__.py similarity index 100% rename from airflow/providers/databricks/utils/__init__.py rename to providers/src/airflow/providers/databricks/sensors/__init__.py diff --git a/airflow/providers/databricks/sensors/databricks_partition.py b/providers/src/airflow/providers/databricks/sensors/databricks_partition.py similarity index 100% rename from airflow/providers/databricks/sensors/databricks_partition.py rename to providers/src/airflow/providers/databricks/sensors/databricks_partition.py diff --git a/airflow/providers/databricks/sensors/databricks_sql.py b/providers/src/airflow/providers/databricks/sensors/databricks_sql.py similarity index 100% rename from airflow/providers/databricks/sensors/databricks_sql.py rename to providers/src/airflow/providers/databricks/sensors/databricks_sql.py diff --git a/airflow/providers/databricks/operators/__init__.py b/providers/src/airflow/providers/databricks/triggers/__init__.py similarity index 100% rename from airflow/providers/databricks/operators/__init__.py rename to providers/src/airflow/providers/databricks/triggers/__init__.py diff --git a/airflow/providers/databricks/triggers/databricks.py b/providers/src/airflow/providers/databricks/triggers/databricks.py similarity index 100% rename from airflow/providers/databricks/triggers/databricks.py rename to providers/src/airflow/providers/databricks/triggers/databricks.py diff --git a/airflow/providers/dbt/__init__.py b/providers/src/airflow/providers/databricks/utils/__init__.py similarity index 100% rename from airflow/providers/dbt/__init__.py rename to providers/src/airflow/providers/databricks/utils/__init__.py diff --git a/airflow/providers/databricks/utils/databricks.py b/providers/src/airflow/providers/databricks/utils/databricks.py similarity index 100% rename from airflow/providers/databricks/utils/databricks.py rename to providers/src/airflow/providers/databricks/utils/databricks.py diff --git a/airflow/providers/datadog/.latest-doc-only-change.txt b/providers/src/airflow/providers/datadog/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/datadog/.latest-doc-only-change.txt rename to providers/src/airflow/providers/datadog/.latest-doc-only-change.txt diff --git a/airflow/providers/datadog/CHANGELOG.rst b/providers/src/airflow/providers/datadog/CHANGELOG.rst similarity index 100% rename from airflow/providers/datadog/CHANGELOG.rst rename to providers/src/airflow/providers/datadog/CHANGELOG.rst diff --git a/airflow/providers/datadog/__init__.py b/providers/src/airflow/providers/datadog/__init__.py similarity index 100% rename from airflow/providers/datadog/__init__.py rename to providers/src/airflow/providers/datadog/__init__.py diff --git a/airflow/providers/databricks/triggers/__init__.py b/providers/src/airflow/providers/datadog/hooks/__init__.py similarity index 100% rename from airflow/providers/databricks/triggers/__init__.py rename to providers/src/airflow/providers/datadog/hooks/__init__.py diff --git a/airflow/providers/datadog/hooks/datadog.py b/providers/src/airflow/providers/datadog/hooks/datadog.py similarity index 100% rename from airflow/providers/datadog/hooks/datadog.py rename to providers/src/airflow/providers/datadog/hooks/datadog.py diff --git a/airflow/providers/datadog/provider.yaml b/providers/src/airflow/providers/datadog/provider.yaml similarity index 100% rename from airflow/providers/datadog/provider.yaml rename to providers/src/airflow/providers/datadog/provider.yaml diff --git a/airflow/providers/datadog/hooks/__init__.py b/providers/src/airflow/providers/datadog/sensors/__init__.py similarity index 100% rename from airflow/providers/datadog/hooks/__init__.py rename to providers/src/airflow/providers/datadog/sensors/__init__.py diff --git a/airflow/providers/datadog/sensors/datadog.py b/providers/src/airflow/providers/datadog/sensors/datadog.py similarity index 100% rename from airflow/providers/datadog/sensors/datadog.py rename to providers/src/airflow/providers/datadog/sensors/datadog.py diff --git a/airflow/providers/dbt/cloud/.latest-doc-only-change.txt b/providers/src/airflow/providers/dbt/cloud/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/dbt/cloud/.latest-doc-only-change.txt rename to providers/src/airflow/providers/dbt/cloud/.latest-doc-only-change.txt diff --git a/airflow/providers/dbt/cloud/CHANGELOG.rst b/providers/src/airflow/providers/dbt/cloud/CHANGELOG.rst similarity index 100% rename from airflow/providers/dbt/cloud/CHANGELOG.rst rename to providers/src/airflow/providers/dbt/cloud/CHANGELOG.rst diff --git a/airflow/providers/dbt/cloud/__init__.py b/providers/src/airflow/providers/dbt/cloud/__init__.py similarity index 100% rename from airflow/providers/dbt/cloud/__init__.py rename to providers/src/airflow/providers/dbt/cloud/__init__.py diff --git a/airflow/providers/dbt/cloud/hooks/__init__.py b/providers/src/airflow/providers/dbt/cloud/hooks/__init__.py similarity index 100% rename from airflow/providers/dbt/cloud/hooks/__init__.py rename to providers/src/airflow/providers/dbt/cloud/hooks/__init__.py diff --git a/airflow/providers/dbt/cloud/hooks/dbt.py b/providers/src/airflow/providers/dbt/cloud/hooks/dbt.py similarity index 99% rename from airflow/providers/dbt/cloud/hooks/dbt.py rename to providers/src/airflow/providers/dbt/cloud/hooks/dbt.py index 4007054be69c2..7c38001c2a767 100644 --- a/airflow/providers/dbt/cloud/hooks/dbt.py +++ b/providers/src/airflow/providers/dbt/cloud/hooks/dbt.py @@ -20,10 +20,11 @@ import json import time import warnings +from collections.abc import Sequence from enum import Enum from functools import cached_property, wraps from inspect import signature -from typing import TYPE_CHECKING, Any, Callable, Sequence, Set, TypeVar, cast +from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast import aiohttp from asgiref.sync import sync_to_async @@ -117,7 +118,7 @@ class DbtCloudJobRunStatus(Enum): @classmethod def check_is_valid(cls, statuses: int | Sequence[int] | set[int]): """Validate input statuses are a known value.""" - if isinstance(statuses, (Sequence, Set)): + if isinstance(statuses, (Sequence, set)): for status in statuses: cls(status) else: diff --git a/airflow/providers/dbt/cloud/operators/__init__.py b/providers/src/airflow/providers/dbt/cloud/operators/__init__.py similarity index 100% rename from airflow/providers/dbt/cloud/operators/__init__.py rename to providers/src/airflow/providers/dbt/cloud/operators/__init__.py diff --git a/airflow/providers/dbt/cloud/operators/dbt.py b/providers/src/airflow/providers/dbt/cloud/operators/dbt.py similarity index 100% rename from airflow/providers/dbt/cloud/operators/dbt.py rename to providers/src/airflow/providers/dbt/cloud/operators/dbt.py diff --git a/airflow/providers/dbt/cloud/provider.yaml b/providers/src/airflow/providers/dbt/cloud/provider.yaml similarity index 100% rename from airflow/providers/dbt/cloud/provider.yaml rename to providers/src/airflow/providers/dbt/cloud/provider.yaml diff --git a/airflow/providers/datadog/sensors/__init__.py b/providers/src/airflow/providers/dbt/cloud/sensors/__init__.py similarity index 100% rename from airflow/providers/datadog/sensors/__init__.py rename to providers/src/airflow/providers/dbt/cloud/sensors/__init__.py diff --git a/airflow/providers/dbt/cloud/sensors/dbt.py b/providers/src/airflow/providers/dbt/cloud/sensors/dbt.py similarity index 100% rename from airflow/providers/dbt/cloud/sensors/dbt.py rename to providers/src/airflow/providers/dbt/cloud/sensors/dbt.py diff --git a/airflow/providers/dbt/cloud/triggers/__init__.py b/providers/src/airflow/providers/dbt/cloud/triggers/__init__.py similarity index 100% rename from airflow/providers/dbt/cloud/triggers/__init__.py rename to providers/src/airflow/providers/dbt/cloud/triggers/__init__.py diff --git a/airflow/providers/dbt/cloud/triggers/dbt.py b/providers/src/airflow/providers/dbt/cloud/triggers/dbt.py similarity index 100% rename from airflow/providers/dbt/cloud/triggers/dbt.py rename to providers/src/airflow/providers/dbt/cloud/triggers/dbt.py diff --git a/airflow/providers/dbt/cloud/utils/__init__.py b/providers/src/airflow/providers/dbt/cloud/utils/__init__.py similarity index 100% rename from airflow/providers/dbt/cloud/utils/__init__.py rename to providers/src/airflow/providers/dbt/cloud/utils/__init__.py diff --git a/airflow/providers/dbt/cloud/utils/openlineage.py b/providers/src/airflow/providers/dbt/cloud/utils/openlineage.py similarity index 100% rename from airflow/providers/dbt/cloud/utils/openlineage.py rename to providers/src/airflow/providers/dbt/cloud/utils/openlineage.py diff --git a/airflow/providers/dingding/.latest-doc-only-change.txt b/providers/src/airflow/providers/dingding/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/dingding/.latest-doc-only-change.txt rename to providers/src/airflow/providers/dingding/.latest-doc-only-change.txt diff --git a/airflow/providers/dingding/CHANGELOG.rst b/providers/src/airflow/providers/dingding/CHANGELOG.rst similarity index 100% rename from airflow/providers/dingding/CHANGELOG.rst rename to providers/src/airflow/providers/dingding/CHANGELOG.rst diff --git a/airflow/providers/dingding/__init__.py b/providers/src/airflow/providers/dingding/__init__.py similarity index 100% rename from airflow/providers/dingding/__init__.py rename to providers/src/airflow/providers/dingding/__init__.py diff --git a/airflow/providers/dbt/cloud/sensors/__init__.py b/providers/src/airflow/providers/dingding/hooks/__init__.py similarity index 100% rename from airflow/providers/dbt/cloud/sensors/__init__.py rename to providers/src/airflow/providers/dingding/hooks/__init__.py diff --git a/airflow/providers/dingding/hooks/dingding.py b/providers/src/airflow/providers/dingding/hooks/dingding.py similarity index 100% rename from airflow/providers/dingding/hooks/dingding.py rename to providers/src/airflow/providers/dingding/hooks/dingding.py diff --git a/airflow/providers/dingding/hooks/__init__.py b/providers/src/airflow/providers/dingding/operators/__init__.py similarity index 100% rename from airflow/providers/dingding/hooks/__init__.py rename to providers/src/airflow/providers/dingding/operators/__init__.py diff --git a/airflow/providers/dingding/operators/dingding.py b/providers/src/airflow/providers/dingding/operators/dingding.py similarity index 100% rename from airflow/providers/dingding/operators/dingding.py rename to providers/src/airflow/providers/dingding/operators/dingding.py diff --git a/airflow/providers/dingding/provider.yaml b/providers/src/airflow/providers/dingding/provider.yaml similarity index 100% rename from airflow/providers/dingding/provider.yaml rename to providers/src/airflow/providers/dingding/provider.yaml diff --git a/airflow/providers/discord/.latest-doc-only-change.txt b/providers/src/airflow/providers/discord/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/discord/.latest-doc-only-change.txt rename to providers/src/airflow/providers/discord/.latest-doc-only-change.txt diff --git a/airflow/providers/discord/CHANGELOG.rst b/providers/src/airflow/providers/discord/CHANGELOG.rst similarity index 100% rename from airflow/providers/discord/CHANGELOG.rst rename to providers/src/airflow/providers/discord/CHANGELOG.rst diff --git a/airflow/providers/discord/__init__.py b/providers/src/airflow/providers/discord/__init__.py similarity index 100% rename from airflow/providers/discord/__init__.py rename to providers/src/airflow/providers/discord/__init__.py diff --git a/airflow/providers/dingding/operators/__init__.py b/providers/src/airflow/providers/discord/hooks/__init__.py similarity index 100% rename from airflow/providers/dingding/operators/__init__.py rename to providers/src/airflow/providers/discord/hooks/__init__.py diff --git a/airflow/providers/discord/hooks/discord_webhook.py b/providers/src/airflow/providers/discord/hooks/discord_webhook.py similarity index 100% rename from airflow/providers/discord/hooks/discord_webhook.py rename to providers/src/airflow/providers/discord/hooks/discord_webhook.py diff --git a/airflow/providers/discord/hooks/__init__.py b/providers/src/airflow/providers/discord/notifications/__init__.py similarity index 100% rename from airflow/providers/discord/hooks/__init__.py rename to providers/src/airflow/providers/discord/notifications/__init__.py diff --git a/airflow/providers/discord/notifications/discord.py b/providers/src/airflow/providers/discord/notifications/discord.py similarity index 100% rename from airflow/providers/discord/notifications/discord.py rename to providers/src/airflow/providers/discord/notifications/discord.py diff --git a/airflow/providers/discord/notifications/__init__.py b/providers/src/airflow/providers/discord/operators/__init__.py similarity index 100% rename from airflow/providers/discord/notifications/__init__.py rename to providers/src/airflow/providers/discord/operators/__init__.py diff --git a/airflow/providers/discord/operators/discord_webhook.py b/providers/src/airflow/providers/discord/operators/discord_webhook.py similarity index 100% rename from airflow/providers/discord/operators/discord_webhook.py rename to providers/src/airflow/providers/discord/operators/discord_webhook.py diff --git a/airflow/providers/discord/provider.yaml b/providers/src/airflow/providers/discord/provider.yaml similarity index 100% rename from airflow/providers/discord/provider.yaml rename to providers/src/airflow/providers/discord/provider.yaml diff --git a/airflow/providers/docker/.latest-doc-only-change.txt b/providers/src/airflow/providers/docker/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/docker/.latest-doc-only-change.txt rename to providers/src/airflow/providers/docker/.latest-doc-only-change.txt diff --git a/airflow/providers/docker/CHANGELOG.rst b/providers/src/airflow/providers/docker/CHANGELOG.rst similarity index 100% rename from airflow/providers/docker/CHANGELOG.rst rename to providers/src/airflow/providers/docker/CHANGELOG.rst diff --git a/airflow/providers/docker/__init__.py b/providers/src/airflow/providers/docker/__init__.py similarity index 100% rename from airflow/providers/docker/__init__.py rename to providers/src/airflow/providers/docker/__init__.py diff --git a/airflow/providers/discord/operators/__init__.py b/providers/src/airflow/providers/docker/decorators/__init__.py similarity index 100% rename from airflow/providers/discord/operators/__init__.py rename to providers/src/airflow/providers/docker/decorators/__init__.py diff --git a/airflow/providers/docker/decorators/docker.py b/providers/src/airflow/providers/docker/decorators/docker.py similarity index 100% rename from airflow/providers/docker/decorators/docker.py rename to providers/src/airflow/providers/docker/decorators/docker.py diff --git a/airflow/providers/docker/exceptions.py b/providers/src/airflow/providers/docker/exceptions.py similarity index 100% rename from airflow/providers/docker/exceptions.py rename to providers/src/airflow/providers/docker/exceptions.py diff --git a/airflow/providers/docker/decorators/__init__.py b/providers/src/airflow/providers/docker/hooks/__init__.py similarity index 100% rename from airflow/providers/docker/decorators/__init__.py rename to providers/src/airflow/providers/docker/hooks/__init__.py diff --git a/airflow/providers/docker/hooks/docker.py b/providers/src/airflow/providers/docker/hooks/docker.py similarity index 100% rename from airflow/providers/docker/hooks/docker.py rename to providers/src/airflow/providers/docker/hooks/docker.py diff --git a/airflow/providers/docker/hooks/__init__.py b/providers/src/airflow/providers/docker/operators/__init__.py similarity index 100% rename from airflow/providers/docker/hooks/__init__.py rename to providers/src/airflow/providers/docker/operators/__init__.py diff --git a/airflow/providers/docker/operators/docker.py b/providers/src/airflow/providers/docker/operators/docker.py similarity index 100% rename from airflow/providers/docker/operators/docker.py rename to providers/src/airflow/providers/docker/operators/docker.py diff --git a/airflow/providers/docker/operators/docker_swarm.py b/providers/src/airflow/providers/docker/operators/docker_swarm.py similarity index 100% rename from airflow/providers/docker/operators/docker_swarm.py rename to providers/src/airflow/providers/docker/operators/docker_swarm.py diff --git a/airflow/providers/docker/provider.yaml b/providers/src/airflow/providers/docker/provider.yaml similarity index 100% rename from airflow/providers/docker/provider.yaml rename to providers/src/airflow/providers/docker/provider.yaml diff --git a/airflow/providers/edge/CHANGELOG.rst b/providers/src/airflow/providers/edge/CHANGELOG.rst similarity index 100% rename from airflow/providers/edge/CHANGELOG.rst rename to providers/src/airflow/providers/edge/CHANGELOG.rst diff --git a/airflow/providers/edge/__init__.py b/providers/src/airflow/providers/edge/__init__.py similarity index 100% rename from airflow/providers/edge/__init__.py rename to providers/src/airflow/providers/edge/__init__.py diff --git a/airflow/providers/edge/api_endpoints/__init__.py b/providers/src/airflow/providers/edge/api_endpoints/__init__.py similarity index 100% rename from airflow/providers/edge/api_endpoints/__init__.py rename to providers/src/airflow/providers/edge/api_endpoints/__init__.py diff --git a/airflow/providers/edge/api_endpoints/health_endpoint.py b/providers/src/airflow/providers/edge/api_endpoints/health_endpoint.py similarity index 100% rename from airflow/providers/edge/api_endpoints/health_endpoint.py rename to providers/src/airflow/providers/edge/api_endpoints/health_endpoint.py diff --git a/airflow/providers/edge/api_endpoints/rpc_api_endpoint.py b/providers/src/airflow/providers/edge/api_endpoints/rpc_api_endpoint.py similarity index 100% rename from airflow/providers/edge/api_endpoints/rpc_api_endpoint.py rename to providers/src/airflow/providers/edge/api_endpoints/rpc_api_endpoint.py diff --git a/airflow/providers/edge/cli/__init__.py b/providers/src/airflow/providers/edge/cli/__init__.py similarity index 100% rename from airflow/providers/edge/cli/__init__.py rename to providers/src/airflow/providers/edge/cli/__init__.py diff --git a/airflow/providers/edge/cli/edge_command.py b/providers/src/airflow/providers/edge/cli/edge_command.py similarity index 100% rename from airflow/providers/edge/cli/edge_command.py rename to providers/src/airflow/providers/edge/cli/edge_command.py diff --git a/airflow/providers/edge/example_dags/__init__.py b/providers/src/airflow/providers/edge/example_dags/__init__.py similarity index 100% rename from airflow/providers/edge/example_dags/__init__.py rename to providers/src/airflow/providers/edge/example_dags/__init__.py diff --git a/airflow/providers/edge/example_dags/integration_test.py b/providers/src/airflow/providers/edge/example_dags/integration_test.py similarity index 100% rename from airflow/providers/edge/example_dags/integration_test.py rename to providers/src/airflow/providers/edge/example_dags/integration_test.py diff --git a/airflow/providers/edge/models/__init__.py b/providers/src/airflow/providers/edge/models/__init__.py similarity index 100% rename from airflow/providers/edge/models/__init__.py rename to providers/src/airflow/providers/edge/models/__init__.py diff --git a/airflow/providers/edge/models/edge_job.py b/providers/src/airflow/providers/edge/models/edge_job.py similarity index 100% rename from airflow/providers/edge/models/edge_job.py rename to providers/src/airflow/providers/edge/models/edge_job.py diff --git a/airflow/providers/edge/models/edge_logs.py b/providers/src/airflow/providers/edge/models/edge_logs.py similarity index 100% rename from airflow/providers/edge/models/edge_logs.py rename to providers/src/airflow/providers/edge/models/edge_logs.py diff --git a/airflow/providers/edge/models/edge_worker.py b/providers/src/airflow/providers/edge/models/edge_worker.py similarity index 100% rename from airflow/providers/edge/models/edge_worker.py rename to providers/src/airflow/providers/edge/models/edge_worker.py diff --git a/airflow/providers/edge/openapi/__init__.py b/providers/src/airflow/providers/edge/openapi/__init__.py similarity index 100% rename from airflow/providers/edge/openapi/__init__.py rename to providers/src/airflow/providers/edge/openapi/__init__.py diff --git a/airflow/providers/edge/openapi/edge_worker_api_v1.yaml b/providers/src/airflow/providers/edge/openapi/edge_worker_api_v1.yaml similarity index 100% rename from airflow/providers/edge/openapi/edge_worker_api_v1.yaml rename to providers/src/airflow/providers/edge/openapi/edge_worker_api_v1.yaml diff --git a/airflow/providers/edge/plugins/__init__.py b/providers/src/airflow/providers/edge/plugins/__init__.py similarity index 100% rename from airflow/providers/edge/plugins/__init__.py rename to providers/src/airflow/providers/edge/plugins/__init__.py diff --git a/airflow/providers/edge/plugins/edge_executor_plugin.py b/providers/src/airflow/providers/edge/plugins/edge_executor_plugin.py similarity index 100% rename from airflow/providers/edge/plugins/edge_executor_plugin.py rename to providers/src/airflow/providers/edge/plugins/edge_executor_plugin.py diff --git a/airflow/providers/edge/plugins/templates/edge_worker_hosts.html b/providers/src/airflow/providers/edge/plugins/templates/edge_worker_hosts.html similarity index 100% rename from airflow/providers/edge/plugins/templates/edge_worker_hosts.html rename to providers/src/airflow/providers/edge/plugins/templates/edge_worker_hosts.html diff --git a/airflow/providers/edge/plugins/templates/edge_worker_jobs.html b/providers/src/airflow/providers/edge/plugins/templates/edge_worker_jobs.html similarity index 100% rename from airflow/providers/edge/plugins/templates/edge_worker_jobs.html rename to providers/src/airflow/providers/edge/plugins/templates/edge_worker_jobs.html diff --git a/airflow/providers/edge/provider.yaml b/providers/src/airflow/providers/edge/provider.yaml similarity index 100% rename from airflow/providers/edge/provider.yaml rename to providers/src/airflow/providers/edge/provider.yaml diff --git a/airflow/providers/elasticsearch/.latest-doc-only-change.txt b/providers/src/airflow/providers/elasticsearch/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/elasticsearch/.latest-doc-only-change.txt rename to providers/src/airflow/providers/elasticsearch/.latest-doc-only-change.txt diff --git a/airflow/providers/elasticsearch/CHANGELOG.rst b/providers/src/airflow/providers/elasticsearch/CHANGELOG.rst similarity index 100% rename from airflow/providers/elasticsearch/CHANGELOG.rst rename to providers/src/airflow/providers/elasticsearch/CHANGELOG.rst diff --git a/airflow/providers/elasticsearch/__init__.py b/providers/src/airflow/providers/elasticsearch/__init__.py similarity index 100% rename from airflow/providers/elasticsearch/__init__.py rename to providers/src/airflow/providers/elasticsearch/__init__.py diff --git a/airflow/providers/docker/operators/__init__.py b/providers/src/airflow/providers/elasticsearch/hooks/__init__.py similarity index 100% rename from airflow/providers/docker/operators/__init__.py rename to providers/src/airflow/providers/elasticsearch/hooks/__init__.py diff --git a/airflow/providers/elasticsearch/hooks/elasticsearch.py b/providers/src/airflow/providers/elasticsearch/hooks/elasticsearch.py similarity index 100% rename from airflow/providers/elasticsearch/hooks/elasticsearch.py rename to providers/src/airflow/providers/elasticsearch/hooks/elasticsearch.py diff --git a/airflow/providers/elasticsearch/log/__init__.py b/providers/src/airflow/providers/elasticsearch/log/__init__.py similarity index 100% rename from airflow/providers/elasticsearch/log/__init__.py rename to providers/src/airflow/providers/elasticsearch/log/__init__.py diff --git a/airflow/providers/elasticsearch/log/es_json_formatter.py b/providers/src/airflow/providers/elasticsearch/log/es_json_formatter.py similarity index 100% rename from airflow/providers/elasticsearch/log/es_json_formatter.py rename to providers/src/airflow/providers/elasticsearch/log/es_json_formatter.py diff --git a/airflow/providers/elasticsearch/log/es_response.py b/providers/src/airflow/providers/elasticsearch/log/es_response.py similarity index 100% rename from airflow/providers/elasticsearch/log/es_response.py rename to providers/src/airflow/providers/elasticsearch/log/es_response.py diff --git a/airflow/providers/elasticsearch/log/es_task_handler.py b/providers/src/airflow/providers/elasticsearch/log/es_task_handler.py similarity index 100% rename from airflow/providers/elasticsearch/log/es_task_handler.py rename to providers/src/airflow/providers/elasticsearch/log/es_task_handler.py diff --git a/airflow/providers/elasticsearch/provider.yaml b/providers/src/airflow/providers/elasticsearch/provider.yaml similarity index 100% rename from airflow/providers/elasticsearch/provider.yaml rename to providers/src/airflow/providers/elasticsearch/provider.yaml diff --git a/airflow/providers/exasol/.latest-doc-only-change.txt b/providers/src/airflow/providers/exasol/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/exasol/.latest-doc-only-change.txt rename to providers/src/airflow/providers/exasol/.latest-doc-only-change.txt diff --git a/airflow/providers/exasol/CHANGELOG.rst b/providers/src/airflow/providers/exasol/CHANGELOG.rst similarity index 100% rename from airflow/providers/exasol/CHANGELOG.rst rename to providers/src/airflow/providers/exasol/CHANGELOG.rst diff --git a/airflow/providers/exasol/__init__.py b/providers/src/airflow/providers/exasol/__init__.py similarity index 100% rename from airflow/providers/exasol/__init__.py rename to providers/src/airflow/providers/exasol/__init__.py diff --git a/airflow/providers/elasticsearch/hooks/__init__.py b/providers/src/airflow/providers/exasol/hooks/__init__.py similarity index 100% rename from airflow/providers/elasticsearch/hooks/__init__.py rename to providers/src/airflow/providers/exasol/hooks/__init__.py diff --git a/airflow/providers/exasol/hooks/exasol.py b/providers/src/airflow/providers/exasol/hooks/exasol.py similarity index 100% rename from airflow/providers/exasol/hooks/exasol.py rename to providers/src/airflow/providers/exasol/hooks/exasol.py diff --git a/airflow/providers/exasol/hooks/__init__.py b/providers/src/airflow/providers/exasol/operators/__init__.py similarity index 100% rename from airflow/providers/exasol/hooks/__init__.py rename to providers/src/airflow/providers/exasol/operators/__init__.py diff --git a/airflow/providers/exasol/operators/exasol.py b/providers/src/airflow/providers/exasol/operators/exasol.py similarity index 100% rename from airflow/providers/exasol/operators/exasol.py rename to providers/src/airflow/providers/exasol/operators/exasol.py diff --git a/airflow/providers/exasol/provider.yaml b/providers/src/airflow/providers/exasol/provider.yaml similarity index 100% rename from airflow/providers/exasol/provider.yaml rename to providers/src/airflow/providers/exasol/provider.yaml diff --git a/airflow/providers/fab/CHANGELOG.rst b/providers/src/airflow/providers/fab/CHANGELOG.rst similarity index 100% rename from airflow/providers/fab/CHANGELOG.rst rename to providers/src/airflow/providers/fab/CHANGELOG.rst diff --git a/airflow/providers/fab/__init__.py b/providers/src/airflow/providers/fab/__init__.py similarity index 100% rename from airflow/providers/fab/__init__.py rename to providers/src/airflow/providers/fab/__init__.py diff --git a/airflow/providers/fab/alembic.ini b/providers/src/airflow/providers/fab/alembic.ini similarity index 100% rename from airflow/providers/fab/alembic.ini rename to providers/src/airflow/providers/fab/alembic.ini diff --git a/airflow/providers/exasol/operators/__init__.py b/providers/src/airflow/providers/fab/auth_manager/__init__.py similarity index 100% rename from airflow/providers/exasol/operators/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/__init__.py diff --git a/airflow/providers/fab/auth_manager/__init__.py b/providers/src/airflow/providers/fab/auth_manager/api/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/api/__init__.py diff --git a/airflow/providers/fab/auth_manager/api/__init__.py b/providers/src/airflow/providers/fab/auth_manager/api/auth/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/api/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/api/auth/__init__.py diff --git a/airflow/providers/fab/auth_manager/api/auth/__init__.py b/providers/src/airflow/providers/fab/auth_manager/api/auth/backend/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/api/auth/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/api/auth/backend/__init__.py diff --git a/airflow/providers/fab/auth_manager/api/auth/backend/basic_auth.py b/providers/src/airflow/providers/fab/auth_manager/api/auth/backend/basic_auth.py similarity index 100% rename from airflow/providers/fab/auth_manager/api/auth/backend/basic_auth.py rename to providers/src/airflow/providers/fab/auth_manager/api/auth/backend/basic_auth.py diff --git a/airflow/providers/fab/auth_manager/api/auth/backend/kerberos_auth.py b/providers/src/airflow/providers/fab/auth_manager/api/auth/backend/kerberos_auth.py similarity index 100% rename from airflow/providers/fab/auth_manager/api/auth/backend/kerberos_auth.py rename to providers/src/airflow/providers/fab/auth_manager/api/auth/backend/kerberos_auth.py diff --git a/airflow/providers/fab/auth_manager/api_endpoints/__init__.py b/providers/src/airflow/providers/fab/auth_manager/api_endpoints/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/api_endpoints/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/api_endpoints/__init__.py diff --git a/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py b/providers/src/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py similarity index 100% rename from airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py rename to providers/src/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py diff --git a/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py b/providers/src/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py similarity index 100% rename from airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py rename to providers/src/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py diff --git a/airflow/providers/fab/auth_manager/api/auth/backend/__init__.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/api/auth/backend/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/cli_commands/__init__.py diff --git a/airflow/providers/fab/auth_manager/cli_commands/db_command.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/db_command.py similarity index 100% rename from airflow/providers/fab/auth_manager/cli_commands/db_command.py rename to providers/src/airflow/providers/fab/auth_manager/cli_commands/db_command.py diff --git a/airflow/providers/fab/auth_manager/cli_commands/definition.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/definition.py similarity index 100% rename from airflow/providers/fab/auth_manager/cli_commands/definition.py rename to providers/src/airflow/providers/fab/auth_manager/cli_commands/definition.py diff --git a/airflow/providers/fab/auth_manager/cli_commands/role_command.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/role_command.py similarity index 100% rename from airflow/providers/fab/auth_manager/cli_commands/role_command.py rename to providers/src/airflow/providers/fab/auth_manager/cli_commands/role_command.py diff --git a/airflow/providers/fab/auth_manager/cli_commands/sync_perm_command.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/sync_perm_command.py similarity index 100% rename from airflow/providers/fab/auth_manager/cli_commands/sync_perm_command.py rename to providers/src/airflow/providers/fab/auth_manager/cli_commands/sync_perm_command.py diff --git a/airflow/providers/fab/auth_manager/cli_commands/user_command.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/user_command.py similarity index 100% rename from airflow/providers/fab/auth_manager/cli_commands/user_command.py rename to providers/src/airflow/providers/fab/auth_manager/cli_commands/user_command.py diff --git a/airflow/providers/fab/auth_manager/cli_commands/utils.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py similarity index 100% rename from airflow/providers/fab/auth_manager/cli_commands/utils.py rename to providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py diff --git a/airflow/providers/fab/auth_manager/cli_commands/__init__.py b/providers/src/airflow/providers/fab/auth_manager/decorators/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/cli_commands/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/decorators/__init__.py diff --git a/airflow/providers/fab/auth_manager/decorators/auth.py b/providers/src/airflow/providers/fab/auth_manager/decorators/auth.py similarity index 100% rename from airflow/providers/fab/auth_manager/decorators/auth.py rename to providers/src/airflow/providers/fab/auth_manager/decorators/auth.py diff --git a/airflow/providers/fab/auth_manager/fab_auth_manager.py b/providers/src/airflow/providers/fab/auth_manager/fab_auth_manager.py similarity index 100% rename from airflow/providers/fab/auth_manager/fab_auth_manager.py rename to providers/src/airflow/providers/fab/auth_manager/fab_auth_manager.py diff --git a/airflow/providers/fab/auth_manager/models/__init__.py b/providers/src/airflow/providers/fab/auth_manager/models/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/models/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/models/__init__.py diff --git a/airflow/providers/fab/auth_manager/models/anonymous_user.py b/providers/src/airflow/providers/fab/auth_manager/models/anonymous_user.py similarity index 100% rename from airflow/providers/fab/auth_manager/models/anonymous_user.py rename to providers/src/airflow/providers/fab/auth_manager/models/anonymous_user.py diff --git a/airflow/providers/fab/auth_manager/models/db.py b/providers/src/airflow/providers/fab/auth_manager/models/db.py similarity index 94% rename from airflow/providers/fab/auth_manager/models/db.py rename to providers/src/airflow/providers/fab/auth_manager/models/db.py index efca4a1041827..ce0efef55a1cd 100644 --- a/airflow/providers/fab/auth_manager/models/db.py +++ b/providers/src/airflow/providers/fab/auth_manager/models/db.py @@ -16,16 +16,15 @@ # under the License. from __future__ import annotations -import os +from pathlib import Path -import airflow from airflow import settings from airflow.exceptions import AirflowException from airflow.providers.fab.auth_manager.models import metadata from airflow.utils.db import _offline_migration, print_happy_cat from airflow.utils.db_manager import BaseDBManager -PACKAGE_DIR = os.path.dirname(airflow.__file__) +PACKAGE_DIR = Path(__file__).parents[2] _REVISION_HEADS_MAP: dict[str, str] = { "1.4.0": "6709f7a774b9", @@ -37,8 +36,8 @@ class FABDBManager(BaseDBManager): metadata = metadata version_table_name = "alembic_version_fab" - migration_dir = os.path.join(PACKAGE_DIR, "providers/fab/migrations") - alembic_file = os.path.join(PACKAGE_DIR, "providers/fab/alembic.ini") + migration_dir = (PACKAGE_DIR / "migrations").as_posix() + alembic_file = (PACKAGE_DIR / "alembic.ini").as_posix() supports_table_dropping = True def upgradedb(self, to_revision=None, from_revision=None, show_sql_only=False): diff --git a/airflow/providers/fab/auth_manager/openapi/__init__.py b/providers/src/airflow/providers/fab/auth_manager/openapi/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/openapi/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/openapi/__init__.py diff --git a/airflow/providers/fab/auth_manager/openapi/v1.yaml b/providers/src/airflow/providers/fab/auth_manager/openapi/v1.yaml similarity index 100% rename from airflow/providers/fab/auth_manager/openapi/v1.yaml rename to providers/src/airflow/providers/fab/auth_manager/openapi/v1.yaml diff --git a/airflow/providers/fab/auth_manager/decorators/__init__.py b/providers/src/airflow/providers/fab/auth_manager/security_manager/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/decorators/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/security_manager/__init__.py diff --git a/airflow/providers/fab/auth_manager/security_manager/constants.py b/providers/src/airflow/providers/fab/auth_manager/security_manager/constants.py similarity index 100% rename from airflow/providers/fab/auth_manager/security_manager/constants.py rename to providers/src/airflow/providers/fab/auth_manager/security_manager/constants.py diff --git a/airflow/providers/fab/auth_manager/security_manager/override.py b/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py similarity index 100% rename from airflow/providers/fab/auth_manager/security_manager/override.py rename to providers/src/airflow/providers/fab/auth_manager/security_manager/override.py diff --git a/airflow/providers/fab/auth_manager/views/__init__.py b/providers/src/airflow/providers/fab/auth_manager/views/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/views/__init__.py rename to providers/src/airflow/providers/fab/auth_manager/views/__init__.py diff --git a/airflow/providers/fab/auth_manager/views/permissions.py b/providers/src/airflow/providers/fab/auth_manager/views/permissions.py similarity index 100% rename from airflow/providers/fab/auth_manager/views/permissions.py rename to providers/src/airflow/providers/fab/auth_manager/views/permissions.py diff --git a/airflow/providers/fab/auth_manager/views/roles_list.py b/providers/src/airflow/providers/fab/auth_manager/views/roles_list.py similarity index 100% rename from airflow/providers/fab/auth_manager/views/roles_list.py rename to providers/src/airflow/providers/fab/auth_manager/views/roles_list.py diff --git a/airflow/providers/fab/auth_manager/views/user.py b/providers/src/airflow/providers/fab/auth_manager/views/user.py similarity index 100% rename from airflow/providers/fab/auth_manager/views/user.py rename to providers/src/airflow/providers/fab/auth_manager/views/user.py diff --git a/airflow/providers/fab/auth_manager/views/user_edit.py b/providers/src/airflow/providers/fab/auth_manager/views/user_edit.py similarity index 100% rename from airflow/providers/fab/auth_manager/views/user_edit.py rename to providers/src/airflow/providers/fab/auth_manager/views/user_edit.py diff --git a/airflow/providers/fab/auth_manager/views/user_stats.py b/providers/src/airflow/providers/fab/auth_manager/views/user_stats.py similarity index 100% rename from airflow/providers/fab/auth_manager/views/user_stats.py rename to providers/src/airflow/providers/fab/auth_manager/views/user_stats.py diff --git a/airflow/providers/fab/migrations/README b/providers/src/airflow/providers/fab/migrations/README similarity index 100% rename from airflow/providers/fab/migrations/README rename to providers/src/airflow/providers/fab/migrations/README diff --git a/airflow/providers/fab/migrations/__init__.py b/providers/src/airflow/providers/fab/migrations/__init__.py similarity index 100% rename from airflow/providers/fab/migrations/__init__.py rename to providers/src/airflow/providers/fab/migrations/__init__.py diff --git a/airflow/providers/fab/migrations/env.py b/providers/src/airflow/providers/fab/migrations/env.py similarity index 100% rename from airflow/providers/fab/migrations/env.py rename to providers/src/airflow/providers/fab/migrations/env.py diff --git a/airflow/providers/fab/migrations/script.py.mako b/providers/src/airflow/providers/fab/migrations/script.py.mako similarity index 100% rename from airflow/providers/fab/migrations/script.py.mako rename to providers/src/airflow/providers/fab/migrations/script.py.mako diff --git a/airflow/providers/fab/migrations/versions/0001_1_4_0_placeholder_migration.py b/providers/src/airflow/providers/fab/migrations/versions/0001_1_4_0_placeholder_migration.py similarity index 100% rename from airflow/providers/fab/migrations/versions/0001_1_4_0_placeholder_migration.py rename to providers/src/airflow/providers/fab/migrations/versions/0001_1_4_0_placeholder_migration.py diff --git a/airflow/providers/fab/migrations/versions/__init__.py b/providers/src/airflow/providers/fab/migrations/versions/__init__.py similarity index 100% rename from airflow/providers/fab/migrations/versions/__init__.py rename to providers/src/airflow/providers/fab/migrations/versions/__init__.py diff --git a/airflow/providers/fab/provider.yaml b/providers/src/airflow/providers/fab/provider.yaml similarity index 100% rename from airflow/providers/fab/provider.yaml rename to providers/src/airflow/providers/fab/provider.yaml diff --git a/airflow/providers/facebook/.latest-doc-only-change.txt b/providers/src/airflow/providers/facebook/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/facebook/.latest-doc-only-change.txt rename to providers/src/airflow/providers/facebook/.latest-doc-only-change.txt diff --git a/airflow/providers/facebook/CHANGELOG.rst b/providers/src/airflow/providers/facebook/CHANGELOG.rst similarity index 100% rename from airflow/providers/facebook/CHANGELOG.rst rename to providers/src/airflow/providers/facebook/CHANGELOG.rst diff --git a/airflow/providers/facebook/__init__.py b/providers/src/airflow/providers/facebook/__init__.py similarity index 100% rename from airflow/providers/facebook/__init__.py rename to providers/src/airflow/providers/facebook/__init__.py diff --git a/airflow/providers/facebook/ads/__init__.py b/providers/src/airflow/providers/facebook/ads/__init__.py similarity index 100% rename from airflow/providers/facebook/ads/__init__.py rename to providers/src/airflow/providers/facebook/ads/__init__.py diff --git a/airflow/providers/facebook/ads/hooks/__init__.py b/providers/src/airflow/providers/facebook/ads/hooks/__init__.py similarity index 100% rename from airflow/providers/facebook/ads/hooks/__init__.py rename to providers/src/airflow/providers/facebook/ads/hooks/__init__.py diff --git a/airflow/providers/facebook/ads/hooks/ads.py b/providers/src/airflow/providers/facebook/ads/hooks/ads.py similarity index 100% rename from airflow/providers/facebook/ads/hooks/ads.py rename to providers/src/airflow/providers/facebook/ads/hooks/ads.py diff --git a/airflow/providers/facebook/provider.yaml b/providers/src/airflow/providers/facebook/provider.yaml similarity index 100% rename from airflow/providers/facebook/provider.yaml rename to providers/src/airflow/providers/facebook/provider.yaml diff --git a/airflow/providers/ftp/.latest-doc-only-change.txt b/providers/src/airflow/providers/ftp/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/ftp/.latest-doc-only-change.txt rename to providers/src/airflow/providers/ftp/.latest-doc-only-change.txt diff --git a/airflow/providers/ftp/CHANGELOG.rst b/providers/src/airflow/providers/ftp/CHANGELOG.rst similarity index 100% rename from airflow/providers/ftp/CHANGELOG.rst rename to providers/src/airflow/providers/ftp/CHANGELOG.rst diff --git a/airflow/providers/ftp/__init__.py b/providers/src/airflow/providers/ftp/__init__.py similarity index 100% rename from airflow/providers/ftp/__init__.py rename to providers/src/airflow/providers/ftp/__init__.py diff --git a/airflow/providers/fab/auth_manager/security_manager/__init__.py b/providers/src/airflow/providers/ftp/hooks/__init__.py similarity index 100% rename from airflow/providers/fab/auth_manager/security_manager/__init__.py rename to providers/src/airflow/providers/ftp/hooks/__init__.py diff --git a/airflow/providers/ftp/hooks/ftp.py b/providers/src/airflow/providers/ftp/hooks/ftp.py similarity index 100% rename from airflow/providers/ftp/hooks/ftp.py rename to providers/src/airflow/providers/ftp/hooks/ftp.py diff --git a/airflow/providers/ftp/operators/__init__.py b/providers/src/airflow/providers/ftp/operators/__init__.py similarity index 100% rename from airflow/providers/ftp/operators/__init__.py rename to providers/src/airflow/providers/ftp/operators/__init__.py diff --git a/airflow/providers/ftp/operators/ftp.py b/providers/src/airflow/providers/ftp/operators/ftp.py similarity index 100% rename from airflow/providers/ftp/operators/ftp.py rename to providers/src/airflow/providers/ftp/operators/ftp.py diff --git a/airflow/providers/ftp/provider.yaml b/providers/src/airflow/providers/ftp/provider.yaml similarity index 100% rename from airflow/providers/ftp/provider.yaml rename to providers/src/airflow/providers/ftp/provider.yaml diff --git a/airflow/providers/ftp/hooks/__init__.py b/providers/src/airflow/providers/ftp/sensors/__init__.py similarity index 100% rename from airflow/providers/ftp/hooks/__init__.py rename to providers/src/airflow/providers/ftp/sensors/__init__.py diff --git a/airflow/providers/ftp/sensors/ftp.py b/providers/src/airflow/providers/ftp/sensors/ftp.py similarity index 100% rename from airflow/providers/ftp/sensors/ftp.py rename to providers/src/airflow/providers/ftp/sensors/ftp.py diff --git a/airflow/providers/github/.latest-doc-only-change.txt b/providers/src/airflow/providers/github/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/github/.latest-doc-only-change.txt rename to providers/src/airflow/providers/github/.latest-doc-only-change.txt diff --git a/airflow/providers/github/CHANGELOG.rst b/providers/src/airflow/providers/github/CHANGELOG.rst similarity index 100% rename from airflow/providers/github/CHANGELOG.rst rename to providers/src/airflow/providers/github/CHANGELOG.rst diff --git a/airflow/providers/github/__init__.py b/providers/src/airflow/providers/github/__init__.py similarity index 100% rename from airflow/providers/github/__init__.py rename to providers/src/airflow/providers/github/__init__.py diff --git a/airflow/providers/github/hooks/__init__.py b/providers/src/airflow/providers/github/hooks/__init__.py similarity index 100% rename from airflow/providers/github/hooks/__init__.py rename to providers/src/airflow/providers/github/hooks/__init__.py diff --git a/airflow/providers/github/hooks/github.py b/providers/src/airflow/providers/github/hooks/github.py similarity index 100% rename from airflow/providers/github/hooks/github.py rename to providers/src/airflow/providers/github/hooks/github.py diff --git a/airflow/providers/github/operators/__init__.py b/providers/src/airflow/providers/github/operators/__init__.py similarity index 100% rename from airflow/providers/github/operators/__init__.py rename to providers/src/airflow/providers/github/operators/__init__.py diff --git a/airflow/providers/github/operators/github.py b/providers/src/airflow/providers/github/operators/github.py similarity index 100% rename from airflow/providers/github/operators/github.py rename to providers/src/airflow/providers/github/operators/github.py diff --git a/airflow/providers/github/provider.yaml b/providers/src/airflow/providers/github/provider.yaml similarity index 100% rename from airflow/providers/github/provider.yaml rename to providers/src/airflow/providers/github/provider.yaml diff --git a/airflow/providers/github/sensors/__init__.py b/providers/src/airflow/providers/github/sensors/__init__.py similarity index 100% rename from airflow/providers/github/sensors/__init__.py rename to providers/src/airflow/providers/github/sensors/__init__.py diff --git a/airflow/providers/github/sensors/github.py b/providers/src/airflow/providers/github/sensors/github.py similarity index 100% rename from airflow/providers/github/sensors/github.py rename to providers/src/airflow/providers/github/sensors/github.py diff --git a/airflow/providers/google/.latest-doc-only-change.txt b/providers/src/airflow/providers/google/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/google/.latest-doc-only-change.txt rename to providers/src/airflow/providers/google/.latest-doc-only-change.txt diff --git a/airflow/providers/google/CHANGELOG.rst b/providers/src/airflow/providers/google/CHANGELOG.rst similarity index 100% rename from airflow/providers/google/CHANGELOG.rst rename to providers/src/airflow/providers/google/CHANGELOG.rst diff --git a/airflow/providers/google/__init__.py b/providers/src/airflow/providers/google/__init__.py similarity index 100% rename from airflow/providers/google/__init__.py rename to providers/src/airflow/providers/google/__init__.py diff --git a/airflow/providers/google/ads/.gitignore b/providers/src/airflow/providers/google/ads/.gitignore similarity index 100% rename from airflow/providers/google/ads/.gitignore rename to providers/src/airflow/providers/google/ads/.gitignore diff --git a/airflow/providers/google/ads/__init__.py b/providers/src/airflow/providers/google/ads/__init__.py similarity index 100% rename from airflow/providers/google/ads/__init__.py rename to providers/src/airflow/providers/google/ads/__init__.py diff --git a/airflow/providers/google/ads/hooks/__init__.py b/providers/src/airflow/providers/google/ads/hooks/__init__.py similarity index 100% rename from airflow/providers/google/ads/hooks/__init__.py rename to providers/src/airflow/providers/google/ads/hooks/__init__.py diff --git a/airflow/providers/google/ads/hooks/ads.py b/providers/src/airflow/providers/google/ads/hooks/ads.py similarity index 100% rename from airflow/providers/google/ads/hooks/ads.py rename to providers/src/airflow/providers/google/ads/hooks/ads.py diff --git a/airflow/providers/google/ads/operators/__init__.py b/providers/src/airflow/providers/google/ads/operators/__init__.py similarity index 100% rename from airflow/providers/google/ads/operators/__init__.py rename to providers/src/airflow/providers/google/ads/operators/__init__.py diff --git a/airflow/providers/google/ads/operators/ads.py b/providers/src/airflow/providers/google/ads/operators/ads.py similarity index 100% rename from airflow/providers/google/ads/operators/ads.py rename to providers/src/airflow/providers/google/ads/operators/ads.py diff --git a/airflow/providers/google/ads/transfers/__init__.py b/providers/src/airflow/providers/google/ads/transfers/__init__.py similarity index 100% rename from airflow/providers/google/ads/transfers/__init__.py rename to providers/src/airflow/providers/google/ads/transfers/__init__.py diff --git a/airflow/providers/google/ads/transfers/ads_to_gcs.py b/providers/src/airflow/providers/google/ads/transfers/ads_to_gcs.py similarity index 100% rename from airflow/providers/google/ads/transfers/ads_to_gcs.py rename to providers/src/airflow/providers/google/ads/transfers/ads_to_gcs.py diff --git a/airflow/providers/google/cloud/__init__.py b/providers/src/airflow/providers/google/cloud/__init__.py similarity index 100% rename from airflow/providers/google/cloud/__init__.py rename to providers/src/airflow/providers/google/cloud/__init__.py diff --git a/airflow/providers/google/cloud/_internal_client/__init__.py b/providers/src/airflow/providers/google/cloud/_internal_client/__init__.py similarity index 100% rename from airflow/providers/google/cloud/_internal_client/__init__.py rename to providers/src/airflow/providers/google/cloud/_internal_client/__init__.py diff --git a/airflow/providers/google/cloud/_internal_client/secret_manager_client.py b/providers/src/airflow/providers/google/cloud/_internal_client/secret_manager_client.py similarity index 100% rename from airflow/providers/google/cloud/_internal_client/secret_manager_client.py rename to providers/src/airflow/providers/google/cloud/_internal_client/secret_manager_client.py diff --git a/airflow/providers/google/cloud/example_dags/__init__.py b/providers/src/airflow/providers/google/cloud/example_dags/__init__.py similarity index 100% rename from airflow/providers/google/cloud/example_dags/__init__.py rename to providers/src/airflow/providers/google/cloud/example_dags/__init__.py diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_task.py b/providers/src/airflow/providers/google/cloud/example_dags/example_cloud_task.py similarity index 100% rename from airflow/providers/google/cloud/example_dags/example_cloud_task.py rename to providers/src/airflow/providers/google/cloud/example_dags/example_cloud_task.py diff --git a/airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py b/providers/src/airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py rename to providers/src/airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py diff --git a/airflow/providers/google/cloud/example_dags/example_looker.py b/providers/src/airflow/providers/google/cloud/example_dags/example_looker.py similarity index 100% rename from airflow/providers/google/cloud/example_dags/example_looker.py rename to providers/src/airflow/providers/google/cloud/example_dags/example_looker.py diff --git a/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py b/providers/src/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py rename to providers/src/airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py diff --git a/airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py b/providers/src/airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py rename to providers/src/airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py diff --git a/airflow/providers/google/cloud/fs/__init__.py b/providers/src/airflow/providers/google/cloud/fs/__init__.py similarity index 100% rename from airflow/providers/google/cloud/fs/__init__.py rename to providers/src/airflow/providers/google/cloud/fs/__init__.py diff --git a/airflow/providers/google/cloud/fs/gcs.py b/providers/src/airflow/providers/google/cloud/fs/gcs.py similarity index 100% rename from airflow/providers/google/cloud/fs/gcs.py rename to providers/src/airflow/providers/google/cloud/fs/gcs.py diff --git a/airflow/providers/google/cloud/hooks/__init__.py b/providers/src/airflow/providers/google/cloud/hooks/__init__.py similarity index 100% rename from airflow/providers/google/cloud/hooks/__init__.py rename to providers/src/airflow/providers/google/cloud/hooks/__init__.py diff --git a/airflow/providers/google/cloud/hooks/automl.py b/providers/src/airflow/providers/google/cloud/hooks/automl.py similarity index 100% rename from airflow/providers/google/cloud/hooks/automl.py rename to providers/src/airflow/providers/google/cloud/hooks/automl.py diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/providers/src/airflow/providers/google/cloud/hooks/bigquery.py similarity index 100% rename from airflow/providers/google/cloud/hooks/bigquery.py rename to providers/src/airflow/providers/google/cloud/hooks/bigquery.py diff --git a/airflow/providers/google/cloud/hooks/bigquery_dts.py b/providers/src/airflow/providers/google/cloud/hooks/bigquery_dts.py similarity index 100% rename from airflow/providers/google/cloud/hooks/bigquery_dts.py rename to providers/src/airflow/providers/google/cloud/hooks/bigquery_dts.py diff --git a/airflow/providers/google/cloud/hooks/bigtable.py b/providers/src/airflow/providers/google/cloud/hooks/bigtable.py similarity index 100% rename from airflow/providers/google/cloud/hooks/bigtable.py rename to providers/src/airflow/providers/google/cloud/hooks/bigtable.py diff --git a/airflow/providers/google/cloud/hooks/cloud_batch.py b/providers/src/airflow/providers/google/cloud/hooks/cloud_batch.py similarity index 100% rename from airflow/providers/google/cloud/hooks/cloud_batch.py rename to providers/src/airflow/providers/google/cloud/hooks/cloud_batch.py diff --git a/airflow/providers/google/cloud/hooks/cloud_build.py b/providers/src/airflow/providers/google/cloud/hooks/cloud_build.py similarity index 100% rename from airflow/providers/google/cloud/hooks/cloud_build.py rename to providers/src/airflow/providers/google/cloud/hooks/cloud_build.py diff --git a/airflow/providers/google/cloud/hooks/cloud_composer.py b/providers/src/airflow/providers/google/cloud/hooks/cloud_composer.py similarity index 100% rename from airflow/providers/google/cloud/hooks/cloud_composer.py rename to providers/src/airflow/providers/google/cloud/hooks/cloud_composer.py diff --git a/airflow/providers/google/cloud/hooks/cloud_memorystore.py b/providers/src/airflow/providers/google/cloud/hooks/cloud_memorystore.py similarity index 100% rename from airflow/providers/google/cloud/hooks/cloud_memorystore.py rename to providers/src/airflow/providers/google/cloud/hooks/cloud_memorystore.py diff --git a/airflow/providers/google/cloud/hooks/cloud_run.py b/providers/src/airflow/providers/google/cloud/hooks/cloud_run.py similarity index 100% rename from airflow/providers/google/cloud/hooks/cloud_run.py rename to providers/src/airflow/providers/google/cloud/hooks/cloud_run.py diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py b/providers/src/airflow/providers/google/cloud/hooks/cloud_sql.py similarity index 100% rename from airflow/providers/google/cloud/hooks/cloud_sql.py rename to providers/src/airflow/providers/google/cloud/hooks/cloud_sql.py diff --git a/airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py b/providers/src/airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py similarity index 100% rename from airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py rename to providers/src/airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py diff --git a/airflow/providers/google/cloud/hooks/compute.py b/providers/src/airflow/providers/google/cloud/hooks/compute.py similarity index 100% rename from airflow/providers/google/cloud/hooks/compute.py rename to providers/src/airflow/providers/google/cloud/hooks/compute.py diff --git a/airflow/providers/google/cloud/hooks/compute_ssh.py b/providers/src/airflow/providers/google/cloud/hooks/compute_ssh.py similarity index 100% rename from airflow/providers/google/cloud/hooks/compute_ssh.py rename to providers/src/airflow/providers/google/cloud/hooks/compute_ssh.py diff --git a/airflow/providers/google/cloud/hooks/datacatalog.py b/providers/src/airflow/providers/google/cloud/hooks/datacatalog.py similarity index 100% rename from airflow/providers/google/cloud/hooks/datacatalog.py rename to providers/src/airflow/providers/google/cloud/hooks/datacatalog.py diff --git a/airflow/providers/google/cloud/hooks/dataflow.py b/providers/src/airflow/providers/google/cloud/hooks/dataflow.py similarity index 100% rename from airflow/providers/google/cloud/hooks/dataflow.py rename to providers/src/airflow/providers/google/cloud/hooks/dataflow.py diff --git a/airflow/providers/google/cloud/hooks/dataform.py b/providers/src/airflow/providers/google/cloud/hooks/dataform.py similarity index 100% rename from airflow/providers/google/cloud/hooks/dataform.py rename to providers/src/airflow/providers/google/cloud/hooks/dataform.py diff --git a/airflow/providers/google/cloud/hooks/datafusion.py b/providers/src/airflow/providers/google/cloud/hooks/datafusion.py similarity index 100% rename from airflow/providers/google/cloud/hooks/datafusion.py rename to providers/src/airflow/providers/google/cloud/hooks/datafusion.py diff --git a/airflow/providers/google/cloud/hooks/datapipeline.py b/providers/src/airflow/providers/google/cloud/hooks/datapipeline.py similarity index 100% rename from airflow/providers/google/cloud/hooks/datapipeline.py rename to providers/src/airflow/providers/google/cloud/hooks/datapipeline.py diff --git a/airflow/providers/google/cloud/hooks/dataplex.py b/providers/src/airflow/providers/google/cloud/hooks/dataplex.py similarity index 100% rename from airflow/providers/google/cloud/hooks/dataplex.py rename to providers/src/airflow/providers/google/cloud/hooks/dataplex.py diff --git a/airflow/providers/google/cloud/hooks/dataprep.py b/providers/src/airflow/providers/google/cloud/hooks/dataprep.py similarity index 100% rename from airflow/providers/google/cloud/hooks/dataprep.py rename to providers/src/airflow/providers/google/cloud/hooks/dataprep.py diff --git a/airflow/providers/google/cloud/hooks/dataproc.py b/providers/src/airflow/providers/google/cloud/hooks/dataproc.py similarity index 100% rename from airflow/providers/google/cloud/hooks/dataproc.py rename to providers/src/airflow/providers/google/cloud/hooks/dataproc.py diff --git a/airflow/providers/google/cloud/hooks/dataproc_metastore.py b/providers/src/airflow/providers/google/cloud/hooks/dataproc_metastore.py similarity index 100% rename from airflow/providers/google/cloud/hooks/dataproc_metastore.py rename to providers/src/airflow/providers/google/cloud/hooks/dataproc_metastore.py diff --git a/airflow/providers/google/cloud/hooks/datastore.py b/providers/src/airflow/providers/google/cloud/hooks/datastore.py similarity index 100% rename from airflow/providers/google/cloud/hooks/datastore.py rename to providers/src/airflow/providers/google/cloud/hooks/datastore.py diff --git a/airflow/providers/google/cloud/hooks/dlp.py b/providers/src/airflow/providers/google/cloud/hooks/dlp.py similarity index 100% rename from airflow/providers/google/cloud/hooks/dlp.py rename to providers/src/airflow/providers/google/cloud/hooks/dlp.py diff --git a/airflow/providers/google/cloud/hooks/functions.py b/providers/src/airflow/providers/google/cloud/hooks/functions.py similarity index 100% rename from airflow/providers/google/cloud/hooks/functions.py rename to providers/src/airflow/providers/google/cloud/hooks/functions.py diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/providers/src/airflow/providers/google/cloud/hooks/gcs.py similarity index 100% rename from airflow/providers/google/cloud/hooks/gcs.py rename to providers/src/airflow/providers/google/cloud/hooks/gcs.py diff --git a/airflow/providers/google/cloud/hooks/gdm.py b/providers/src/airflow/providers/google/cloud/hooks/gdm.py similarity index 100% rename from airflow/providers/google/cloud/hooks/gdm.py rename to providers/src/airflow/providers/google/cloud/hooks/gdm.py diff --git a/airflow/providers/google/cloud/hooks/kms.py b/providers/src/airflow/providers/google/cloud/hooks/kms.py similarity index 100% rename from airflow/providers/google/cloud/hooks/kms.py rename to providers/src/airflow/providers/google/cloud/hooks/kms.py diff --git a/airflow/providers/google/cloud/hooks/kubernetes_engine.py b/providers/src/airflow/providers/google/cloud/hooks/kubernetes_engine.py similarity index 100% rename from airflow/providers/google/cloud/hooks/kubernetes_engine.py rename to providers/src/airflow/providers/google/cloud/hooks/kubernetes_engine.py diff --git a/airflow/providers/google/cloud/hooks/life_sciences.py b/providers/src/airflow/providers/google/cloud/hooks/life_sciences.py similarity index 100% rename from airflow/providers/google/cloud/hooks/life_sciences.py rename to providers/src/airflow/providers/google/cloud/hooks/life_sciences.py diff --git a/airflow/providers/google/cloud/hooks/looker.py b/providers/src/airflow/providers/google/cloud/hooks/looker.py similarity index 100% rename from airflow/providers/google/cloud/hooks/looker.py rename to providers/src/airflow/providers/google/cloud/hooks/looker.py diff --git a/airflow/providers/google/cloud/hooks/mlengine.py b/providers/src/airflow/providers/google/cloud/hooks/mlengine.py similarity index 100% rename from airflow/providers/google/cloud/hooks/mlengine.py rename to providers/src/airflow/providers/google/cloud/hooks/mlengine.py diff --git a/airflow/providers/google/cloud/hooks/natural_language.py b/providers/src/airflow/providers/google/cloud/hooks/natural_language.py similarity index 100% rename from airflow/providers/google/cloud/hooks/natural_language.py rename to providers/src/airflow/providers/google/cloud/hooks/natural_language.py diff --git a/airflow/providers/google/cloud/hooks/os_login.py b/providers/src/airflow/providers/google/cloud/hooks/os_login.py similarity index 100% rename from airflow/providers/google/cloud/hooks/os_login.py rename to providers/src/airflow/providers/google/cloud/hooks/os_login.py diff --git a/airflow/providers/google/cloud/hooks/pubsub.py b/providers/src/airflow/providers/google/cloud/hooks/pubsub.py similarity index 100% rename from airflow/providers/google/cloud/hooks/pubsub.py rename to providers/src/airflow/providers/google/cloud/hooks/pubsub.py diff --git a/airflow/providers/google/cloud/hooks/secret_manager.py b/providers/src/airflow/providers/google/cloud/hooks/secret_manager.py similarity index 100% rename from airflow/providers/google/cloud/hooks/secret_manager.py rename to providers/src/airflow/providers/google/cloud/hooks/secret_manager.py diff --git a/airflow/providers/google/cloud/hooks/spanner.py b/providers/src/airflow/providers/google/cloud/hooks/spanner.py similarity index 100% rename from airflow/providers/google/cloud/hooks/spanner.py rename to providers/src/airflow/providers/google/cloud/hooks/spanner.py diff --git a/airflow/providers/google/cloud/hooks/speech_to_text.py b/providers/src/airflow/providers/google/cloud/hooks/speech_to_text.py similarity index 100% rename from airflow/providers/google/cloud/hooks/speech_to_text.py rename to providers/src/airflow/providers/google/cloud/hooks/speech_to_text.py diff --git a/airflow/providers/google/cloud/hooks/stackdriver.py b/providers/src/airflow/providers/google/cloud/hooks/stackdriver.py similarity index 100% rename from airflow/providers/google/cloud/hooks/stackdriver.py rename to providers/src/airflow/providers/google/cloud/hooks/stackdriver.py diff --git a/airflow/providers/google/cloud/hooks/tasks.py b/providers/src/airflow/providers/google/cloud/hooks/tasks.py similarity index 100% rename from airflow/providers/google/cloud/hooks/tasks.py rename to providers/src/airflow/providers/google/cloud/hooks/tasks.py diff --git a/airflow/providers/google/cloud/hooks/text_to_speech.py b/providers/src/airflow/providers/google/cloud/hooks/text_to_speech.py similarity index 100% rename from airflow/providers/google/cloud/hooks/text_to_speech.py rename to providers/src/airflow/providers/google/cloud/hooks/text_to_speech.py diff --git a/airflow/providers/google/cloud/hooks/translate.py b/providers/src/airflow/providers/google/cloud/hooks/translate.py similarity index 100% rename from airflow/providers/google/cloud/hooks/translate.py rename to providers/src/airflow/providers/google/cloud/hooks/translate.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/__init__.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/__init__.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/__init__.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/__init__.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/dataset.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/dataset.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/dataset.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/dataset.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/model_service.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/model_service.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/model_service.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/model_service.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py rename to providers/src/airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py diff --git a/airflow/providers/google/cloud/hooks/video_intelligence.py b/providers/src/airflow/providers/google/cloud/hooks/video_intelligence.py similarity index 100% rename from airflow/providers/google/cloud/hooks/video_intelligence.py rename to providers/src/airflow/providers/google/cloud/hooks/video_intelligence.py diff --git a/airflow/providers/google/cloud/hooks/vision.py b/providers/src/airflow/providers/google/cloud/hooks/vision.py similarity index 100% rename from airflow/providers/google/cloud/hooks/vision.py rename to providers/src/airflow/providers/google/cloud/hooks/vision.py diff --git a/airflow/providers/google/cloud/hooks/workflows.py b/providers/src/airflow/providers/google/cloud/hooks/workflows.py similarity index 100% rename from airflow/providers/google/cloud/hooks/workflows.py rename to providers/src/airflow/providers/google/cloud/hooks/workflows.py diff --git a/airflow/providers/google/cloud/links/__init__.py b/providers/src/airflow/providers/google/cloud/links/__init__.py similarity index 100% rename from airflow/providers/google/cloud/links/__init__.py rename to providers/src/airflow/providers/google/cloud/links/__init__.py diff --git a/airflow/providers/google/cloud/links/automl.py b/providers/src/airflow/providers/google/cloud/links/automl.py similarity index 100% rename from airflow/providers/google/cloud/links/automl.py rename to providers/src/airflow/providers/google/cloud/links/automl.py diff --git a/airflow/providers/google/cloud/links/base.py b/providers/src/airflow/providers/google/cloud/links/base.py similarity index 100% rename from airflow/providers/google/cloud/links/base.py rename to providers/src/airflow/providers/google/cloud/links/base.py diff --git a/airflow/providers/google/cloud/links/bigquery.py b/providers/src/airflow/providers/google/cloud/links/bigquery.py similarity index 100% rename from airflow/providers/google/cloud/links/bigquery.py rename to providers/src/airflow/providers/google/cloud/links/bigquery.py diff --git a/airflow/providers/google/cloud/links/bigquery_dts.py b/providers/src/airflow/providers/google/cloud/links/bigquery_dts.py similarity index 100% rename from airflow/providers/google/cloud/links/bigquery_dts.py rename to providers/src/airflow/providers/google/cloud/links/bigquery_dts.py diff --git a/airflow/providers/google/cloud/links/bigtable.py b/providers/src/airflow/providers/google/cloud/links/bigtable.py similarity index 100% rename from airflow/providers/google/cloud/links/bigtable.py rename to providers/src/airflow/providers/google/cloud/links/bigtable.py diff --git a/airflow/providers/google/cloud/links/cloud_build.py b/providers/src/airflow/providers/google/cloud/links/cloud_build.py similarity index 100% rename from airflow/providers/google/cloud/links/cloud_build.py rename to providers/src/airflow/providers/google/cloud/links/cloud_build.py diff --git a/airflow/providers/google/cloud/links/cloud_functions.py b/providers/src/airflow/providers/google/cloud/links/cloud_functions.py similarity index 100% rename from airflow/providers/google/cloud/links/cloud_functions.py rename to providers/src/airflow/providers/google/cloud/links/cloud_functions.py diff --git a/airflow/providers/google/cloud/links/cloud_memorystore.py b/providers/src/airflow/providers/google/cloud/links/cloud_memorystore.py similarity index 100% rename from airflow/providers/google/cloud/links/cloud_memorystore.py rename to providers/src/airflow/providers/google/cloud/links/cloud_memorystore.py diff --git a/airflow/providers/google/cloud/links/cloud_sql.py b/providers/src/airflow/providers/google/cloud/links/cloud_sql.py similarity index 100% rename from airflow/providers/google/cloud/links/cloud_sql.py rename to providers/src/airflow/providers/google/cloud/links/cloud_sql.py diff --git a/airflow/providers/google/cloud/links/cloud_storage_transfer.py b/providers/src/airflow/providers/google/cloud/links/cloud_storage_transfer.py similarity index 100% rename from airflow/providers/google/cloud/links/cloud_storage_transfer.py rename to providers/src/airflow/providers/google/cloud/links/cloud_storage_transfer.py diff --git a/airflow/providers/google/cloud/links/cloud_tasks.py b/providers/src/airflow/providers/google/cloud/links/cloud_tasks.py similarity index 100% rename from airflow/providers/google/cloud/links/cloud_tasks.py rename to providers/src/airflow/providers/google/cloud/links/cloud_tasks.py diff --git a/airflow/providers/google/cloud/links/compute.py b/providers/src/airflow/providers/google/cloud/links/compute.py similarity index 100% rename from airflow/providers/google/cloud/links/compute.py rename to providers/src/airflow/providers/google/cloud/links/compute.py diff --git a/airflow/providers/google/cloud/links/data_loss_prevention.py b/providers/src/airflow/providers/google/cloud/links/data_loss_prevention.py similarity index 100% rename from airflow/providers/google/cloud/links/data_loss_prevention.py rename to providers/src/airflow/providers/google/cloud/links/data_loss_prevention.py diff --git a/airflow/providers/google/cloud/links/datacatalog.py b/providers/src/airflow/providers/google/cloud/links/datacatalog.py similarity index 100% rename from airflow/providers/google/cloud/links/datacatalog.py rename to providers/src/airflow/providers/google/cloud/links/datacatalog.py diff --git a/airflow/providers/google/cloud/links/dataflow.py b/providers/src/airflow/providers/google/cloud/links/dataflow.py similarity index 100% rename from airflow/providers/google/cloud/links/dataflow.py rename to providers/src/airflow/providers/google/cloud/links/dataflow.py diff --git a/airflow/providers/google/cloud/links/dataform.py b/providers/src/airflow/providers/google/cloud/links/dataform.py similarity index 100% rename from airflow/providers/google/cloud/links/dataform.py rename to providers/src/airflow/providers/google/cloud/links/dataform.py diff --git a/airflow/providers/google/cloud/links/datafusion.py b/providers/src/airflow/providers/google/cloud/links/datafusion.py similarity index 100% rename from airflow/providers/google/cloud/links/datafusion.py rename to providers/src/airflow/providers/google/cloud/links/datafusion.py diff --git a/airflow/providers/google/cloud/links/dataplex.py b/providers/src/airflow/providers/google/cloud/links/dataplex.py similarity index 100% rename from airflow/providers/google/cloud/links/dataplex.py rename to providers/src/airflow/providers/google/cloud/links/dataplex.py diff --git a/airflow/providers/google/cloud/links/dataprep.py b/providers/src/airflow/providers/google/cloud/links/dataprep.py similarity index 100% rename from airflow/providers/google/cloud/links/dataprep.py rename to providers/src/airflow/providers/google/cloud/links/dataprep.py diff --git a/airflow/providers/google/cloud/links/dataproc.py b/providers/src/airflow/providers/google/cloud/links/dataproc.py similarity index 100% rename from airflow/providers/google/cloud/links/dataproc.py rename to providers/src/airflow/providers/google/cloud/links/dataproc.py diff --git a/airflow/providers/google/cloud/links/datastore.py b/providers/src/airflow/providers/google/cloud/links/datastore.py similarity index 100% rename from airflow/providers/google/cloud/links/datastore.py rename to providers/src/airflow/providers/google/cloud/links/datastore.py diff --git a/airflow/providers/google/cloud/links/kubernetes_engine.py b/providers/src/airflow/providers/google/cloud/links/kubernetes_engine.py similarity index 100% rename from airflow/providers/google/cloud/links/kubernetes_engine.py rename to providers/src/airflow/providers/google/cloud/links/kubernetes_engine.py diff --git a/airflow/providers/google/cloud/links/life_sciences.py b/providers/src/airflow/providers/google/cloud/links/life_sciences.py similarity index 100% rename from airflow/providers/google/cloud/links/life_sciences.py rename to providers/src/airflow/providers/google/cloud/links/life_sciences.py diff --git a/airflow/providers/google/cloud/links/mlengine.py b/providers/src/airflow/providers/google/cloud/links/mlengine.py similarity index 100% rename from airflow/providers/google/cloud/links/mlengine.py rename to providers/src/airflow/providers/google/cloud/links/mlengine.py diff --git a/airflow/providers/google/cloud/links/pubsub.py b/providers/src/airflow/providers/google/cloud/links/pubsub.py similarity index 100% rename from airflow/providers/google/cloud/links/pubsub.py rename to providers/src/airflow/providers/google/cloud/links/pubsub.py diff --git a/airflow/providers/google/cloud/links/spanner.py b/providers/src/airflow/providers/google/cloud/links/spanner.py similarity index 100% rename from airflow/providers/google/cloud/links/spanner.py rename to providers/src/airflow/providers/google/cloud/links/spanner.py diff --git a/airflow/providers/google/cloud/links/stackdriver.py b/providers/src/airflow/providers/google/cloud/links/stackdriver.py similarity index 100% rename from airflow/providers/google/cloud/links/stackdriver.py rename to providers/src/airflow/providers/google/cloud/links/stackdriver.py diff --git a/airflow/providers/google/cloud/links/translate.py b/providers/src/airflow/providers/google/cloud/links/translate.py similarity index 100% rename from airflow/providers/google/cloud/links/translate.py rename to providers/src/airflow/providers/google/cloud/links/translate.py diff --git a/airflow/providers/google/cloud/links/vertex_ai.py b/providers/src/airflow/providers/google/cloud/links/vertex_ai.py similarity index 100% rename from airflow/providers/google/cloud/links/vertex_ai.py rename to providers/src/airflow/providers/google/cloud/links/vertex_ai.py diff --git a/airflow/providers/google/cloud/links/workflows.py b/providers/src/airflow/providers/google/cloud/links/workflows.py similarity index 100% rename from airflow/providers/google/cloud/links/workflows.py rename to providers/src/airflow/providers/google/cloud/links/workflows.py diff --git a/airflow/providers/google/cloud/log/__init__.py b/providers/src/airflow/providers/google/cloud/log/__init__.py similarity index 100% rename from airflow/providers/google/cloud/log/__init__.py rename to providers/src/airflow/providers/google/cloud/log/__init__.py diff --git a/airflow/providers/google/cloud/log/gcs_task_handler.py b/providers/src/airflow/providers/google/cloud/log/gcs_task_handler.py similarity index 100% rename from airflow/providers/google/cloud/log/gcs_task_handler.py rename to providers/src/airflow/providers/google/cloud/log/gcs_task_handler.py diff --git a/airflow/providers/google/cloud/log/stackdriver_task_handler.py b/providers/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py similarity index 100% rename from airflow/providers/google/cloud/log/stackdriver_task_handler.py rename to providers/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py diff --git a/airflow/providers/google/cloud/openlineage/BigQueryErrorRunFacet.json b/providers/src/airflow/providers/google/cloud/openlineage/BigQueryErrorRunFacet.json similarity index 100% rename from airflow/providers/google/cloud/openlineage/BigQueryErrorRunFacet.json rename to providers/src/airflow/providers/google/cloud/openlineage/BigQueryErrorRunFacet.json diff --git a/airflow/providers/google/cloud/openlineage/BigQueryJobRunFacet.json b/providers/src/airflow/providers/google/cloud/openlineage/BigQueryJobRunFacet.json similarity index 100% rename from airflow/providers/google/cloud/openlineage/BigQueryJobRunFacet.json rename to providers/src/airflow/providers/google/cloud/openlineage/BigQueryJobRunFacet.json diff --git a/airflow/providers/google/cloud/openlineage/__init__.py b/providers/src/airflow/providers/google/cloud/openlineage/__init__.py similarity index 100% rename from airflow/providers/google/cloud/openlineage/__init__.py rename to providers/src/airflow/providers/google/cloud/openlineage/__init__.py diff --git a/airflow/providers/google/cloud/openlineage/mixins.py b/providers/src/airflow/providers/google/cloud/openlineage/mixins.py similarity index 100% rename from airflow/providers/google/cloud/openlineage/mixins.py rename to providers/src/airflow/providers/google/cloud/openlineage/mixins.py diff --git a/airflow/providers/google/cloud/openlineage/utils.py b/providers/src/airflow/providers/google/cloud/openlineage/utils.py similarity index 100% rename from airflow/providers/google/cloud/openlineage/utils.py rename to providers/src/airflow/providers/google/cloud/openlineage/utils.py diff --git a/airflow/providers/google/cloud/operators/__init__.py b/providers/src/airflow/providers/google/cloud/operators/__init__.py similarity index 100% rename from airflow/providers/google/cloud/operators/__init__.py rename to providers/src/airflow/providers/google/cloud/operators/__init__.py diff --git a/airflow/providers/google/cloud/operators/automl.py b/providers/src/airflow/providers/google/cloud/operators/automl.py similarity index 100% rename from airflow/providers/google/cloud/operators/automl.py rename to providers/src/airflow/providers/google/cloud/operators/automl.py diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/providers/src/airflow/providers/google/cloud/operators/bigquery.py similarity index 99% rename from airflow/providers/google/cloud/operators/bigquery.py rename to providers/src/airflow/providers/google/cloud/operators/bigquery.py index 1637f51d8f89e..876ff8d51f162 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/providers/src/airflow/providers/google/cloud/operators/bigquery.py @@ -35,7 +35,7 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException from airflow.models import BaseOperator, BaseOperatorLink from airflow.models.xcom import XCom -from airflow.providers.common.sql.operators.sql import ( +from airflow.providers.common.sql.operators.sql import ( # type: ignore[attr-defined] # for _parse_boolean SQLCheckOperator, SQLColumnCheckOperator, SQLIntervalCheckOperator, diff --git a/airflow/providers/google/cloud/operators/bigquery_dts.py b/providers/src/airflow/providers/google/cloud/operators/bigquery_dts.py similarity index 100% rename from airflow/providers/google/cloud/operators/bigquery_dts.py rename to providers/src/airflow/providers/google/cloud/operators/bigquery_dts.py diff --git a/airflow/providers/google/cloud/operators/bigtable.py b/providers/src/airflow/providers/google/cloud/operators/bigtable.py similarity index 100% rename from airflow/providers/google/cloud/operators/bigtable.py rename to providers/src/airflow/providers/google/cloud/operators/bigtable.py diff --git a/airflow/providers/google/cloud/operators/cloud_base.py b/providers/src/airflow/providers/google/cloud/operators/cloud_base.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_base.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_base.py diff --git a/airflow/providers/google/cloud/operators/cloud_batch.py b/providers/src/airflow/providers/google/cloud/operators/cloud_batch.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_batch.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_batch.py diff --git a/airflow/providers/google/cloud/operators/cloud_build.py b/providers/src/airflow/providers/google/cloud/operators/cloud_build.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_build.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_build.py diff --git a/airflow/providers/google/cloud/operators/cloud_composer.py b/providers/src/airflow/providers/google/cloud/operators/cloud_composer.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_composer.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_composer.py diff --git a/airflow/providers/google/cloud/operators/cloud_memorystore.py b/providers/src/airflow/providers/google/cloud/operators/cloud_memorystore.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_memorystore.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_memorystore.py diff --git a/airflow/providers/google/cloud/operators/cloud_run.py b/providers/src/airflow/providers/google/cloud/operators/cloud_run.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_run.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_run.py diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/providers/src/airflow/providers/google/cloud/operators/cloud_sql.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_sql.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_sql.py diff --git a/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py b/providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py similarity index 100% rename from airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py rename to providers/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py diff --git a/airflow/providers/google/cloud/operators/compute.py b/providers/src/airflow/providers/google/cloud/operators/compute.py similarity index 100% rename from airflow/providers/google/cloud/operators/compute.py rename to providers/src/airflow/providers/google/cloud/operators/compute.py diff --git a/airflow/providers/google/cloud/operators/datacatalog.py b/providers/src/airflow/providers/google/cloud/operators/datacatalog.py similarity index 100% rename from airflow/providers/google/cloud/operators/datacatalog.py rename to providers/src/airflow/providers/google/cloud/operators/datacatalog.py diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/providers/src/airflow/providers/google/cloud/operators/dataflow.py similarity index 100% rename from airflow/providers/google/cloud/operators/dataflow.py rename to providers/src/airflow/providers/google/cloud/operators/dataflow.py diff --git a/airflow/providers/google/cloud/operators/dataform.py b/providers/src/airflow/providers/google/cloud/operators/dataform.py similarity index 100% rename from airflow/providers/google/cloud/operators/dataform.py rename to providers/src/airflow/providers/google/cloud/operators/dataform.py diff --git a/airflow/providers/google/cloud/operators/datafusion.py b/providers/src/airflow/providers/google/cloud/operators/datafusion.py similarity index 100% rename from airflow/providers/google/cloud/operators/datafusion.py rename to providers/src/airflow/providers/google/cloud/operators/datafusion.py diff --git a/airflow/providers/google/cloud/operators/datapipeline.py b/providers/src/airflow/providers/google/cloud/operators/datapipeline.py similarity index 100% rename from airflow/providers/google/cloud/operators/datapipeline.py rename to providers/src/airflow/providers/google/cloud/operators/datapipeline.py diff --git a/airflow/providers/google/cloud/operators/dataplex.py b/providers/src/airflow/providers/google/cloud/operators/dataplex.py similarity index 100% rename from airflow/providers/google/cloud/operators/dataplex.py rename to providers/src/airflow/providers/google/cloud/operators/dataplex.py diff --git a/airflow/providers/google/cloud/operators/dataprep.py b/providers/src/airflow/providers/google/cloud/operators/dataprep.py similarity index 100% rename from airflow/providers/google/cloud/operators/dataprep.py rename to providers/src/airflow/providers/google/cloud/operators/dataprep.py diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/providers/src/airflow/providers/google/cloud/operators/dataproc.py similarity index 100% rename from airflow/providers/google/cloud/operators/dataproc.py rename to providers/src/airflow/providers/google/cloud/operators/dataproc.py diff --git a/airflow/providers/google/cloud/operators/dataproc_metastore.py b/providers/src/airflow/providers/google/cloud/operators/dataproc_metastore.py similarity index 100% rename from airflow/providers/google/cloud/operators/dataproc_metastore.py rename to providers/src/airflow/providers/google/cloud/operators/dataproc_metastore.py diff --git a/airflow/providers/google/cloud/operators/datastore.py b/providers/src/airflow/providers/google/cloud/operators/datastore.py similarity index 100% rename from airflow/providers/google/cloud/operators/datastore.py rename to providers/src/airflow/providers/google/cloud/operators/datastore.py diff --git a/airflow/providers/google/cloud/operators/dlp.py b/providers/src/airflow/providers/google/cloud/operators/dlp.py similarity index 100% rename from airflow/providers/google/cloud/operators/dlp.py rename to providers/src/airflow/providers/google/cloud/operators/dlp.py diff --git a/airflow/providers/google/cloud/operators/functions.py b/providers/src/airflow/providers/google/cloud/operators/functions.py similarity index 100% rename from airflow/providers/google/cloud/operators/functions.py rename to providers/src/airflow/providers/google/cloud/operators/functions.py diff --git a/airflow/providers/google/cloud/operators/gcs.py b/providers/src/airflow/providers/google/cloud/operators/gcs.py similarity index 100% rename from airflow/providers/google/cloud/operators/gcs.py rename to providers/src/airflow/providers/google/cloud/operators/gcs.py diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/providers/src/airflow/providers/google/cloud/operators/kubernetes_engine.py similarity index 100% rename from airflow/providers/google/cloud/operators/kubernetes_engine.py rename to providers/src/airflow/providers/google/cloud/operators/kubernetes_engine.py diff --git a/airflow/providers/google/cloud/operators/life_sciences.py b/providers/src/airflow/providers/google/cloud/operators/life_sciences.py similarity index 100% rename from airflow/providers/google/cloud/operators/life_sciences.py rename to providers/src/airflow/providers/google/cloud/operators/life_sciences.py diff --git a/airflow/providers/google/cloud/operators/looker.py b/providers/src/airflow/providers/google/cloud/operators/looker.py similarity index 100% rename from airflow/providers/google/cloud/operators/looker.py rename to providers/src/airflow/providers/google/cloud/operators/looker.py diff --git a/airflow/providers/google/cloud/operators/mlengine.py b/providers/src/airflow/providers/google/cloud/operators/mlengine.py similarity index 100% rename from airflow/providers/google/cloud/operators/mlengine.py rename to providers/src/airflow/providers/google/cloud/operators/mlengine.py diff --git a/airflow/providers/google/cloud/operators/natural_language.py b/providers/src/airflow/providers/google/cloud/operators/natural_language.py similarity index 100% rename from airflow/providers/google/cloud/operators/natural_language.py rename to providers/src/airflow/providers/google/cloud/operators/natural_language.py diff --git a/airflow/providers/google/cloud/operators/pubsub.py b/providers/src/airflow/providers/google/cloud/operators/pubsub.py similarity index 100% rename from airflow/providers/google/cloud/operators/pubsub.py rename to providers/src/airflow/providers/google/cloud/operators/pubsub.py diff --git a/airflow/providers/google/cloud/operators/spanner.py b/providers/src/airflow/providers/google/cloud/operators/spanner.py similarity index 100% rename from airflow/providers/google/cloud/operators/spanner.py rename to providers/src/airflow/providers/google/cloud/operators/spanner.py diff --git a/airflow/providers/google/cloud/operators/speech_to_text.py b/providers/src/airflow/providers/google/cloud/operators/speech_to_text.py similarity index 100% rename from airflow/providers/google/cloud/operators/speech_to_text.py rename to providers/src/airflow/providers/google/cloud/operators/speech_to_text.py diff --git a/airflow/providers/google/cloud/operators/stackdriver.py b/providers/src/airflow/providers/google/cloud/operators/stackdriver.py similarity index 100% rename from airflow/providers/google/cloud/operators/stackdriver.py rename to providers/src/airflow/providers/google/cloud/operators/stackdriver.py diff --git a/airflow/providers/google/cloud/operators/tasks.py b/providers/src/airflow/providers/google/cloud/operators/tasks.py similarity index 100% rename from airflow/providers/google/cloud/operators/tasks.py rename to providers/src/airflow/providers/google/cloud/operators/tasks.py diff --git a/airflow/providers/google/cloud/operators/text_to_speech.py b/providers/src/airflow/providers/google/cloud/operators/text_to_speech.py similarity index 100% rename from airflow/providers/google/cloud/operators/text_to_speech.py rename to providers/src/airflow/providers/google/cloud/operators/text_to_speech.py diff --git a/airflow/providers/google/cloud/operators/translate.py b/providers/src/airflow/providers/google/cloud/operators/translate.py similarity index 100% rename from airflow/providers/google/cloud/operators/translate.py rename to providers/src/airflow/providers/google/cloud/operators/translate.py diff --git a/airflow/providers/google/cloud/operators/translate_speech.py b/providers/src/airflow/providers/google/cloud/operators/translate_speech.py similarity index 100% rename from airflow/providers/google/cloud/operators/translate_speech.py rename to providers/src/airflow/providers/google/cloud/operators/translate_speech.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/__init__.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/__init__.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/__init__.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/__init__.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/custom_job.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/dataset.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/dataset.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/dataset.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/dataset.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/generative_model.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/generative_model.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/generative_model.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/generative_model.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/model_service.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/model_service.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/model_service.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/model_service.py diff --git a/airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py similarity index 100% rename from airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py rename to providers/src/airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py diff --git a/airflow/providers/google/cloud/operators/video_intelligence.py b/providers/src/airflow/providers/google/cloud/operators/video_intelligence.py similarity index 100% rename from airflow/providers/google/cloud/operators/video_intelligence.py rename to providers/src/airflow/providers/google/cloud/operators/video_intelligence.py diff --git a/airflow/providers/google/cloud/operators/vision.py b/providers/src/airflow/providers/google/cloud/operators/vision.py similarity index 100% rename from airflow/providers/google/cloud/operators/vision.py rename to providers/src/airflow/providers/google/cloud/operators/vision.py diff --git a/airflow/providers/google/cloud/operators/workflows.py b/providers/src/airflow/providers/google/cloud/operators/workflows.py similarity index 100% rename from airflow/providers/google/cloud/operators/workflows.py rename to providers/src/airflow/providers/google/cloud/operators/workflows.py diff --git a/airflow/providers/google/cloud/secrets/__init__.py b/providers/src/airflow/providers/google/cloud/secrets/__init__.py similarity index 100% rename from airflow/providers/google/cloud/secrets/__init__.py rename to providers/src/airflow/providers/google/cloud/secrets/__init__.py diff --git a/airflow/providers/google/cloud/secrets/secret_manager.py b/providers/src/airflow/providers/google/cloud/secrets/secret_manager.py similarity index 100% rename from airflow/providers/google/cloud/secrets/secret_manager.py rename to providers/src/airflow/providers/google/cloud/secrets/secret_manager.py diff --git a/airflow/providers/google/cloud/sensors/__init__.py b/providers/src/airflow/providers/google/cloud/sensors/__init__.py similarity index 100% rename from airflow/providers/google/cloud/sensors/__init__.py rename to providers/src/airflow/providers/google/cloud/sensors/__init__.py diff --git a/airflow/providers/google/cloud/sensors/bigquery.py b/providers/src/airflow/providers/google/cloud/sensors/bigquery.py similarity index 100% rename from airflow/providers/google/cloud/sensors/bigquery.py rename to providers/src/airflow/providers/google/cloud/sensors/bigquery.py diff --git a/airflow/providers/google/cloud/sensors/bigquery_dts.py b/providers/src/airflow/providers/google/cloud/sensors/bigquery_dts.py similarity index 100% rename from airflow/providers/google/cloud/sensors/bigquery_dts.py rename to providers/src/airflow/providers/google/cloud/sensors/bigquery_dts.py diff --git a/airflow/providers/google/cloud/sensors/bigtable.py b/providers/src/airflow/providers/google/cloud/sensors/bigtable.py similarity index 100% rename from airflow/providers/google/cloud/sensors/bigtable.py rename to providers/src/airflow/providers/google/cloud/sensors/bigtable.py diff --git a/airflow/providers/google/cloud/sensors/cloud_composer.py b/providers/src/airflow/providers/google/cloud/sensors/cloud_composer.py similarity index 100% rename from airflow/providers/google/cloud/sensors/cloud_composer.py rename to providers/src/airflow/providers/google/cloud/sensors/cloud_composer.py diff --git a/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py b/providers/src/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py similarity index 100% rename from airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py rename to providers/src/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py diff --git a/airflow/providers/google/cloud/sensors/dataflow.py b/providers/src/airflow/providers/google/cloud/sensors/dataflow.py similarity index 100% rename from airflow/providers/google/cloud/sensors/dataflow.py rename to providers/src/airflow/providers/google/cloud/sensors/dataflow.py diff --git a/airflow/providers/google/cloud/sensors/dataform.py b/providers/src/airflow/providers/google/cloud/sensors/dataform.py similarity index 100% rename from airflow/providers/google/cloud/sensors/dataform.py rename to providers/src/airflow/providers/google/cloud/sensors/dataform.py diff --git a/airflow/providers/google/cloud/sensors/datafusion.py b/providers/src/airflow/providers/google/cloud/sensors/datafusion.py similarity index 100% rename from airflow/providers/google/cloud/sensors/datafusion.py rename to providers/src/airflow/providers/google/cloud/sensors/datafusion.py diff --git a/airflow/providers/google/cloud/sensors/dataplex.py b/providers/src/airflow/providers/google/cloud/sensors/dataplex.py similarity index 100% rename from airflow/providers/google/cloud/sensors/dataplex.py rename to providers/src/airflow/providers/google/cloud/sensors/dataplex.py diff --git a/airflow/providers/google/cloud/sensors/dataprep.py b/providers/src/airflow/providers/google/cloud/sensors/dataprep.py similarity index 100% rename from airflow/providers/google/cloud/sensors/dataprep.py rename to providers/src/airflow/providers/google/cloud/sensors/dataprep.py diff --git a/airflow/providers/google/cloud/sensors/dataproc.py b/providers/src/airflow/providers/google/cloud/sensors/dataproc.py similarity index 100% rename from airflow/providers/google/cloud/sensors/dataproc.py rename to providers/src/airflow/providers/google/cloud/sensors/dataproc.py diff --git a/airflow/providers/google/cloud/sensors/dataproc_metastore.py b/providers/src/airflow/providers/google/cloud/sensors/dataproc_metastore.py similarity index 100% rename from airflow/providers/google/cloud/sensors/dataproc_metastore.py rename to providers/src/airflow/providers/google/cloud/sensors/dataproc_metastore.py diff --git a/airflow/providers/google/cloud/sensors/gcs.py b/providers/src/airflow/providers/google/cloud/sensors/gcs.py similarity index 100% rename from airflow/providers/google/cloud/sensors/gcs.py rename to providers/src/airflow/providers/google/cloud/sensors/gcs.py diff --git a/airflow/providers/google/cloud/sensors/looker.py b/providers/src/airflow/providers/google/cloud/sensors/looker.py similarity index 100% rename from airflow/providers/google/cloud/sensors/looker.py rename to providers/src/airflow/providers/google/cloud/sensors/looker.py diff --git a/airflow/providers/google/cloud/sensors/pubsub.py b/providers/src/airflow/providers/google/cloud/sensors/pubsub.py similarity index 100% rename from airflow/providers/google/cloud/sensors/pubsub.py rename to providers/src/airflow/providers/google/cloud/sensors/pubsub.py diff --git a/airflow/providers/google/cloud/sensors/tasks.py b/providers/src/airflow/providers/google/cloud/sensors/tasks.py similarity index 100% rename from airflow/providers/google/cloud/sensors/tasks.py rename to providers/src/airflow/providers/google/cloud/sensors/tasks.py diff --git a/airflow/providers/google/cloud/sensors/workflows.py b/providers/src/airflow/providers/google/cloud/sensors/workflows.py similarity index 100% rename from airflow/providers/google/cloud/sensors/workflows.py rename to providers/src/airflow/providers/google/cloud/sensors/workflows.py diff --git a/airflow/providers/google/cloud/transfers/__init__.py b/providers/src/airflow/providers/google/cloud/transfers/__init__.py similarity index 100% rename from airflow/providers/google/cloud/transfers/__init__.py rename to providers/src/airflow/providers/google/cloud/transfers/__init__.py diff --git a/airflow/providers/google/cloud/transfers/adls_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/adls_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/adls_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/adls_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py b/providers/src/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py similarity index 100% rename from airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py rename to providers/src/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/bigquery_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py b/providers/src/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py similarity index 100% rename from airflow/providers/google/cloud/transfers/bigquery_to_mssql.py rename to providers/src/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py b/providers/src/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py similarity index 100% rename from airflow/providers/google/cloud/transfers/bigquery_to_mysql.py rename to providers/src/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py b/providers/src/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py similarity index 100% rename from airflow/providers/google/cloud/transfers/bigquery_to_postgres.py rename to providers/src/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_sql.py b/providers/src/airflow/providers/google/cloud/transfers/bigquery_to_sql.py similarity index 100% rename from airflow/providers/google/cloud/transfers/bigquery_to_sql.py rename to providers/src/airflow/providers/google/cloud/transfers/bigquery_to_sql.py diff --git a/airflow/providers/google/cloud/transfers/calendar_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/calendar_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/calendar_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/calendar_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/cassandra_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/providers/src/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py similarity index 100% rename from airflow/providers/google/cloud/transfers/gcs_to_bigquery.py rename to providers/src/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/gcs_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/gcs_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/gcs_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/gcs_to_local.py b/providers/src/airflow/providers/google/cloud/transfers/gcs_to_local.py similarity index 100% rename from airflow/providers/google/cloud/transfers/gcs_to_local.py rename to providers/src/airflow/providers/google/cloud/transfers/gcs_to_local.py diff --git a/airflow/providers/google/cloud/transfers/gcs_to_sftp.py b/providers/src/airflow/providers/google/cloud/transfers/gcs_to_sftp.py similarity index 100% rename from airflow/providers/google/cloud/transfers/gcs_to_sftp.py rename to providers/src/airflow/providers/google/cloud/transfers/gcs_to_sftp.py diff --git a/airflow/providers/google/cloud/transfers/gdrive_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/gdrive_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/gdrive_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/gdrive_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/gdrive_to_local.py b/providers/src/airflow/providers/google/cloud/transfers/gdrive_to_local.py similarity index 100% rename from airflow/providers/google/cloud/transfers/gdrive_to_local.py rename to providers/src/airflow/providers/google/cloud/transfers/gdrive_to_local.py diff --git a/airflow/providers/google/cloud/transfers/local_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/local_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/local_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/local_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/mssql_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/mssql_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/mssql_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/mssql_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/mysql_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/mysql_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/mysql_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/mysql_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/oracle_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/oracle_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/oracle_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/oracle_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/postgres_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/postgres_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/postgres_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/postgres_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/presto_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/presto_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/presto_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/presto_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/s3_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/s3_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/s3_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/s3_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/salesforce_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/salesforce_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/salesforce_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/salesforce_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/sftp_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/sftp_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/sftp_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/sftp_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/sheets_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/sheets_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/sheets_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/sheets_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/sql_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/sql_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/sql_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/sql_to_gcs.py diff --git a/airflow/providers/google/cloud/transfers/trino_to_gcs.py b/providers/src/airflow/providers/google/cloud/transfers/trino_to_gcs.py similarity index 100% rename from airflow/providers/google/cloud/transfers/trino_to_gcs.py rename to providers/src/airflow/providers/google/cloud/transfers/trino_to_gcs.py diff --git a/airflow/providers/google/cloud/triggers/__init__.py b/providers/src/airflow/providers/google/cloud/triggers/__init__.py similarity index 100% rename from airflow/providers/google/cloud/triggers/__init__.py rename to providers/src/airflow/providers/google/cloud/triggers/__init__.py diff --git a/airflow/providers/google/cloud/triggers/bigquery.py b/providers/src/airflow/providers/google/cloud/triggers/bigquery.py similarity index 100% rename from airflow/providers/google/cloud/triggers/bigquery.py rename to providers/src/airflow/providers/google/cloud/triggers/bigquery.py diff --git a/airflow/providers/google/cloud/triggers/bigquery_dts.py b/providers/src/airflow/providers/google/cloud/triggers/bigquery_dts.py similarity index 100% rename from airflow/providers/google/cloud/triggers/bigquery_dts.py rename to providers/src/airflow/providers/google/cloud/triggers/bigquery_dts.py diff --git a/airflow/providers/google/cloud/triggers/cloud_batch.py b/providers/src/airflow/providers/google/cloud/triggers/cloud_batch.py similarity index 100% rename from airflow/providers/google/cloud/triggers/cloud_batch.py rename to providers/src/airflow/providers/google/cloud/triggers/cloud_batch.py diff --git a/airflow/providers/google/cloud/triggers/cloud_build.py b/providers/src/airflow/providers/google/cloud/triggers/cloud_build.py similarity index 100% rename from airflow/providers/google/cloud/triggers/cloud_build.py rename to providers/src/airflow/providers/google/cloud/triggers/cloud_build.py diff --git a/airflow/providers/google/cloud/triggers/cloud_composer.py b/providers/src/airflow/providers/google/cloud/triggers/cloud_composer.py similarity index 100% rename from airflow/providers/google/cloud/triggers/cloud_composer.py rename to providers/src/airflow/providers/google/cloud/triggers/cloud_composer.py diff --git a/airflow/providers/google/cloud/triggers/cloud_run.py b/providers/src/airflow/providers/google/cloud/triggers/cloud_run.py similarity index 100% rename from airflow/providers/google/cloud/triggers/cloud_run.py rename to providers/src/airflow/providers/google/cloud/triggers/cloud_run.py diff --git a/airflow/providers/google/cloud/triggers/cloud_sql.py b/providers/src/airflow/providers/google/cloud/triggers/cloud_sql.py similarity index 100% rename from airflow/providers/google/cloud/triggers/cloud_sql.py rename to providers/src/airflow/providers/google/cloud/triggers/cloud_sql.py diff --git a/airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py b/providers/src/airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py similarity index 100% rename from airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py rename to providers/src/airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py diff --git a/airflow/providers/google/cloud/triggers/dataflow.py b/providers/src/airflow/providers/google/cloud/triggers/dataflow.py similarity index 100% rename from airflow/providers/google/cloud/triggers/dataflow.py rename to providers/src/airflow/providers/google/cloud/triggers/dataflow.py diff --git a/airflow/providers/google/cloud/triggers/datafusion.py b/providers/src/airflow/providers/google/cloud/triggers/datafusion.py similarity index 100% rename from airflow/providers/google/cloud/triggers/datafusion.py rename to providers/src/airflow/providers/google/cloud/triggers/datafusion.py diff --git a/airflow/providers/google/cloud/triggers/dataplex.py b/providers/src/airflow/providers/google/cloud/triggers/dataplex.py similarity index 100% rename from airflow/providers/google/cloud/triggers/dataplex.py rename to providers/src/airflow/providers/google/cloud/triggers/dataplex.py diff --git a/airflow/providers/google/cloud/triggers/dataproc.py b/providers/src/airflow/providers/google/cloud/triggers/dataproc.py similarity index 100% rename from airflow/providers/google/cloud/triggers/dataproc.py rename to providers/src/airflow/providers/google/cloud/triggers/dataproc.py diff --git a/airflow/providers/google/cloud/triggers/gcs.py b/providers/src/airflow/providers/google/cloud/triggers/gcs.py similarity index 100% rename from airflow/providers/google/cloud/triggers/gcs.py rename to providers/src/airflow/providers/google/cloud/triggers/gcs.py diff --git a/airflow/providers/google/cloud/triggers/kubernetes_engine.py b/providers/src/airflow/providers/google/cloud/triggers/kubernetes_engine.py similarity index 100% rename from airflow/providers/google/cloud/triggers/kubernetes_engine.py rename to providers/src/airflow/providers/google/cloud/triggers/kubernetes_engine.py diff --git a/airflow/providers/google/cloud/triggers/mlengine.py b/providers/src/airflow/providers/google/cloud/triggers/mlengine.py similarity index 100% rename from airflow/providers/google/cloud/triggers/mlengine.py rename to providers/src/airflow/providers/google/cloud/triggers/mlengine.py diff --git a/airflow/providers/google/cloud/triggers/pubsub.py b/providers/src/airflow/providers/google/cloud/triggers/pubsub.py similarity index 100% rename from airflow/providers/google/cloud/triggers/pubsub.py rename to providers/src/airflow/providers/google/cloud/triggers/pubsub.py diff --git a/airflow/providers/google/cloud/triggers/vertex_ai.py b/providers/src/airflow/providers/google/cloud/triggers/vertex_ai.py similarity index 100% rename from airflow/providers/google/cloud/triggers/vertex_ai.py rename to providers/src/airflow/providers/google/cloud/triggers/vertex_ai.py diff --git a/airflow/providers/ftp/sensors/__init__.py b/providers/src/airflow/providers/google/cloud/utils/__init__.py similarity index 100% rename from airflow/providers/ftp/sensors/__init__.py rename to providers/src/airflow/providers/google/cloud/utils/__init__.py diff --git a/airflow/providers/google/cloud/utils/bigquery.py b/providers/src/airflow/providers/google/cloud/utils/bigquery.py similarity index 100% rename from airflow/providers/google/cloud/utils/bigquery.py rename to providers/src/airflow/providers/google/cloud/utils/bigquery.py diff --git a/airflow/providers/google/cloud/utils/bigquery_get_data.py b/providers/src/airflow/providers/google/cloud/utils/bigquery_get_data.py similarity index 100% rename from airflow/providers/google/cloud/utils/bigquery_get_data.py rename to providers/src/airflow/providers/google/cloud/utils/bigquery_get_data.py diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py b/providers/src/airflow/providers/google/cloud/utils/credentials_provider.py similarity index 100% rename from airflow/providers/google/cloud/utils/credentials_provider.py rename to providers/src/airflow/providers/google/cloud/utils/credentials_provider.py diff --git a/airflow/providers/google/cloud/utils/dataform.py b/providers/src/airflow/providers/google/cloud/utils/dataform.py similarity index 100% rename from airflow/providers/google/cloud/utils/dataform.py rename to providers/src/airflow/providers/google/cloud/utils/dataform.py diff --git a/airflow/providers/google/cloud/utils/datafusion.py b/providers/src/airflow/providers/google/cloud/utils/datafusion.py similarity index 100% rename from airflow/providers/google/cloud/utils/datafusion.py rename to providers/src/airflow/providers/google/cloud/utils/datafusion.py diff --git a/airflow/providers/google/cloud/utils/dataproc.py b/providers/src/airflow/providers/google/cloud/utils/dataproc.py similarity index 100% rename from airflow/providers/google/cloud/utils/dataproc.py rename to providers/src/airflow/providers/google/cloud/utils/dataproc.py diff --git a/airflow/providers/google/cloud/utils/external_token_supplier.py b/providers/src/airflow/providers/google/cloud/utils/external_token_supplier.py similarity index 100% rename from airflow/providers/google/cloud/utils/external_token_supplier.py rename to providers/src/airflow/providers/google/cloud/utils/external_token_supplier.py diff --git a/airflow/providers/google/cloud/utils/field_sanitizer.py b/providers/src/airflow/providers/google/cloud/utils/field_sanitizer.py similarity index 100% rename from airflow/providers/google/cloud/utils/field_sanitizer.py rename to providers/src/airflow/providers/google/cloud/utils/field_sanitizer.py diff --git a/airflow/providers/google/cloud/utils/field_validator.py b/providers/src/airflow/providers/google/cloud/utils/field_validator.py similarity index 100% rename from airflow/providers/google/cloud/utils/field_validator.py rename to providers/src/airflow/providers/google/cloud/utils/field_validator.py diff --git a/airflow/providers/google/cloud/utils/helpers.py b/providers/src/airflow/providers/google/cloud/utils/helpers.py similarity index 100% rename from airflow/providers/google/cloud/utils/helpers.py rename to providers/src/airflow/providers/google/cloud/utils/helpers.py diff --git a/airflow/providers/google/cloud/utils/mlengine_operator_utils.py b/providers/src/airflow/providers/google/cloud/utils/mlengine_operator_utils.py similarity index 100% rename from airflow/providers/google/cloud/utils/mlengine_operator_utils.py rename to providers/src/airflow/providers/google/cloud/utils/mlengine_operator_utils.py diff --git a/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py b/providers/src/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py similarity index 100% rename from airflow/providers/google/cloud/utils/mlengine_prediction_summary.py rename to providers/src/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py diff --git a/airflow/providers/google/common/__init__.py b/providers/src/airflow/providers/google/common/__init__.py similarity index 100% rename from airflow/providers/google/common/__init__.py rename to providers/src/airflow/providers/google/common/__init__.py diff --git a/airflow/providers/google/common/auth_backend/__init__.py b/providers/src/airflow/providers/google/common/auth_backend/__init__.py similarity index 100% rename from airflow/providers/google/common/auth_backend/__init__.py rename to providers/src/airflow/providers/google/common/auth_backend/__init__.py diff --git a/airflow/providers/google/common/auth_backend/google_openid.py b/providers/src/airflow/providers/google/common/auth_backend/google_openid.py similarity index 100% rename from airflow/providers/google/common/auth_backend/google_openid.py rename to providers/src/airflow/providers/google/common/auth_backend/google_openid.py diff --git a/airflow/providers/google/common/consts.py b/providers/src/airflow/providers/google/common/consts.py similarity index 100% rename from airflow/providers/google/common/consts.py rename to providers/src/airflow/providers/google/common/consts.py diff --git a/airflow/providers/google/common/deprecated.py b/providers/src/airflow/providers/google/common/deprecated.py similarity index 100% rename from airflow/providers/google/common/deprecated.py rename to providers/src/airflow/providers/google/common/deprecated.py diff --git a/airflow/providers/google/common/hooks/__init__.py b/providers/src/airflow/providers/google/common/hooks/__init__.py similarity index 100% rename from airflow/providers/google/common/hooks/__init__.py rename to providers/src/airflow/providers/google/common/hooks/__init__.py diff --git a/airflow/providers/google/common/hooks/base_google.py b/providers/src/airflow/providers/google/common/hooks/base_google.py similarity index 100% rename from airflow/providers/google/common/hooks/base_google.py rename to providers/src/airflow/providers/google/common/hooks/base_google.py diff --git a/airflow/providers/google/common/hooks/discovery_api.py b/providers/src/airflow/providers/google/common/hooks/discovery_api.py similarity index 100% rename from airflow/providers/google/common/hooks/discovery_api.py rename to providers/src/airflow/providers/google/common/hooks/discovery_api.py diff --git a/airflow/providers/google/common/links/__init__.py b/providers/src/airflow/providers/google/common/links/__init__.py similarity index 100% rename from airflow/providers/google/common/links/__init__.py rename to providers/src/airflow/providers/google/common/links/__init__.py diff --git a/airflow/providers/google/common/links/storage.py b/providers/src/airflow/providers/google/common/links/storage.py similarity index 100% rename from airflow/providers/google/common/links/storage.py rename to providers/src/airflow/providers/google/common/links/storage.py diff --git a/airflow/providers/google/common/utils/__init__.py b/providers/src/airflow/providers/google/common/utils/__init__.py similarity index 100% rename from airflow/providers/google/common/utils/__init__.py rename to providers/src/airflow/providers/google/common/utils/__init__.py diff --git a/airflow/providers/google/common/utils/id_token_credentials.py b/providers/src/airflow/providers/google/common/utils/id_token_credentials.py similarity index 100% rename from airflow/providers/google/common/utils/id_token_credentials.py rename to providers/src/airflow/providers/google/common/utils/id_token_credentials.py diff --git a/airflow/providers/google/datasets/__init__.py b/providers/src/airflow/providers/google/datasets/__init__.py similarity index 100% rename from airflow/providers/google/datasets/__init__.py rename to providers/src/airflow/providers/google/datasets/__init__.py diff --git a/airflow/providers/google/datasets/bigquery.py b/providers/src/airflow/providers/google/datasets/bigquery.py similarity index 100% rename from airflow/providers/google/datasets/bigquery.py rename to providers/src/airflow/providers/google/datasets/bigquery.py diff --git a/airflow/providers/google/firebase/__init__.py b/providers/src/airflow/providers/google/firebase/__init__.py similarity index 100% rename from airflow/providers/google/firebase/__init__.py rename to providers/src/airflow/providers/google/firebase/__init__.py diff --git a/airflow/providers/google/firebase/hooks/__init__.py b/providers/src/airflow/providers/google/firebase/hooks/__init__.py similarity index 100% rename from airflow/providers/google/firebase/hooks/__init__.py rename to providers/src/airflow/providers/google/firebase/hooks/__init__.py diff --git a/airflow/providers/google/firebase/hooks/firestore.py b/providers/src/airflow/providers/google/firebase/hooks/firestore.py similarity index 100% rename from airflow/providers/google/firebase/hooks/firestore.py rename to providers/src/airflow/providers/google/firebase/hooks/firestore.py diff --git a/airflow/providers/google/firebase/operators/__init__.py b/providers/src/airflow/providers/google/firebase/operators/__init__.py similarity index 100% rename from airflow/providers/google/firebase/operators/__init__.py rename to providers/src/airflow/providers/google/firebase/operators/__init__.py diff --git a/airflow/providers/google/firebase/operators/firestore.py b/providers/src/airflow/providers/google/firebase/operators/firestore.py similarity index 100% rename from airflow/providers/google/firebase/operators/firestore.py rename to providers/src/airflow/providers/google/firebase/operators/firestore.py diff --git a/airflow/providers/google/go_module_utils.py b/providers/src/airflow/providers/google/go_module_utils.py similarity index 100% rename from airflow/providers/google/go_module_utils.py rename to providers/src/airflow/providers/google/go_module_utils.py diff --git a/airflow/providers/google/leveldb/__init__.py b/providers/src/airflow/providers/google/leveldb/__init__.py similarity index 100% rename from airflow/providers/google/leveldb/__init__.py rename to providers/src/airflow/providers/google/leveldb/__init__.py diff --git a/airflow/providers/google/leveldb/hooks/__init__.py b/providers/src/airflow/providers/google/leveldb/hooks/__init__.py similarity index 100% rename from airflow/providers/google/leveldb/hooks/__init__.py rename to providers/src/airflow/providers/google/leveldb/hooks/__init__.py diff --git a/airflow/providers/google/leveldb/hooks/leveldb.py b/providers/src/airflow/providers/google/leveldb/hooks/leveldb.py similarity index 100% rename from airflow/providers/google/leveldb/hooks/leveldb.py rename to providers/src/airflow/providers/google/leveldb/hooks/leveldb.py diff --git a/airflow/providers/google/leveldb/operators/__init__.py b/providers/src/airflow/providers/google/leveldb/operators/__init__.py similarity index 100% rename from airflow/providers/google/leveldb/operators/__init__.py rename to providers/src/airflow/providers/google/leveldb/operators/__init__.py diff --git a/airflow/providers/google/leveldb/operators/leveldb.py b/providers/src/airflow/providers/google/leveldb/operators/leveldb.py similarity index 100% rename from airflow/providers/google/leveldb/operators/leveldb.py rename to providers/src/airflow/providers/google/leveldb/operators/leveldb.py diff --git a/airflow/providers/google/marketing_platform/__init__.py b/providers/src/airflow/providers/google/marketing_platform/__init__.py similarity index 100% rename from airflow/providers/google/marketing_platform/__init__.py rename to providers/src/airflow/providers/google/marketing_platform/__init__.py diff --git a/airflow/providers/google/marketing_platform/example_dags/__init__.py b/providers/src/airflow/providers/google/marketing_platform/example_dags/__init__.py similarity index 100% rename from airflow/providers/google/marketing_platform/example_dags/__init__.py rename to providers/src/airflow/providers/google/marketing_platform/example_dags/__init__.py diff --git a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py b/providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py similarity index 100% rename from airflow/providers/google/marketing_platform/example_dags/example_display_video.py rename to providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py diff --git a/airflow/providers/google/marketing_platform/hooks/__init__.py b/providers/src/airflow/providers/google/marketing_platform/hooks/__init__.py similarity index 100% rename from airflow/providers/google/marketing_platform/hooks/__init__.py rename to providers/src/airflow/providers/google/marketing_platform/hooks/__init__.py diff --git a/airflow/providers/google/marketing_platform/hooks/analytics.py b/providers/src/airflow/providers/google/marketing_platform/hooks/analytics.py similarity index 100% rename from airflow/providers/google/marketing_platform/hooks/analytics.py rename to providers/src/airflow/providers/google/marketing_platform/hooks/analytics.py diff --git a/airflow/providers/google/marketing_platform/hooks/analytics_admin.py b/providers/src/airflow/providers/google/marketing_platform/hooks/analytics_admin.py similarity index 100% rename from airflow/providers/google/marketing_platform/hooks/analytics_admin.py rename to providers/src/airflow/providers/google/marketing_platform/hooks/analytics_admin.py diff --git a/airflow/providers/google/marketing_platform/hooks/campaign_manager.py b/providers/src/airflow/providers/google/marketing_platform/hooks/campaign_manager.py similarity index 100% rename from airflow/providers/google/marketing_platform/hooks/campaign_manager.py rename to providers/src/airflow/providers/google/marketing_platform/hooks/campaign_manager.py diff --git a/airflow/providers/google/marketing_platform/hooks/display_video.py b/providers/src/airflow/providers/google/marketing_platform/hooks/display_video.py similarity index 100% rename from airflow/providers/google/marketing_platform/hooks/display_video.py rename to providers/src/airflow/providers/google/marketing_platform/hooks/display_video.py diff --git a/airflow/providers/google/marketing_platform/hooks/search_ads.py b/providers/src/airflow/providers/google/marketing_platform/hooks/search_ads.py similarity index 100% rename from airflow/providers/google/marketing_platform/hooks/search_ads.py rename to providers/src/airflow/providers/google/marketing_platform/hooks/search_ads.py diff --git a/airflow/providers/google/marketing_platform/links/__init__.py b/providers/src/airflow/providers/google/marketing_platform/links/__init__.py similarity index 100% rename from airflow/providers/google/marketing_platform/links/__init__.py rename to providers/src/airflow/providers/google/marketing_platform/links/__init__.py diff --git a/airflow/providers/google/marketing_platform/links/analytics_admin.py b/providers/src/airflow/providers/google/marketing_platform/links/analytics_admin.py similarity index 100% rename from airflow/providers/google/marketing_platform/links/analytics_admin.py rename to providers/src/airflow/providers/google/marketing_platform/links/analytics_admin.py diff --git a/airflow/providers/google/marketing_platform/operators/__init__.py b/providers/src/airflow/providers/google/marketing_platform/operators/__init__.py similarity index 100% rename from airflow/providers/google/marketing_platform/operators/__init__.py rename to providers/src/airflow/providers/google/marketing_platform/operators/__init__.py diff --git a/airflow/providers/google/marketing_platform/operators/analytics.py b/providers/src/airflow/providers/google/marketing_platform/operators/analytics.py similarity index 100% rename from airflow/providers/google/marketing_platform/operators/analytics.py rename to providers/src/airflow/providers/google/marketing_platform/operators/analytics.py diff --git a/airflow/providers/google/marketing_platform/operators/analytics_admin.py b/providers/src/airflow/providers/google/marketing_platform/operators/analytics_admin.py similarity index 100% rename from airflow/providers/google/marketing_platform/operators/analytics_admin.py rename to providers/src/airflow/providers/google/marketing_platform/operators/analytics_admin.py diff --git a/airflow/providers/google/marketing_platform/operators/campaign_manager.py b/providers/src/airflow/providers/google/marketing_platform/operators/campaign_manager.py similarity index 100% rename from airflow/providers/google/marketing_platform/operators/campaign_manager.py rename to providers/src/airflow/providers/google/marketing_platform/operators/campaign_manager.py diff --git a/airflow/providers/google/marketing_platform/operators/display_video.py b/providers/src/airflow/providers/google/marketing_platform/operators/display_video.py similarity index 100% rename from airflow/providers/google/marketing_platform/operators/display_video.py rename to providers/src/airflow/providers/google/marketing_platform/operators/display_video.py diff --git a/airflow/providers/google/marketing_platform/operators/search_ads.py b/providers/src/airflow/providers/google/marketing_platform/operators/search_ads.py similarity index 100% rename from airflow/providers/google/marketing_platform/operators/search_ads.py rename to providers/src/airflow/providers/google/marketing_platform/operators/search_ads.py diff --git a/airflow/providers/google/marketing_platform/sensors/__init__.py b/providers/src/airflow/providers/google/marketing_platform/sensors/__init__.py similarity index 100% rename from airflow/providers/google/marketing_platform/sensors/__init__.py rename to providers/src/airflow/providers/google/marketing_platform/sensors/__init__.py diff --git a/airflow/providers/google/marketing_platform/sensors/campaign_manager.py b/providers/src/airflow/providers/google/marketing_platform/sensors/campaign_manager.py similarity index 100% rename from airflow/providers/google/marketing_platform/sensors/campaign_manager.py rename to providers/src/airflow/providers/google/marketing_platform/sensors/campaign_manager.py diff --git a/airflow/providers/google/marketing_platform/sensors/display_video.py b/providers/src/airflow/providers/google/marketing_platform/sensors/display_video.py similarity index 100% rename from airflow/providers/google/marketing_platform/sensors/display_video.py rename to providers/src/airflow/providers/google/marketing_platform/sensors/display_video.py diff --git a/airflow/providers/google/provider.yaml b/providers/src/airflow/providers/google/provider.yaml similarity index 100% rename from airflow/providers/google/provider.yaml rename to providers/src/airflow/providers/google/provider.yaml diff --git a/airflow/providers/google/cloud/utils/__init__.py b/providers/src/airflow/providers/google/suite/__init__.py similarity index 100% rename from airflow/providers/google/cloud/utils/__init__.py rename to providers/src/airflow/providers/google/suite/__init__.py diff --git a/airflow/providers/google/suite/__init__.py b/providers/src/airflow/providers/google/suite/hooks/__init__.py similarity index 100% rename from airflow/providers/google/suite/__init__.py rename to providers/src/airflow/providers/google/suite/hooks/__init__.py diff --git a/airflow/providers/google/suite/hooks/calendar.py b/providers/src/airflow/providers/google/suite/hooks/calendar.py similarity index 100% rename from airflow/providers/google/suite/hooks/calendar.py rename to providers/src/airflow/providers/google/suite/hooks/calendar.py diff --git a/airflow/providers/google/suite/hooks/drive.py b/providers/src/airflow/providers/google/suite/hooks/drive.py similarity index 100% rename from airflow/providers/google/suite/hooks/drive.py rename to providers/src/airflow/providers/google/suite/hooks/drive.py diff --git a/airflow/providers/google/suite/hooks/sheets.py b/providers/src/airflow/providers/google/suite/hooks/sheets.py similarity index 100% rename from airflow/providers/google/suite/hooks/sheets.py rename to providers/src/airflow/providers/google/suite/hooks/sheets.py diff --git a/airflow/providers/google/suite/hooks/__init__.py b/providers/src/airflow/providers/google/suite/operators/__init__.py similarity index 100% rename from airflow/providers/google/suite/hooks/__init__.py rename to providers/src/airflow/providers/google/suite/operators/__init__.py diff --git a/airflow/providers/google/suite/operators/sheets.py b/providers/src/airflow/providers/google/suite/operators/sheets.py similarity index 100% rename from airflow/providers/google/suite/operators/sheets.py rename to providers/src/airflow/providers/google/suite/operators/sheets.py diff --git a/airflow/providers/google/suite/sensors/__init__.py b/providers/src/airflow/providers/google/suite/sensors/__init__.py similarity index 100% rename from airflow/providers/google/suite/sensors/__init__.py rename to providers/src/airflow/providers/google/suite/sensors/__init__.py diff --git a/airflow/providers/google/suite/sensors/drive.py b/providers/src/airflow/providers/google/suite/sensors/drive.py similarity index 100% rename from airflow/providers/google/suite/sensors/drive.py rename to providers/src/airflow/providers/google/suite/sensors/drive.py diff --git a/airflow/providers/google/suite/transfers/__init__.py b/providers/src/airflow/providers/google/suite/transfers/__init__.py similarity index 100% rename from airflow/providers/google/suite/transfers/__init__.py rename to providers/src/airflow/providers/google/suite/transfers/__init__.py diff --git a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py b/providers/src/airflow/providers/google/suite/transfers/gcs_to_gdrive.py similarity index 100% rename from airflow/providers/google/suite/transfers/gcs_to_gdrive.py rename to providers/src/airflow/providers/google/suite/transfers/gcs_to_gdrive.py diff --git a/airflow/providers/google/suite/transfers/gcs_to_sheets.py b/providers/src/airflow/providers/google/suite/transfers/gcs_to_sheets.py similarity index 100% rename from airflow/providers/google/suite/transfers/gcs_to_sheets.py rename to providers/src/airflow/providers/google/suite/transfers/gcs_to_sheets.py diff --git a/airflow/providers/google/suite/transfers/local_to_drive.py b/providers/src/airflow/providers/google/suite/transfers/local_to_drive.py similarity index 100% rename from airflow/providers/google/suite/transfers/local_to_drive.py rename to providers/src/airflow/providers/google/suite/transfers/local_to_drive.py diff --git a/airflow/providers/google/suite/transfers/sql_to_sheets.py b/providers/src/airflow/providers/google/suite/transfers/sql_to_sheets.py similarity index 100% rename from airflow/providers/google/suite/transfers/sql_to_sheets.py rename to providers/src/airflow/providers/google/suite/transfers/sql_to_sheets.py diff --git a/airflow/providers/grpc/.latest-doc-only-change.txt b/providers/src/airflow/providers/grpc/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/grpc/.latest-doc-only-change.txt rename to providers/src/airflow/providers/grpc/.latest-doc-only-change.txt diff --git a/airflow/providers/grpc/CHANGELOG.rst b/providers/src/airflow/providers/grpc/CHANGELOG.rst similarity index 100% rename from airflow/providers/grpc/CHANGELOG.rst rename to providers/src/airflow/providers/grpc/CHANGELOG.rst diff --git a/airflow/providers/grpc/__init__.py b/providers/src/airflow/providers/grpc/__init__.py similarity index 100% rename from airflow/providers/grpc/__init__.py rename to providers/src/airflow/providers/grpc/__init__.py diff --git a/airflow/providers/google/suite/operators/__init__.py b/providers/src/airflow/providers/grpc/hooks/__init__.py similarity index 100% rename from airflow/providers/google/suite/operators/__init__.py rename to providers/src/airflow/providers/grpc/hooks/__init__.py diff --git a/airflow/providers/grpc/hooks/grpc.py b/providers/src/airflow/providers/grpc/hooks/grpc.py similarity index 100% rename from airflow/providers/grpc/hooks/grpc.py rename to providers/src/airflow/providers/grpc/hooks/grpc.py diff --git a/airflow/providers/grpc/hooks/__init__.py b/providers/src/airflow/providers/grpc/operators/__init__.py similarity index 100% rename from airflow/providers/grpc/hooks/__init__.py rename to providers/src/airflow/providers/grpc/operators/__init__.py diff --git a/airflow/providers/grpc/operators/grpc.py b/providers/src/airflow/providers/grpc/operators/grpc.py similarity index 100% rename from airflow/providers/grpc/operators/grpc.py rename to providers/src/airflow/providers/grpc/operators/grpc.py diff --git a/airflow/providers/grpc/provider.yaml b/providers/src/airflow/providers/grpc/provider.yaml similarity index 100% rename from airflow/providers/grpc/provider.yaml rename to providers/src/airflow/providers/grpc/provider.yaml diff --git a/airflow/providers/hashicorp/.latest-doc-only-change.txt b/providers/src/airflow/providers/hashicorp/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/hashicorp/.latest-doc-only-change.txt rename to providers/src/airflow/providers/hashicorp/.latest-doc-only-change.txt diff --git a/airflow/providers/hashicorp/CHANGELOG.rst b/providers/src/airflow/providers/hashicorp/CHANGELOG.rst similarity index 100% rename from airflow/providers/hashicorp/CHANGELOG.rst rename to providers/src/airflow/providers/hashicorp/CHANGELOG.rst diff --git a/airflow/providers/hashicorp/__init__.py b/providers/src/airflow/providers/hashicorp/__init__.py similarity index 100% rename from airflow/providers/hashicorp/__init__.py rename to providers/src/airflow/providers/hashicorp/__init__.py diff --git a/airflow/providers/hashicorp/_internal_client/__init__.py b/providers/src/airflow/providers/hashicorp/_internal_client/__init__.py similarity index 100% rename from airflow/providers/hashicorp/_internal_client/__init__.py rename to providers/src/airflow/providers/hashicorp/_internal_client/__init__.py diff --git a/airflow/providers/hashicorp/_internal_client/vault_client.py b/providers/src/airflow/providers/hashicorp/_internal_client/vault_client.py similarity index 100% rename from airflow/providers/hashicorp/_internal_client/vault_client.py rename to providers/src/airflow/providers/hashicorp/_internal_client/vault_client.py diff --git a/airflow/providers/hashicorp/hooks/__init__.py b/providers/src/airflow/providers/hashicorp/hooks/__init__.py similarity index 100% rename from airflow/providers/hashicorp/hooks/__init__.py rename to providers/src/airflow/providers/hashicorp/hooks/__init__.py diff --git a/airflow/providers/hashicorp/hooks/vault.py b/providers/src/airflow/providers/hashicorp/hooks/vault.py similarity index 100% rename from airflow/providers/hashicorp/hooks/vault.py rename to providers/src/airflow/providers/hashicorp/hooks/vault.py diff --git a/airflow/providers/hashicorp/provider.yaml b/providers/src/airflow/providers/hashicorp/provider.yaml similarity index 100% rename from airflow/providers/hashicorp/provider.yaml rename to providers/src/airflow/providers/hashicorp/provider.yaml diff --git a/airflow/providers/hashicorp/secrets/__init__.py b/providers/src/airflow/providers/hashicorp/secrets/__init__.py similarity index 100% rename from airflow/providers/hashicorp/secrets/__init__.py rename to providers/src/airflow/providers/hashicorp/secrets/__init__.py diff --git a/airflow/providers/hashicorp/secrets/vault.py b/providers/src/airflow/providers/hashicorp/secrets/vault.py similarity index 100% rename from airflow/providers/hashicorp/secrets/vault.py rename to providers/src/airflow/providers/hashicorp/secrets/vault.py diff --git a/airflow/providers/http/.latest-doc-only-change.txt b/providers/src/airflow/providers/http/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/http/.latest-doc-only-change.txt rename to providers/src/airflow/providers/http/.latest-doc-only-change.txt diff --git a/airflow/providers/http/CHANGELOG.rst b/providers/src/airflow/providers/http/CHANGELOG.rst similarity index 100% rename from airflow/providers/http/CHANGELOG.rst rename to providers/src/airflow/providers/http/CHANGELOG.rst diff --git a/airflow/providers/http/__init__.py b/providers/src/airflow/providers/http/__init__.py similarity index 100% rename from airflow/providers/http/__init__.py rename to providers/src/airflow/providers/http/__init__.py diff --git a/airflow/providers/grpc/operators/__init__.py b/providers/src/airflow/providers/http/hooks/__init__.py similarity index 100% rename from airflow/providers/grpc/operators/__init__.py rename to providers/src/airflow/providers/http/hooks/__init__.py diff --git a/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py similarity index 100% rename from airflow/providers/http/hooks/http.py rename to providers/src/airflow/providers/http/hooks/http.py diff --git a/airflow/providers/http/hooks/__init__.py b/providers/src/airflow/providers/http/operators/__init__.py similarity index 100% rename from airflow/providers/http/hooks/__init__.py rename to providers/src/airflow/providers/http/operators/__init__.py diff --git a/airflow/providers/http/operators/http.py b/providers/src/airflow/providers/http/operators/http.py similarity index 100% rename from airflow/providers/http/operators/http.py rename to providers/src/airflow/providers/http/operators/http.py diff --git a/airflow/providers/http/provider.yaml b/providers/src/airflow/providers/http/provider.yaml similarity index 100% rename from airflow/providers/http/provider.yaml rename to providers/src/airflow/providers/http/provider.yaml diff --git a/airflow/providers/http/operators/__init__.py b/providers/src/airflow/providers/http/sensors/__init__.py similarity index 100% rename from airflow/providers/http/operators/__init__.py rename to providers/src/airflow/providers/http/sensors/__init__.py diff --git a/airflow/providers/http/sensors/http.py b/providers/src/airflow/providers/http/sensors/http.py similarity index 100% rename from airflow/providers/http/sensors/http.py rename to providers/src/airflow/providers/http/sensors/http.py diff --git a/airflow/providers/http/sensors/__init__.py b/providers/src/airflow/providers/http/triggers/__init__.py similarity index 100% rename from airflow/providers/http/sensors/__init__.py rename to providers/src/airflow/providers/http/triggers/__init__.py diff --git a/airflow/providers/http/triggers/http.py b/providers/src/airflow/providers/http/triggers/http.py similarity index 100% rename from airflow/providers/http/triggers/http.py rename to providers/src/airflow/providers/http/triggers/http.py diff --git a/airflow/providers/imap/.latest-doc-only-change.txt b/providers/src/airflow/providers/imap/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/imap/.latest-doc-only-change.txt rename to providers/src/airflow/providers/imap/.latest-doc-only-change.txt diff --git a/airflow/providers/imap/CHANGELOG.rst b/providers/src/airflow/providers/imap/CHANGELOG.rst similarity index 100% rename from airflow/providers/imap/CHANGELOG.rst rename to providers/src/airflow/providers/imap/CHANGELOG.rst diff --git a/airflow/providers/imap/__init__.py b/providers/src/airflow/providers/imap/__init__.py similarity index 100% rename from airflow/providers/imap/__init__.py rename to providers/src/airflow/providers/imap/__init__.py diff --git a/airflow/providers/http/triggers/__init__.py b/providers/src/airflow/providers/imap/hooks/__init__.py similarity index 100% rename from airflow/providers/http/triggers/__init__.py rename to providers/src/airflow/providers/imap/hooks/__init__.py diff --git a/airflow/providers/imap/hooks/imap.py b/providers/src/airflow/providers/imap/hooks/imap.py similarity index 100% rename from airflow/providers/imap/hooks/imap.py rename to providers/src/airflow/providers/imap/hooks/imap.py diff --git a/airflow/providers/imap/provider.yaml b/providers/src/airflow/providers/imap/provider.yaml similarity index 100% rename from airflow/providers/imap/provider.yaml rename to providers/src/airflow/providers/imap/provider.yaml diff --git a/airflow/providers/imap/hooks/__init__.py b/providers/src/airflow/providers/imap/sensors/__init__.py similarity index 100% rename from airflow/providers/imap/hooks/__init__.py rename to providers/src/airflow/providers/imap/sensors/__init__.py diff --git a/airflow/providers/imap/sensors/imap_attachment.py b/providers/src/airflow/providers/imap/sensors/imap_attachment.py similarity index 100% rename from airflow/providers/imap/sensors/imap_attachment.py rename to providers/src/airflow/providers/imap/sensors/imap_attachment.py diff --git a/airflow/providers/influxdb/.latest-doc-only-change.txt b/providers/src/airflow/providers/influxdb/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/influxdb/.latest-doc-only-change.txt rename to providers/src/airflow/providers/influxdb/.latest-doc-only-change.txt diff --git a/airflow/providers/influxdb/CHANGELOG.rst b/providers/src/airflow/providers/influxdb/CHANGELOG.rst similarity index 100% rename from airflow/providers/influxdb/CHANGELOG.rst rename to providers/src/airflow/providers/influxdb/CHANGELOG.rst diff --git a/airflow/providers/influxdb/__init__.py b/providers/src/airflow/providers/influxdb/__init__.py similarity index 100% rename from airflow/providers/influxdb/__init__.py rename to providers/src/airflow/providers/influxdb/__init__.py diff --git a/airflow/providers/influxdb/hooks/__init__.py b/providers/src/airflow/providers/influxdb/hooks/__init__.py similarity index 100% rename from airflow/providers/influxdb/hooks/__init__.py rename to providers/src/airflow/providers/influxdb/hooks/__init__.py diff --git a/airflow/providers/influxdb/hooks/influxdb.py b/providers/src/airflow/providers/influxdb/hooks/influxdb.py similarity index 100% rename from airflow/providers/influxdb/hooks/influxdb.py rename to providers/src/airflow/providers/influxdb/hooks/influxdb.py diff --git a/airflow/providers/influxdb/operators/__init__.py b/providers/src/airflow/providers/influxdb/operators/__init__.py similarity index 100% rename from airflow/providers/influxdb/operators/__init__.py rename to providers/src/airflow/providers/influxdb/operators/__init__.py diff --git a/airflow/providers/influxdb/operators/influxdb.py b/providers/src/airflow/providers/influxdb/operators/influxdb.py similarity index 100% rename from airflow/providers/influxdb/operators/influxdb.py rename to providers/src/airflow/providers/influxdb/operators/influxdb.py diff --git a/airflow/providers/influxdb/provider.yaml b/providers/src/airflow/providers/influxdb/provider.yaml similarity index 100% rename from airflow/providers/influxdb/provider.yaml rename to providers/src/airflow/providers/influxdb/provider.yaml diff --git a/airflow/providers/jdbc/.latest-doc-only-change.txt b/providers/src/airflow/providers/jdbc/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/jdbc/.latest-doc-only-change.txt rename to providers/src/airflow/providers/jdbc/.latest-doc-only-change.txt diff --git a/airflow/providers/jdbc/CHANGELOG.rst b/providers/src/airflow/providers/jdbc/CHANGELOG.rst similarity index 100% rename from airflow/providers/jdbc/CHANGELOG.rst rename to providers/src/airflow/providers/jdbc/CHANGELOG.rst diff --git a/airflow/providers/jdbc/__init__.py b/providers/src/airflow/providers/jdbc/__init__.py similarity index 100% rename from airflow/providers/jdbc/__init__.py rename to providers/src/airflow/providers/jdbc/__init__.py diff --git a/airflow/providers/imap/sensors/__init__.py b/providers/src/airflow/providers/jdbc/hooks/__init__.py similarity index 100% rename from airflow/providers/imap/sensors/__init__.py rename to providers/src/airflow/providers/jdbc/hooks/__init__.py diff --git a/airflow/providers/jdbc/hooks/jdbc.py b/providers/src/airflow/providers/jdbc/hooks/jdbc.py similarity index 100% rename from airflow/providers/jdbc/hooks/jdbc.py rename to providers/src/airflow/providers/jdbc/hooks/jdbc.py diff --git a/airflow/providers/jdbc/hooks/__init__.py b/providers/src/airflow/providers/jdbc/operators/__init__.py similarity index 100% rename from airflow/providers/jdbc/hooks/__init__.py rename to providers/src/airflow/providers/jdbc/operators/__init__.py diff --git a/airflow/providers/jdbc/operators/jdbc.py b/providers/src/airflow/providers/jdbc/operators/jdbc.py similarity index 100% rename from airflow/providers/jdbc/operators/jdbc.py rename to providers/src/airflow/providers/jdbc/operators/jdbc.py diff --git a/airflow/providers/jdbc/provider.yaml b/providers/src/airflow/providers/jdbc/provider.yaml similarity index 100% rename from airflow/providers/jdbc/provider.yaml rename to providers/src/airflow/providers/jdbc/provider.yaml diff --git a/airflow/providers/jenkins/.latest-doc-only-change.txt b/providers/src/airflow/providers/jenkins/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/jenkins/.latest-doc-only-change.txt rename to providers/src/airflow/providers/jenkins/.latest-doc-only-change.txt diff --git a/airflow/providers/jenkins/CHANGELOG.rst b/providers/src/airflow/providers/jenkins/CHANGELOG.rst similarity index 100% rename from airflow/providers/jenkins/CHANGELOG.rst rename to providers/src/airflow/providers/jenkins/CHANGELOG.rst diff --git a/airflow/providers/jenkins/__init__.py b/providers/src/airflow/providers/jenkins/__init__.py similarity index 100% rename from airflow/providers/jenkins/__init__.py rename to providers/src/airflow/providers/jenkins/__init__.py diff --git a/airflow/providers/jdbc/operators/__init__.py b/providers/src/airflow/providers/jenkins/hooks/__init__.py similarity index 100% rename from airflow/providers/jdbc/operators/__init__.py rename to providers/src/airflow/providers/jenkins/hooks/__init__.py diff --git a/airflow/providers/jenkins/hooks/jenkins.py b/providers/src/airflow/providers/jenkins/hooks/jenkins.py similarity index 100% rename from airflow/providers/jenkins/hooks/jenkins.py rename to providers/src/airflow/providers/jenkins/hooks/jenkins.py diff --git a/airflow/providers/jenkins/hooks/__init__.py b/providers/src/airflow/providers/jenkins/operators/__init__.py similarity index 100% rename from airflow/providers/jenkins/hooks/__init__.py rename to providers/src/airflow/providers/jenkins/operators/__init__.py diff --git a/airflow/providers/jenkins/operators/jenkins_job_trigger.py b/providers/src/airflow/providers/jenkins/operators/jenkins_job_trigger.py similarity index 100% rename from airflow/providers/jenkins/operators/jenkins_job_trigger.py rename to providers/src/airflow/providers/jenkins/operators/jenkins_job_trigger.py diff --git a/airflow/providers/jenkins/provider.yaml b/providers/src/airflow/providers/jenkins/provider.yaml similarity index 100% rename from airflow/providers/jenkins/provider.yaml rename to providers/src/airflow/providers/jenkins/provider.yaml diff --git a/airflow/providers/jenkins/operators/__init__.py b/providers/src/airflow/providers/jenkins/sensors/__init__.py similarity index 100% rename from airflow/providers/jenkins/operators/__init__.py rename to providers/src/airflow/providers/jenkins/sensors/__init__.py diff --git a/airflow/providers/jenkins/sensors/jenkins.py b/providers/src/airflow/providers/jenkins/sensors/jenkins.py similarity index 100% rename from airflow/providers/jenkins/sensors/jenkins.py rename to providers/src/airflow/providers/jenkins/sensors/jenkins.py diff --git a/airflow/providers/microsoft/azure/.latest-doc-only-change.txt b/providers/src/airflow/providers/microsoft/azure/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/microsoft/azure/.latest-doc-only-change.txt rename to providers/src/airflow/providers/microsoft/azure/.latest-doc-only-change.txt diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/providers/src/airflow/providers/microsoft/azure/CHANGELOG.rst similarity index 100% rename from airflow/providers/microsoft/azure/CHANGELOG.rst rename to providers/src/airflow/providers/microsoft/azure/CHANGELOG.rst diff --git a/airflow/providers/microsoft/azure/__init__.py b/providers/src/airflow/providers/microsoft/azure/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/__init__.py rename to providers/src/airflow/providers/microsoft/azure/__init__.py diff --git a/airflow/providers/microsoft/azure/fs/__init__.py b/providers/src/airflow/providers/microsoft/azure/fs/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/fs/__init__.py rename to providers/src/airflow/providers/microsoft/azure/fs/__init__.py diff --git a/airflow/providers/microsoft/azure/fs/adls.py b/providers/src/airflow/providers/microsoft/azure/fs/adls.py similarity index 100% rename from airflow/providers/microsoft/azure/fs/adls.py rename to providers/src/airflow/providers/microsoft/azure/fs/adls.py diff --git a/airflow/providers/jenkins/sensors/__init__.py b/providers/src/airflow/providers/microsoft/azure/hooks/__init__.py similarity index 100% rename from airflow/providers/jenkins/sensors/__init__.py rename to providers/src/airflow/providers/microsoft/azure/hooks/__init__.py diff --git a/airflow/providers/microsoft/azure/hooks/adx.py b/providers/src/airflow/providers/microsoft/azure/hooks/adx.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/adx.py rename to providers/src/airflow/providers/microsoft/azure/hooks/adx.py diff --git a/airflow/providers/microsoft/azure/hooks/asb.py b/providers/src/airflow/providers/microsoft/azure/hooks/asb.py similarity index 96% rename from airflow/providers/microsoft/azure/hooks/asb.py rename to providers/src/airflow/providers/microsoft/azure/hooks/asb.py index c90833f52fea7..317447d111703 100644 --- a/airflow/providers/microsoft/azure/hooks/asb.py +++ b/providers/src/airflow/providers/microsoft/azure/hooks/asb.py @@ -251,9 +251,11 @@ def send_message(self, queue_name: str, messages: str | list[str], batch_message raise TypeError("Queue name cannot be None.") if not messages: raise ValueError("Messages list cannot be empty.") - with self.get_conn() as service_bus_client, service_bus_client.get_queue_sender( - queue_name=queue_name - ) as sender, sender: + with ( + self.get_conn() as service_bus_client, + service_bus_client.get_queue_sender(queue_name=queue_name) as sender, + sender, + ): if isinstance(messages, str): if not batch_message_flag: msg = ServiceBusMessage(messages) @@ -298,9 +300,11 @@ def receive_message( if queue_name is None: raise TypeError("Queue name cannot be None.") - with self.get_conn() as service_bus_client, service_bus_client.get_queue_receiver( - queue_name=queue_name - ) as receiver, receiver: + with ( + self.get_conn() as service_bus_client, + service_bus_client.get_queue_receiver(queue_name=queue_name) as receiver, + receiver, + ): received_msgs = receiver.receive_messages( max_message_count=max_message_count, max_wait_time=max_wait_time ) @@ -335,9 +339,13 @@ def receive_subscription_message( raise TypeError("Subscription name cannot be None.") if topic_name is None: raise TypeError("Topic name cannot be None.") - with self.get_conn() as service_bus_client, service_bus_client.get_subscription_receiver( - topic_name, subscription_name - ) as subscription_receiver, subscription_receiver: + with ( + self.get_conn() as service_bus_client, + service_bus_client.get_subscription_receiver( + topic_name, subscription_name + ) as subscription_receiver, + subscription_receiver, + ): received_msgs = subscription_receiver.receive_messages( max_message_count=max_message_count, max_wait_time=max_wait_time ) diff --git a/airflow/providers/microsoft/azure/hooks/base_azure.py b/providers/src/airflow/providers/microsoft/azure/hooks/base_azure.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/base_azure.py rename to providers/src/airflow/providers/microsoft/azure/hooks/base_azure.py diff --git a/airflow/providers/microsoft/azure/hooks/batch.py b/providers/src/airflow/providers/microsoft/azure/hooks/batch.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/batch.py rename to providers/src/airflow/providers/microsoft/azure/hooks/batch.py diff --git a/airflow/providers/microsoft/azure/hooks/container_instance.py b/providers/src/airflow/providers/microsoft/azure/hooks/container_instance.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/container_instance.py rename to providers/src/airflow/providers/microsoft/azure/hooks/container_instance.py diff --git a/airflow/providers/microsoft/azure/hooks/container_registry.py b/providers/src/airflow/providers/microsoft/azure/hooks/container_registry.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/container_registry.py rename to providers/src/airflow/providers/microsoft/azure/hooks/container_registry.py diff --git a/airflow/providers/microsoft/azure/hooks/container_volume.py b/providers/src/airflow/providers/microsoft/azure/hooks/container_volume.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/container_volume.py rename to providers/src/airflow/providers/microsoft/azure/hooks/container_volume.py diff --git a/airflow/providers/microsoft/azure/hooks/cosmos.py b/providers/src/airflow/providers/microsoft/azure/hooks/cosmos.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/cosmos.py rename to providers/src/airflow/providers/microsoft/azure/hooks/cosmos.py diff --git a/airflow/providers/microsoft/azure/hooks/data_factory.py b/providers/src/airflow/providers/microsoft/azure/hooks/data_factory.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/data_factory.py rename to providers/src/airflow/providers/microsoft/azure/hooks/data_factory.py diff --git a/airflow/providers/microsoft/azure/hooks/data_lake.py b/providers/src/airflow/providers/microsoft/azure/hooks/data_lake.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/data_lake.py rename to providers/src/airflow/providers/microsoft/azure/hooks/data_lake.py diff --git a/airflow/providers/microsoft/azure/hooks/fileshare.py b/providers/src/airflow/providers/microsoft/azure/hooks/fileshare.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/fileshare.py rename to providers/src/airflow/providers/microsoft/azure/hooks/fileshare.py diff --git a/airflow/providers/microsoft/azure/hooks/msgraph.py b/providers/src/airflow/providers/microsoft/azure/hooks/msgraph.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/msgraph.py rename to providers/src/airflow/providers/microsoft/azure/hooks/msgraph.py diff --git a/airflow/providers/microsoft/azure/hooks/powerbi.py b/providers/src/airflow/providers/microsoft/azure/hooks/powerbi.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/powerbi.py rename to providers/src/airflow/providers/microsoft/azure/hooks/powerbi.py diff --git a/airflow/providers/microsoft/azure/hooks/synapse.py b/providers/src/airflow/providers/microsoft/azure/hooks/synapse.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/synapse.py rename to providers/src/airflow/providers/microsoft/azure/hooks/synapse.py diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/providers/src/airflow/providers/microsoft/azure/hooks/wasb.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/wasb.py rename to providers/src/airflow/providers/microsoft/azure/hooks/wasb.py diff --git a/airflow/providers/microsoft/azure/log/__init__.py b/providers/src/airflow/providers/microsoft/azure/log/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/log/__init__.py rename to providers/src/airflow/providers/microsoft/azure/log/__init__.py diff --git a/airflow/providers/microsoft/azure/log/wasb_task_handler.py b/providers/src/airflow/providers/microsoft/azure/log/wasb_task_handler.py similarity index 100% rename from airflow/providers/microsoft/azure/log/wasb_task_handler.py rename to providers/src/airflow/providers/microsoft/azure/log/wasb_task_handler.py diff --git a/airflow/providers/microsoft/__init__.py b/providers/src/airflow/providers/microsoft/azure/operators/__init__.py similarity index 100% rename from airflow/providers/microsoft/__init__.py rename to providers/src/airflow/providers/microsoft/azure/operators/__init__.py diff --git a/airflow/providers/microsoft/azure/operators/adls.py b/providers/src/airflow/providers/microsoft/azure/operators/adls.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/adls.py rename to providers/src/airflow/providers/microsoft/azure/operators/adls.py diff --git a/airflow/providers/microsoft/azure/operators/adx.py b/providers/src/airflow/providers/microsoft/azure/operators/adx.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/adx.py rename to providers/src/airflow/providers/microsoft/azure/operators/adx.py diff --git a/airflow/providers/microsoft/azure/operators/asb.py b/providers/src/airflow/providers/microsoft/azure/operators/asb.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/asb.py rename to providers/src/airflow/providers/microsoft/azure/operators/asb.py diff --git a/airflow/providers/microsoft/azure/operators/batch.py b/providers/src/airflow/providers/microsoft/azure/operators/batch.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/batch.py rename to providers/src/airflow/providers/microsoft/azure/operators/batch.py diff --git a/airflow/providers/microsoft/azure/operators/container_instances.py b/providers/src/airflow/providers/microsoft/azure/operators/container_instances.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/container_instances.py rename to providers/src/airflow/providers/microsoft/azure/operators/container_instances.py diff --git a/airflow/providers/microsoft/azure/operators/cosmos.py b/providers/src/airflow/providers/microsoft/azure/operators/cosmos.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/cosmos.py rename to providers/src/airflow/providers/microsoft/azure/operators/cosmos.py diff --git a/airflow/providers/microsoft/azure/operators/data_factory.py b/providers/src/airflow/providers/microsoft/azure/operators/data_factory.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/data_factory.py rename to providers/src/airflow/providers/microsoft/azure/operators/data_factory.py diff --git a/airflow/providers/microsoft/azure/operators/msgraph.py b/providers/src/airflow/providers/microsoft/azure/operators/msgraph.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/msgraph.py rename to providers/src/airflow/providers/microsoft/azure/operators/msgraph.py diff --git a/airflow/providers/microsoft/azure/operators/powerbi.py b/providers/src/airflow/providers/microsoft/azure/operators/powerbi.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/powerbi.py rename to providers/src/airflow/providers/microsoft/azure/operators/powerbi.py diff --git a/airflow/providers/microsoft/azure/operators/synapse.py b/providers/src/airflow/providers/microsoft/azure/operators/synapse.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/synapse.py rename to providers/src/airflow/providers/microsoft/azure/operators/synapse.py diff --git a/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py b/providers/src/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/wasb_delete_blob.py rename to providers/src/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py diff --git a/airflow/providers/microsoft/azure/provider.yaml b/providers/src/airflow/providers/microsoft/azure/provider.yaml similarity index 100% rename from airflow/providers/microsoft/azure/provider.yaml rename to providers/src/airflow/providers/microsoft/azure/provider.yaml diff --git a/airflow/providers/microsoft/azure/secrets/__init__.py b/providers/src/airflow/providers/microsoft/azure/secrets/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/secrets/__init__.py rename to providers/src/airflow/providers/microsoft/azure/secrets/__init__.py diff --git a/airflow/providers/microsoft/azure/secrets/key_vault.py b/providers/src/airflow/providers/microsoft/azure/secrets/key_vault.py similarity index 100% rename from airflow/providers/microsoft/azure/secrets/key_vault.py rename to providers/src/airflow/providers/microsoft/azure/secrets/key_vault.py diff --git a/airflow/providers/microsoft/azure/hooks/__init__.py b/providers/src/airflow/providers/microsoft/azure/sensors/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/hooks/__init__.py rename to providers/src/airflow/providers/microsoft/azure/sensors/__init__.py diff --git a/airflow/providers/microsoft/azure/sensors/cosmos.py b/providers/src/airflow/providers/microsoft/azure/sensors/cosmos.py similarity index 100% rename from airflow/providers/microsoft/azure/sensors/cosmos.py rename to providers/src/airflow/providers/microsoft/azure/sensors/cosmos.py diff --git a/airflow/providers/microsoft/azure/sensors/data_factory.py b/providers/src/airflow/providers/microsoft/azure/sensors/data_factory.py similarity index 100% rename from airflow/providers/microsoft/azure/sensors/data_factory.py rename to providers/src/airflow/providers/microsoft/azure/sensors/data_factory.py diff --git a/airflow/providers/microsoft/azure/sensors/msgraph.py b/providers/src/airflow/providers/microsoft/azure/sensors/msgraph.py similarity index 100% rename from airflow/providers/microsoft/azure/sensors/msgraph.py rename to providers/src/airflow/providers/microsoft/azure/sensors/msgraph.py diff --git a/airflow/providers/microsoft/azure/sensors/wasb.py b/providers/src/airflow/providers/microsoft/azure/sensors/wasb.py similarity index 100% rename from airflow/providers/microsoft/azure/sensors/wasb.py rename to providers/src/airflow/providers/microsoft/azure/sensors/wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/__init__.py b/providers/src/airflow/providers/microsoft/azure/transfers/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/transfers/__init__.py rename to providers/src/airflow/providers/microsoft/azure/transfers/__init__.py diff --git a/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py b/providers/src/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py similarity index 100% rename from airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py rename to providers/src/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py diff --git a/airflow/providers/microsoft/azure/transfers/local_to_adls.py b/providers/src/airflow/providers/microsoft/azure/transfers/local_to_adls.py similarity index 100% rename from airflow/providers/microsoft/azure/transfers/local_to_adls.py rename to providers/src/airflow/providers/microsoft/azure/transfers/local_to_adls.py diff --git a/airflow/providers/microsoft/azure/transfers/local_to_wasb.py b/providers/src/airflow/providers/microsoft/azure/transfers/local_to_wasb.py similarity index 100% rename from airflow/providers/microsoft/azure/transfers/local_to_wasb.py rename to providers/src/airflow/providers/microsoft/azure/transfers/local_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py b/providers/src/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py similarity index 100% rename from airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py rename to providers/src/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py diff --git a/airflow/providers/microsoft/azure/transfers/s3_to_wasb.py b/providers/src/airflow/providers/microsoft/azure/transfers/s3_to_wasb.py similarity index 100% rename from airflow/providers/microsoft/azure/transfers/s3_to_wasb.py rename to providers/src/airflow/providers/microsoft/azure/transfers/s3_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/providers/src/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py similarity index 100% rename from airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py rename to providers/src/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py diff --git a/airflow/providers/microsoft/azure/triggers/__init__.py b/providers/src/airflow/providers/microsoft/azure/triggers/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/triggers/__init__.py rename to providers/src/airflow/providers/microsoft/azure/triggers/__init__.py diff --git a/airflow/providers/microsoft/azure/triggers/data_factory.py b/providers/src/airflow/providers/microsoft/azure/triggers/data_factory.py similarity index 100% rename from airflow/providers/microsoft/azure/triggers/data_factory.py rename to providers/src/airflow/providers/microsoft/azure/triggers/data_factory.py diff --git a/airflow/providers/microsoft/azure/triggers/msgraph.py b/providers/src/airflow/providers/microsoft/azure/triggers/msgraph.py similarity index 100% rename from airflow/providers/microsoft/azure/triggers/msgraph.py rename to providers/src/airflow/providers/microsoft/azure/triggers/msgraph.py diff --git a/airflow/providers/microsoft/azure/triggers/powerbi.py b/providers/src/airflow/providers/microsoft/azure/triggers/powerbi.py similarity index 100% rename from airflow/providers/microsoft/azure/triggers/powerbi.py rename to providers/src/airflow/providers/microsoft/azure/triggers/powerbi.py diff --git a/airflow/providers/microsoft/azure/triggers/wasb.py b/providers/src/airflow/providers/microsoft/azure/triggers/wasb.py similarity index 100% rename from airflow/providers/microsoft/azure/triggers/wasb.py rename to providers/src/airflow/providers/microsoft/azure/triggers/wasb.py diff --git a/airflow/providers/microsoft/azure/utils.py b/providers/src/airflow/providers/microsoft/azure/utils.py similarity index 100% rename from airflow/providers/microsoft/azure/utils.py rename to providers/src/airflow/providers/microsoft/azure/utils.py diff --git a/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt b/providers/src/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/microsoft/mssql/.latest-doc-only-change.txt rename to providers/src/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt diff --git a/airflow/providers/microsoft/mssql/CHANGELOG.rst b/providers/src/airflow/providers/microsoft/mssql/CHANGELOG.rst similarity index 100% rename from airflow/providers/microsoft/mssql/CHANGELOG.rst rename to providers/src/airflow/providers/microsoft/mssql/CHANGELOG.rst diff --git a/airflow/providers/microsoft/mssql/__init__.py b/providers/src/airflow/providers/microsoft/mssql/__init__.py similarity index 100% rename from airflow/providers/microsoft/mssql/__init__.py rename to providers/src/airflow/providers/microsoft/mssql/__init__.py diff --git a/airflow/providers/microsoft/azure/operators/__init__.py b/providers/src/airflow/providers/microsoft/mssql/hooks/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/operators/__init__.py rename to providers/src/airflow/providers/microsoft/mssql/hooks/__init__.py diff --git a/airflow/providers/microsoft/mssql/hooks/mssql.py b/providers/src/airflow/providers/microsoft/mssql/hooks/mssql.py similarity index 100% rename from airflow/providers/microsoft/mssql/hooks/mssql.py rename to providers/src/airflow/providers/microsoft/mssql/hooks/mssql.py diff --git a/airflow/providers/microsoft/azure/sensors/__init__.py b/providers/src/airflow/providers/microsoft/mssql/operators/__init__.py similarity index 100% rename from airflow/providers/microsoft/azure/sensors/__init__.py rename to providers/src/airflow/providers/microsoft/mssql/operators/__init__.py diff --git a/airflow/providers/microsoft/mssql/operators/mssql.py b/providers/src/airflow/providers/microsoft/mssql/operators/mssql.py similarity index 100% rename from airflow/providers/microsoft/mssql/operators/mssql.py rename to providers/src/airflow/providers/microsoft/mssql/operators/mssql.py diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/providers/src/airflow/providers/microsoft/mssql/provider.yaml similarity index 100% rename from airflow/providers/microsoft/mssql/provider.yaml rename to providers/src/airflow/providers/microsoft/mssql/provider.yaml diff --git a/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt b/providers/src/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/microsoft/psrp/.latest-doc-only-change.txt rename to providers/src/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt diff --git a/airflow/providers/microsoft/psrp/CHANGELOG.rst b/providers/src/airflow/providers/microsoft/psrp/CHANGELOG.rst similarity index 100% rename from airflow/providers/microsoft/psrp/CHANGELOG.rst rename to providers/src/airflow/providers/microsoft/psrp/CHANGELOG.rst diff --git a/airflow/providers/microsoft/psrp/__init__.py b/providers/src/airflow/providers/microsoft/psrp/__init__.py similarity index 100% rename from airflow/providers/microsoft/psrp/__init__.py rename to providers/src/airflow/providers/microsoft/psrp/__init__.py diff --git a/airflow/providers/microsoft/mssql/hooks/__init__.py b/providers/src/airflow/providers/microsoft/psrp/hooks/__init__.py similarity index 100% rename from airflow/providers/microsoft/mssql/hooks/__init__.py rename to providers/src/airflow/providers/microsoft/psrp/hooks/__init__.py diff --git a/airflow/providers/microsoft/psrp/hooks/psrp.py b/providers/src/airflow/providers/microsoft/psrp/hooks/psrp.py similarity index 100% rename from airflow/providers/microsoft/psrp/hooks/psrp.py rename to providers/src/airflow/providers/microsoft/psrp/hooks/psrp.py diff --git a/airflow/providers/microsoft/mssql/operators/__init__.py b/providers/src/airflow/providers/microsoft/psrp/operators/__init__.py similarity index 100% rename from airflow/providers/microsoft/mssql/operators/__init__.py rename to providers/src/airflow/providers/microsoft/psrp/operators/__init__.py diff --git a/airflow/providers/microsoft/psrp/operators/psrp.py b/providers/src/airflow/providers/microsoft/psrp/operators/psrp.py similarity index 93% rename from airflow/providers/microsoft/psrp/operators/psrp.py rename to providers/src/airflow/providers/microsoft/psrp/operators/psrp.py index dd7da36a9e156..6927159a44ea2 100644 --- a/airflow/providers/microsoft/psrp/operators/psrp.py +++ b/providers/src/airflow/providers/microsoft/psrp/operators/psrp.py @@ -17,8 +17,9 @@ # under the License. from __future__ import annotations +from collections.abc import Sequence from logging import DEBUG -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any from jinja2.nativetypes import NativeEnvironment from pypsrp.serializer import TaggedValue @@ -128,13 +129,16 @@ def __init__( self.psrp_session_init = psrp_session_init def execute(self, context: Context) -> list[Any] | None: - with PsrpHook( - self.conn_id, - logging_level=self.logging_level, - runspace_options=self.runspace_options, - wsman_options=self.wsman_options, - on_output_callback=self.log.info if not self.do_xcom_push else None, - ) as hook, hook.invoke() as ps: + with ( + PsrpHook( + self.conn_id, + logging_level=self.logging_level, + runspace_options=self.runspace_options, + wsman_options=self.wsman_options, + on_output_callback=self.log.info if not self.do_xcom_push else None, + ) as hook, + hook.invoke() as ps, + ): if self.psrp_session_init is not None: ps.add_command(self.psrp_session_init) if self.command: diff --git a/airflow/providers/microsoft/psrp/provider.yaml b/providers/src/airflow/providers/microsoft/psrp/provider.yaml similarity index 100% rename from airflow/providers/microsoft/psrp/provider.yaml rename to providers/src/airflow/providers/microsoft/psrp/provider.yaml diff --git a/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt b/providers/src/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/microsoft/winrm/.latest-doc-only-change.txt rename to providers/src/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt diff --git a/airflow/providers/microsoft/winrm/CHANGELOG.rst b/providers/src/airflow/providers/microsoft/winrm/CHANGELOG.rst similarity index 100% rename from airflow/providers/microsoft/winrm/CHANGELOG.rst rename to providers/src/airflow/providers/microsoft/winrm/CHANGELOG.rst diff --git a/airflow/providers/microsoft/winrm/__init__.py b/providers/src/airflow/providers/microsoft/winrm/__init__.py similarity index 100% rename from airflow/providers/microsoft/winrm/__init__.py rename to providers/src/airflow/providers/microsoft/winrm/__init__.py diff --git a/airflow/providers/microsoft/psrp/hooks/__init__.py b/providers/src/airflow/providers/microsoft/winrm/hooks/__init__.py similarity index 100% rename from airflow/providers/microsoft/psrp/hooks/__init__.py rename to providers/src/airflow/providers/microsoft/winrm/hooks/__init__.py diff --git a/airflow/providers/microsoft/winrm/hooks/winrm.py b/providers/src/airflow/providers/microsoft/winrm/hooks/winrm.py similarity index 100% rename from airflow/providers/microsoft/winrm/hooks/winrm.py rename to providers/src/airflow/providers/microsoft/winrm/hooks/winrm.py diff --git a/airflow/providers/microsoft/psrp/operators/__init__.py b/providers/src/airflow/providers/microsoft/winrm/operators/__init__.py similarity index 100% rename from airflow/providers/microsoft/psrp/operators/__init__.py rename to providers/src/airflow/providers/microsoft/winrm/operators/__init__.py diff --git a/airflow/providers/microsoft/winrm/operators/winrm.py b/providers/src/airflow/providers/microsoft/winrm/operators/winrm.py similarity index 100% rename from airflow/providers/microsoft/winrm/operators/winrm.py rename to providers/src/airflow/providers/microsoft/winrm/operators/winrm.py diff --git a/airflow/providers/microsoft/winrm/provider.yaml b/providers/src/airflow/providers/microsoft/winrm/provider.yaml similarity index 100% rename from airflow/providers/microsoft/winrm/provider.yaml rename to providers/src/airflow/providers/microsoft/winrm/provider.yaml diff --git a/airflow/providers/mongo/.latest-doc-only-change.txt b/providers/src/airflow/providers/mongo/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/mongo/.latest-doc-only-change.txt rename to providers/src/airflow/providers/mongo/.latest-doc-only-change.txt diff --git a/airflow/providers/mongo/CHANGELOG.rst b/providers/src/airflow/providers/mongo/CHANGELOG.rst similarity index 100% rename from airflow/providers/mongo/CHANGELOG.rst rename to providers/src/airflow/providers/mongo/CHANGELOG.rst diff --git a/airflow/providers/mongo/__init__.py b/providers/src/airflow/providers/mongo/__init__.py similarity index 100% rename from airflow/providers/mongo/__init__.py rename to providers/src/airflow/providers/mongo/__init__.py diff --git a/airflow/providers/microsoft/winrm/hooks/__init__.py b/providers/src/airflow/providers/mongo/hooks/__init__.py similarity index 100% rename from airflow/providers/microsoft/winrm/hooks/__init__.py rename to providers/src/airflow/providers/mongo/hooks/__init__.py diff --git a/airflow/providers/mongo/hooks/mongo.py b/providers/src/airflow/providers/mongo/hooks/mongo.py similarity index 100% rename from airflow/providers/mongo/hooks/mongo.py rename to providers/src/airflow/providers/mongo/hooks/mongo.py diff --git a/airflow/providers/mongo/provider.yaml b/providers/src/airflow/providers/mongo/provider.yaml similarity index 100% rename from airflow/providers/mongo/provider.yaml rename to providers/src/airflow/providers/mongo/provider.yaml diff --git a/airflow/providers/microsoft/winrm/operators/__init__.py b/providers/src/airflow/providers/mongo/sensors/__init__.py similarity index 100% rename from airflow/providers/microsoft/winrm/operators/__init__.py rename to providers/src/airflow/providers/mongo/sensors/__init__.py diff --git a/airflow/providers/mongo/sensors/mongo.py b/providers/src/airflow/providers/mongo/sensors/mongo.py similarity index 100% rename from airflow/providers/mongo/sensors/mongo.py rename to providers/src/airflow/providers/mongo/sensors/mongo.py diff --git a/airflow/providers/mysql/.latest-doc-only-change.txt b/providers/src/airflow/providers/mysql/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/mysql/.latest-doc-only-change.txt rename to providers/src/airflow/providers/mysql/.latest-doc-only-change.txt diff --git a/airflow/providers/mysql/CHANGELOG.rst b/providers/src/airflow/providers/mysql/CHANGELOG.rst similarity index 100% rename from airflow/providers/mysql/CHANGELOG.rst rename to providers/src/airflow/providers/mysql/CHANGELOG.rst diff --git a/airflow/providers/mysql/__init__.py b/providers/src/airflow/providers/mysql/__init__.py similarity index 100% rename from airflow/providers/mysql/__init__.py rename to providers/src/airflow/providers/mysql/__init__.py diff --git a/airflow/providers/mysql/assets/__init__.py b/providers/src/airflow/providers/mysql/assets/__init__.py similarity index 100% rename from airflow/providers/mysql/assets/__init__.py rename to providers/src/airflow/providers/mysql/assets/__init__.py diff --git a/airflow/providers/mysql/assets/mysql.py b/providers/src/airflow/providers/mysql/assets/mysql.py similarity index 100% rename from airflow/providers/mysql/assets/mysql.py rename to providers/src/airflow/providers/mysql/assets/mysql.py diff --git a/airflow/providers/mongo/hooks/__init__.py b/providers/src/airflow/providers/mysql/hooks/__init__.py similarity index 100% rename from airflow/providers/mongo/hooks/__init__.py rename to providers/src/airflow/providers/mysql/hooks/__init__.py diff --git a/airflow/providers/mysql/hooks/mysql.py b/providers/src/airflow/providers/mysql/hooks/mysql.py similarity index 100% rename from airflow/providers/mysql/hooks/mysql.py rename to providers/src/airflow/providers/mysql/hooks/mysql.py diff --git a/airflow/providers/mongo/sensors/__init__.py b/providers/src/airflow/providers/mysql/operators/__init__.py similarity index 100% rename from airflow/providers/mongo/sensors/__init__.py rename to providers/src/airflow/providers/mysql/operators/__init__.py diff --git a/airflow/providers/mysql/operators/mysql.py b/providers/src/airflow/providers/mysql/operators/mysql.py similarity index 100% rename from airflow/providers/mysql/operators/mysql.py rename to providers/src/airflow/providers/mysql/operators/mysql.py diff --git a/airflow/providers/mysql/provider.yaml b/providers/src/airflow/providers/mysql/provider.yaml similarity index 100% rename from airflow/providers/mysql/provider.yaml rename to providers/src/airflow/providers/mysql/provider.yaml diff --git a/airflow/providers/mysql/transfers/__init__.py b/providers/src/airflow/providers/mysql/transfers/__init__.py similarity index 100% rename from airflow/providers/mysql/transfers/__init__.py rename to providers/src/airflow/providers/mysql/transfers/__init__.py diff --git a/airflow/providers/mysql/transfers/presto_to_mysql.py b/providers/src/airflow/providers/mysql/transfers/presto_to_mysql.py similarity index 100% rename from airflow/providers/mysql/transfers/presto_to_mysql.py rename to providers/src/airflow/providers/mysql/transfers/presto_to_mysql.py diff --git a/airflow/providers/mysql/transfers/s3_to_mysql.py b/providers/src/airflow/providers/mysql/transfers/s3_to_mysql.py similarity index 100% rename from airflow/providers/mysql/transfers/s3_to_mysql.py rename to providers/src/airflow/providers/mysql/transfers/s3_to_mysql.py diff --git a/airflow/providers/mysql/transfers/trino_to_mysql.py b/providers/src/airflow/providers/mysql/transfers/trino_to_mysql.py similarity index 100% rename from airflow/providers/mysql/transfers/trino_to_mysql.py rename to providers/src/airflow/providers/mysql/transfers/trino_to_mysql.py diff --git a/airflow/providers/mysql/transfers/vertica_to_mysql.py b/providers/src/airflow/providers/mysql/transfers/vertica_to_mysql.py similarity index 100% rename from airflow/providers/mysql/transfers/vertica_to_mysql.py rename to providers/src/airflow/providers/mysql/transfers/vertica_to_mysql.py diff --git a/airflow/providers/neo4j/.latest-doc-only-change.txt b/providers/src/airflow/providers/neo4j/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/neo4j/.latest-doc-only-change.txt rename to providers/src/airflow/providers/neo4j/.latest-doc-only-change.txt diff --git a/airflow/providers/neo4j/CHANGELOG.rst b/providers/src/airflow/providers/neo4j/CHANGELOG.rst similarity index 100% rename from airflow/providers/neo4j/CHANGELOG.rst rename to providers/src/airflow/providers/neo4j/CHANGELOG.rst diff --git a/airflow/providers/neo4j/README.md b/providers/src/airflow/providers/neo4j/README.md similarity index 100% rename from airflow/providers/neo4j/README.md rename to providers/src/airflow/providers/neo4j/README.md diff --git a/airflow/providers/neo4j/__init__.py b/providers/src/airflow/providers/neo4j/__init__.py similarity index 100% rename from airflow/providers/neo4j/__init__.py rename to providers/src/airflow/providers/neo4j/__init__.py diff --git a/airflow/providers/mysql/hooks/__init__.py b/providers/src/airflow/providers/neo4j/hooks/__init__.py similarity index 100% rename from airflow/providers/mysql/hooks/__init__.py rename to providers/src/airflow/providers/neo4j/hooks/__init__.py diff --git a/airflow/providers/neo4j/hooks/neo4j.py b/providers/src/airflow/providers/neo4j/hooks/neo4j.py similarity index 100% rename from airflow/providers/neo4j/hooks/neo4j.py rename to providers/src/airflow/providers/neo4j/hooks/neo4j.py diff --git a/airflow/providers/mysql/operators/__init__.py b/providers/src/airflow/providers/neo4j/operators/__init__.py similarity index 100% rename from airflow/providers/mysql/operators/__init__.py rename to providers/src/airflow/providers/neo4j/operators/__init__.py diff --git a/airflow/providers/neo4j/operators/neo4j.py b/providers/src/airflow/providers/neo4j/operators/neo4j.py similarity index 100% rename from airflow/providers/neo4j/operators/neo4j.py rename to providers/src/airflow/providers/neo4j/operators/neo4j.py diff --git a/airflow/providers/neo4j/provider.yaml b/providers/src/airflow/providers/neo4j/provider.yaml similarity index 100% rename from airflow/providers/neo4j/provider.yaml rename to providers/src/airflow/providers/neo4j/provider.yaml diff --git a/airflow/providers/odbc/.latest-doc-only-change.txt b/providers/src/airflow/providers/odbc/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/odbc/.latest-doc-only-change.txt rename to providers/src/airflow/providers/odbc/.latest-doc-only-change.txt diff --git a/airflow/providers/odbc/CHANGELOG.rst b/providers/src/airflow/providers/odbc/CHANGELOG.rst similarity index 100% rename from airflow/providers/odbc/CHANGELOG.rst rename to providers/src/airflow/providers/odbc/CHANGELOG.rst diff --git a/airflow/providers/odbc/__init__.py b/providers/src/airflow/providers/odbc/__init__.py similarity index 100% rename from airflow/providers/odbc/__init__.py rename to providers/src/airflow/providers/odbc/__init__.py diff --git a/airflow/providers/odbc/hooks/__init__.py b/providers/src/airflow/providers/odbc/hooks/__init__.py similarity index 100% rename from airflow/providers/odbc/hooks/__init__.py rename to providers/src/airflow/providers/odbc/hooks/__init__.py diff --git a/airflow/providers/odbc/hooks/odbc.py b/providers/src/airflow/providers/odbc/hooks/odbc.py similarity index 100% rename from airflow/providers/odbc/hooks/odbc.py rename to providers/src/airflow/providers/odbc/hooks/odbc.py diff --git a/airflow/providers/odbc/provider.yaml b/providers/src/airflow/providers/odbc/provider.yaml similarity index 100% rename from airflow/providers/odbc/provider.yaml rename to providers/src/airflow/providers/odbc/provider.yaml diff --git a/airflow/providers/openai/.latest-doc-only-change.txt b/providers/src/airflow/providers/openai/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/openai/.latest-doc-only-change.txt rename to providers/src/airflow/providers/openai/.latest-doc-only-change.txt diff --git a/airflow/providers/openai/CHANGELOG.rst b/providers/src/airflow/providers/openai/CHANGELOG.rst similarity index 100% rename from airflow/providers/openai/CHANGELOG.rst rename to providers/src/airflow/providers/openai/CHANGELOG.rst diff --git a/airflow/providers/openai/__init__.py b/providers/src/airflow/providers/openai/__init__.py similarity index 100% rename from airflow/providers/openai/__init__.py rename to providers/src/airflow/providers/openai/__init__.py diff --git a/airflow/providers/openai/exceptions.py b/providers/src/airflow/providers/openai/exceptions.py similarity index 100% rename from airflow/providers/openai/exceptions.py rename to providers/src/airflow/providers/openai/exceptions.py diff --git a/airflow/providers/openai/hooks/__init__.py b/providers/src/airflow/providers/openai/hooks/__init__.py similarity index 100% rename from airflow/providers/openai/hooks/__init__.py rename to providers/src/airflow/providers/openai/hooks/__init__.py diff --git a/airflow/providers/openai/hooks/openai.py b/providers/src/airflow/providers/openai/hooks/openai.py similarity index 100% rename from airflow/providers/openai/hooks/openai.py rename to providers/src/airflow/providers/openai/hooks/openai.py diff --git a/airflow/providers/openai/operators/__init__.py b/providers/src/airflow/providers/openai/operators/__init__.py similarity index 100% rename from airflow/providers/openai/operators/__init__.py rename to providers/src/airflow/providers/openai/operators/__init__.py diff --git a/airflow/providers/openai/operators/openai.py b/providers/src/airflow/providers/openai/operators/openai.py similarity index 100% rename from airflow/providers/openai/operators/openai.py rename to providers/src/airflow/providers/openai/operators/openai.py diff --git a/airflow/providers/openai/provider.yaml b/providers/src/airflow/providers/openai/provider.yaml similarity index 100% rename from airflow/providers/openai/provider.yaml rename to providers/src/airflow/providers/openai/provider.yaml diff --git a/airflow/providers/openai/triggers/__init__.py b/providers/src/airflow/providers/openai/triggers/__init__.py similarity index 100% rename from airflow/providers/openai/triggers/__init__.py rename to providers/src/airflow/providers/openai/triggers/__init__.py diff --git a/airflow/providers/openai/triggers/openai.py b/providers/src/airflow/providers/openai/triggers/openai.py similarity index 100% rename from airflow/providers/openai/triggers/openai.py rename to providers/src/airflow/providers/openai/triggers/openai.py diff --git a/airflow/providers/openfaas/.latest-doc-only-change.txt b/providers/src/airflow/providers/openfaas/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/openfaas/.latest-doc-only-change.txt rename to providers/src/airflow/providers/openfaas/.latest-doc-only-change.txt diff --git a/airflow/providers/openfaas/CHANGELOG.rst b/providers/src/airflow/providers/openfaas/CHANGELOG.rst similarity index 100% rename from airflow/providers/openfaas/CHANGELOG.rst rename to providers/src/airflow/providers/openfaas/CHANGELOG.rst diff --git a/airflow/providers/openfaas/__init__.py b/providers/src/airflow/providers/openfaas/__init__.py similarity index 100% rename from airflow/providers/openfaas/__init__.py rename to providers/src/airflow/providers/openfaas/__init__.py diff --git a/airflow/providers/neo4j/hooks/__init__.py b/providers/src/airflow/providers/openfaas/hooks/__init__.py similarity index 100% rename from airflow/providers/neo4j/hooks/__init__.py rename to providers/src/airflow/providers/openfaas/hooks/__init__.py diff --git a/airflow/providers/openfaas/hooks/openfaas.py b/providers/src/airflow/providers/openfaas/hooks/openfaas.py similarity index 100% rename from airflow/providers/openfaas/hooks/openfaas.py rename to providers/src/airflow/providers/openfaas/hooks/openfaas.py diff --git a/airflow/providers/openfaas/provider.yaml b/providers/src/airflow/providers/openfaas/provider.yaml similarity index 100% rename from airflow/providers/openfaas/provider.yaml rename to providers/src/airflow/providers/openfaas/provider.yaml diff --git a/airflow/providers/openlineage/.latest-doc-only-change.txt b/providers/src/airflow/providers/openlineage/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/openlineage/.latest-doc-only-change.txt rename to providers/src/airflow/providers/openlineage/.latest-doc-only-change.txt diff --git a/airflow/providers/openlineage/CHANGELOG.rst b/providers/src/airflow/providers/openlineage/CHANGELOG.rst similarity index 100% rename from airflow/providers/openlineage/CHANGELOG.rst rename to providers/src/airflow/providers/openlineage/CHANGELOG.rst diff --git a/airflow/providers/openlineage/__init__.py b/providers/src/airflow/providers/openlineage/__init__.py similarity index 100% rename from airflow/providers/openlineage/__init__.py rename to providers/src/airflow/providers/openlineage/__init__.py diff --git a/airflow/providers/openlineage/conf.py b/providers/src/airflow/providers/openlineage/conf.py similarity index 100% rename from airflow/providers/openlineage/conf.py rename to providers/src/airflow/providers/openlineage/conf.py diff --git a/airflow/providers/openlineage/extractors/__init__.py b/providers/src/airflow/providers/openlineage/extractors/__init__.py similarity index 100% rename from airflow/providers/openlineage/extractors/__init__.py rename to providers/src/airflow/providers/openlineage/extractors/__init__.py diff --git a/airflow/providers/openlineage/extractors/base.py b/providers/src/airflow/providers/openlineage/extractors/base.py similarity index 100% rename from airflow/providers/openlineage/extractors/base.py rename to providers/src/airflow/providers/openlineage/extractors/base.py diff --git a/airflow/providers/openlineage/extractors/bash.py b/providers/src/airflow/providers/openlineage/extractors/bash.py similarity index 100% rename from airflow/providers/openlineage/extractors/bash.py rename to providers/src/airflow/providers/openlineage/extractors/bash.py diff --git a/airflow/providers/openlineage/extractors/manager.py b/providers/src/airflow/providers/openlineage/extractors/manager.py similarity index 100% rename from airflow/providers/openlineage/extractors/manager.py rename to providers/src/airflow/providers/openlineage/extractors/manager.py diff --git a/airflow/providers/openlineage/extractors/python.py b/providers/src/airflow/providers/openlineage/extractors/python.py similarity index 100% rename from airflow/providers/openlineage/extractors/python.py rename to providers/src/airflow/providers/openlineage/extractors/python.py diff --git a/airflow/providers/openlineage/facets/AirflowDagRunFacet.json b/providers/src/airflow/providers/openlineage/facets/AirflowDagRunFacet.json similarity index 100% rename from airflow/providers/openlineage/facets/AirflowDagRunFacet.json rename to providers/src/airflow/providers/openlineage/facets/AirflowDagRunFacet.json diff --git a/airflow/providers/openlineage/facets/AirflowDebugRunFacet.json b/providers/src/airflow/providers/openlineage/facets/AirflowDebugRunFacet.json similarity index 100% rename from airflow/providers/openlineage/facets/AirflowDebugRunFacet.json rename to providers/src/airflow/providers/openlineage/facets/AirflowDebugRunFacet.json diff --git a/airflow/providers/openlineage/facets/AirflowJobFacet.json b/providers/src/airflow/providers/openlineage/facets/AirflowJobFacet.json similarity index 100% rename from airflow/providers/openlineage/facets/AirflowJobFacet.json rename to providers/src/airflow/providers/openlineage/facets/AirflowJobFacet.json diff --git a/airflow/providers/openlineage/facets/AirflowRunFacet.json b/providers/src/airflow/providers/openlineage/facets/AirflowRunFacet.json similarity index 100% rename from airflow/providers/openlineage/facets/AirflowRunFacet.json rename to providers/src/airflow/providers/openlineage/facets/AirflowRunFacet.json diff --git a/airflow/providers/openlineage/facets/AirflowStateRunFacet.json b/providers/src/airflow/providers/openlineage/facets/AirflowStateRunFacet.json similarity index 100% rename from airflow/providers/openlineage/facets/AirflowStateRunFacet.json rename to providers/src/airflow/providers/openlineage/facets/AirflowStateRunFacet.json diff --git a/airflow/providers/openlineage/facets/__init__.py b/providers/src/airflow/providers/openlineage/facets/__init__.py similarity index 100% rename from airflow/providers/openlineage/facets/__init__.py rename to providers/src/airflow/providers/openlineage/facets/__init__.py diff --git a/airflow/providers/openlineage/plugins/__init__.py b/providers/src/airflow/providers/openlineage/plugins/__init__.py similarity index 100% rename from airflow/providers/openlineage/plugins/__init__.py rename to providers/src/airflow/providers/openlineage/plugins/__init__.py diff --git a/airflow/providers/openlineage/plugins/adapter.py b/providers/src/airflow/providers/openlineage/plugins/adapter.py similarity index 100% rename from airflow/providers/openlineage/plugins/adapter.py rename to providers/src/airflow/providers/openlineage/plugins/adapter.py diff --git a/airflow/providers/openlineage/plugins/facets.py b/providers/src/airflow/providers/openlineage/plugins/facets.py similarity index 100% rename from airflow/providers/openlineage/plugins/facets.py rename to providers/src/airflow/providers/openlineage/plugins/facets.py diff --git a/airflow/providers/openlineage/plugins/listener.py b/providers/src/airflow/providers/openlineage/plugins/listener.py similarity index 100% rename from airflow/providers/openlineage/plugins/listener.py rename to providers/src/airflow/providers/openlineage/plugins/listener.py diff --git a/airflow/providers/openlineage/plugins/macros.py b/providers/src/airflow/providers/openlineage/plugins/macros.py similarity index 100% rename from airflow/providers/openlineage/plugins/macros.py rename to providers/src/airflow/providers/openlineage/plugins/macros.py diff --git a/airflow/providers/openlineage/plugins/openlineage.py b/providers/src/airflow/providers/openlineage/plugins/openlineage.py similarity index 100% rename from airflow/providers/openlineage/plugins/openlineage.py rename to providers/src/airflow/providers/openlineage/plugins/openlineage.py diff --git a/airflow/providers/openlineage/provider.yaml b/providers/src/airflow/providers/openlineage/provider.yaml similarity index 100% rename from airflow/providers/openlineage/provider.yaml rename to providers/src/airflow/providers/openlineage/provider.yaml diff --git a/airflow/providers/openlineage/sqlparser.py b/providers/src/airflow/providers/openlineage/sqlparser.py similarity index 100% rename from airflow/providers/openlineage/sqlparser.py rename to providers/src/airflow/providers/openlineage/sqlparser.py diff --git a/airflow/providers/openlineage/utils/__init__.py b/providers/src/airflow/providers/openlineage/utils/__init__.py similarity index 100% rename from airflow/providers/openlineage/utils/__init__.py rename to providers/src/airflow/providers/openlineage/utils/__init__.py diff --git a/airflow/providers/openlineage/utils/asset_compat_lineage_collector.py b/providers/src/airflow/providers/openlineage/utils/asset_compat_lineage_collector.py similarity index 100% rename from airflow/providers/openlineage/utils/asset_compat_lineage_collector.py rename to providers/src/airflow/providers/openlineage/utils/asset_compat_lineage_collector.py diff --git a/airflow/providers/openlineage/utils/selective_enable.py b/providers/src/airflow/providers/openlineage/utils/selective_enable.py similarity index 100% rename from airflow/providers/openlineage/utils/selective_enable.py rename to providers/src/airflow/providers/openlineage/utils/selective_enable.py diff --git a/airflow/providers/openlineage/utils/sql.py b/providers/src/airflow/providers/openlineage/utils/sql.py similarity index 100% rename from airflow/providers/openlineage/utils/sql.py rename to providers/src/airflow/providers/openlineage/utils/sql.py diff --git a/airflow/providers/openlineage/utils/utils.py b/providers/src/airflow/providers/openlineage/utils/utils.py similarity index 100% rename from airflow/providers/openlineage/utils/utils.py rename to providers/src/airflow/providers/openlineage/utils/utils.py diff --git a/airflow/providers/opensearch/.latest-doc-only-change.txt b/providers/src/airflow/providers/opensearch/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/opensearch/.latest-doc-only-change.txt rename to providers/src/airflow/providers/opensearch/.latest-doc-only-change.txt diff --git a/airflow/providers/opensearch/CHANGELOG.rst b/providers/src/airflow/providers/opensearch/CHANGELOG.rst similarity index 100% rename from airflow/providers/opensearch/CHANGELOG.rst rename to providers/src/airflow/providers/opensearch/CHANGELOG.rst diff --git a/airflow/providers/opensearch/__init__.py b/providers/src/airflow/providers/opensearch/__init__.py similarity index 100% rename from airflow/providers/opensearch/__init__.py rename to providers/src/airflow/providers/opensearch/__init__.py diff --git a/airflow/providers/opensearch/hooks/__init__.py b/providers/src/airflow/providers/opensearch/hooks/__init__.py similarity index 100% rename from airflow/providers/opensearch/hooks/__init__.py rename to providers/src/airflow/providers/opensearch/hooks/__init__.py diff --git a/airflow/providers/opensearch/hooks/opensearch.py b/providers/src/airflow/providers/opensearch/hooks/opensearch.py similarity index 100% rename from airflow/providers/opensearch/hooks/opensearch.py rename to providers/src/airflow/providers/opensearch/hooks/opensearch.py diff --git a/airflow/providers/opensearch/log/__init__.py b/providers/src/airflow/providers/opensearch/log/__init__.py similarity index 100% rename from airflow/providers/opensearch/log/__init__.py rename to providers/src/airflow/providers/opensearch/log/__init__.py diff --git a/airflow/providers/opensearch/log/os_json_formatter.py b/providers/src/airflow/providers/opensearch/log/os_json_formatter.py similarity index 100% rename from airflow/providers/opensearch/log/os_json_formatter.py rename to providers/src/airflow/providers/opensearch/log/os_json_formatter.py diff --git a/airflow/providers/opensearch/log/os_response.py b/providers/src/airflow/providers/opensearch/log/os_response.py similarity index 100% rename from airflow/providers/opensearch/log/os_response.py rename to providers/src/airflow/providers/opensearch/log/os_response.py diff --git a/airflow/providers/opensearch/log/os_task_handler.py b/providers/src/airflow/providers/opensearch/log/os_task_handler.py similarity index 100% rename from airflow/providers/opensearch/log/os_task_handler.py rename to providers/src/airflow/providers/opensearch/log/os_task_handler.py diff --git a/airflow/providers/opensearch/operators/__init__.py b/providers/src/airflow/providers/opensearch/operators/__init__.py similarity index 100% rename from airflow/providers/opensearch/operators/__init__.py rename to providers/src/airflow/providers/opensearch/operators/__init__.py diff --git a/airflow/providers/opensearch/operators/opensearch.py b/providers/src/airflow/providers/opensearch/operators/opensearch.py similarity index 100% rename from airflow/providers/opensearch/operators/opensearch.py rename to providers/src/airflow/providers/opensearch/operators/opensearch.py diff --git a/airflow/providers/opensearch/provider.yaml b/providers/src/airflow/providers/opensearch/provider.yaml similarity index 100% rename from airflow/providers/opensearch/provider.yaml rename to providers/src/airflow/providers/opensearch/provider.yaml diff --git a/airflow/providers/opsgenie/.latest-doc-only-change.txt b/providers/src/airflow/providers/opsgenie/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/opsgenie/.latest-doc-only-change.txt rename to providers/src/airflow/providers/opsgenie/.latest-doc-only-change.txt diff --git a/airflow/providers/opsgenie/CHANGELOG.rst b/providers/src/airflow/providers/opsgenie/CHANGELOG.rst similarity index 100% rename from airflow/providers/opsgenie/CHANGELOG.rst rename to providers/src/airflow/providers/opsgenie/CHANGELOG.rst diff --git a/airflow/providers/opsgenie/__init__.py b/providers/src/airflow/providers/opsgenie/__init__.py similarity index 100% rename from airflow/providers/opsgenie/__init__.py rename to providers/src/airflow/providers/opsgenie/__init__.py diff --git a/airflow/providers/neo4j/operators/__init__.py b/providers/src/airflow/providers/opsgenie/hooks/__init__.py similarity index 100% rename from airflow/providers/neo4j/operators/__init__.py rename to providers/src/airflow/providers/opsgenie/hooks/__init__.py diff --git a/airflow/providers/opsgenie/hooks/opsgenie.py b/providers/src/airflow/providers/opsgenie/hooks/opsgenie.py similarity index 100% rename from airflow/providers/opsgenie/hooks/opsgenie.py rename to providers/src/airflow/providers/opsgenie/hooks/opsgenie.py diff --git a/airflow/providers/opsgenie/notifications/__init__.py b/providers/src/airflow/providers/opsgenie/notifications/__init__.py similarity index 100% rename from airflow/providers/opsgenie/notifications/__init__.py rename to providers/src/airflow/providers/opsgenie/notifications/__init__.py diff --git a/airflow/providers/opsgenie/notifications/opsgenie.py b/providers/src/airflow/providers/opsgenie/notifications/opsgenie.py similarity index 100% rename from airflow/providers/opsgenie/notifications/opsgenie.py rename to providers/src/airflow/providers/opsgenie/notifications/opsgenie.py diff --git a/airflow/providers/openfaas/hooks/__init__.py b/providers/src/airflow/providers/opsgenie/operators/__init__.py similarity index 100% rename from airflow/providers/openfaas/hooks/__init__.py rename to providers/src/airflow/providers/opsgenie/operators/__init__.py diff --git a/airflow/providers/opsgenie/operators/opsgenie.py b/providers/src/airflow/providers/opsgenie/operators/opsgenie.py similarity index 100% rename from airflow/providers/opsgenie/operators/opsgenie.py rename to providers/src/airflow/providers/opsgenie/operators/opsgenie.py diff --git a/airflow/providers/opsgenie/provider.yaml b/providers/src/airflow/providers/opsgenie/provider.yaml similarity index 100% rename from airflow/providers/opsgenie/provider.yaml rename to providers/src/airflow/providers/opsgenie/provider.yaml diff --git a/airflow/providers/opsgenie/typing/__init__.py b/providers/src/airflow/providers/opsgenie/typing/__init__.py similarity index 100% rename from airflow/providers/opsgenie/typing/__init__.py rename to providers/src/airflow/providers/opsgenie/typing/__init__.py diff --git a/airflow/providers/opsgenie/typing/opsgenie.py b/providers/src/airflow/providers/opsgenie/typing/opsgenie.py similarity index 100% rename from airflow/providers/opsgenie/typing/opsgenie.py rename to providers/src/airflow/providers/opsgenie/typing/opsgenie.py diff --git a/airflow/providers/oracle/.latest-doc-only-change.txt b/providers/src/airflow/providers/oracle/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/oracle/.latest-doc-only-change.txt rename to providers/src/airflow/providers/oracle/.latest-doc-only-change.txt diff --git a/airflow/providers/oracle/CHANGELOG.rst b/providers/src/airflow/providers/oracle/CHANGELOG.rst similarity index 100% rename from airflow/providers/oracle/CHANGELOG.rst rename to providers/src/airflow/providers/oracle/CHANGELOG.rst diff --git a/airflow/providers/oracle/__init__.py b/providers/src/airflow/providers/oracle/__init__.py similarity index 100% rename from airflow/providers/oracle/__init__.py rename to providers/src/airflow/providers/oracle/__init__.py diff --git a/airflow/providers/oracle/example_dags/__init__.py b/providers/src/airflow/providers/oracle/example_dags/__init__.py similarity index 100% rename from airflow/providers/oracle/example_dags/__init__.py rename to providers/src/airflow/providers/oracle/example_dags/__init__.py diff --git a/airflow/providers/oracle/example_dags/example_oracle.py b/providers/src/airflow/providers/oracle/example_dags/example_oracle.py similarity index 100% rename from airflow/providers/oracle/example_dags/example_oracle.py rename to providers/src/airflow/providers/oracle/example_dags/example_oracle.py diff --git a/airflow/providers/opsgenie/hooks/__init__.py b/providers/src/airflow/providers/oracle/hooks/__init__.py similarity index 100% rename from airflow/providers/opsgenie/hooks/__init__.py rename to providers/src/airflow/providers/oracle/hooks/__init__.py diff --git a/airflow/providers/oracle/hooks/oracle.py b/providers/src/airflow/providers/oracle/hooks/oracle.py similarity index 100% rename from airflow/providers/oracle/hooks/oracle.py rename to providers/src/airflow/providers/oracle/hooks/oracle.py diff --git a/airflow/providers/opsgenie/operators/__init__.py b/providers/src/airflow/providers/oracle/operators/__init__.py similarity index 100% rename from airflow/providers/opsgenie/operators/__init__.py rename to providers/src/airflow/providers/oracle/operators/__init__.py diff --git a/airflow/providers/oracle/operators/oracle.py b/providers/src/airflow/providers/oracle/operators/oracle.py similarity index 100% rename from airflow/providers/oracle/operators/oracle.py rename to providers/src/airflow/providers/oracle/operators/oracle.py diff --git a/airflow/providers/oracle/provider.yaml b/providers/src/airflow/providers/oracle/provider.yaml similarity index 100% rename from airflow/providers/oracle/provider.yaml rename to providers/src/airflow/providers/oracle/provider.yaml diff --git a/airflow/providers/oracle/transfers/__init__.py b/providers/src/airflow/providers/oracle/transfers/__init__.py similarity index 100% rename from airflow/providers/oracle/transfers/__init__.py rename to providers/src/airflow/providers/oracle/transfers/__init__.py diff --git a/airflow/providers/oracle/transfers/oracle_to_oracle.py b/providers/src/airflow/providers/oracle/transfers/oracle_to_oracle.py similarity index 100% rename from airflow/providers/oracle/transfers/oracle_to_oracle.py rename to providers/src/airflow/providers/oracle/transfers/oracle_to_oracle.py diff --git a/airflow/providers/pagerduty/.latest-doc-only-change.txt b/providers/src/airflow/providers/pagerduty/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/pagerduty/.latest-doc-only-change.txt rename to providers/src/airflow/providers/pagerduty/.latest-doc-only-change.txt diff --git a/airflow/providers/pagerduty/CHANGELOG.rst b/providers/src/airflow/providers/pagerduty/CHANGELOG.rst similarity index 100% rename from airflow/providers/pagerduty/CHANGELOG.rst rename to providers/src/airflow/providers/pagerduty/CHANGELOG.rst diff --git a/airflow/providers/pagerduty/__init__.py b/providers/src/airflow/providers/pagerduty/__init__.py similarity index 100% rename from airflow/providers/pagerduty/__init__.py rename to providers/src/airflow/providers/pagerduty/__init__.py diff --git a/airflow/providers/oracle/hooks/__init__.py b/providers/src/airflow/providers/pagerduty/hooks/__init__.py similarity index 100% rename from airflow/providers/oracle/hooks/__init__.py rename to providers/src/airflow/providers/pagerduty/hooks/__init__.py diff --git a/airflow/providers/pagerduty/hooks/pagerduty.py b/providers/src/airflow/providers/pagerduty/hooks/pagerduty.py similarity index 100% rename from airflow/providers/pagerduty/hooks/pagerduty.py rename to providers/src/airflow/providers/pagerduty/hooks/pagerduty.py diff --git a/airflow/providers/pagerduty/hooks/pagerduty_events.py b/providers/src/airflow/providers/pagerduty/hooks/pagerduty_events.py similarity index 100% rename from airflow/providers/pagerduty/hooks/pagerduty_events.py rename to providers/src/airflow/providers/pagerduty/hooks/pagerduty_events.py diff --git a/airflow/providers/pagerduty/notifications/__init__.py b/providers/src/airflow/providers/pagerduty/notifications/__init__.py similarity index 100% rename from airflow/providers/pagerduty/notifications/__init__.py rename to providers/src/airflow/providers/pagerduty/notifications/__init__.py diff --git a/airflow/providers/pagerduty/notifications/pagerduty.py b/providers/src/airflow/providers/pagerduty/notifications/pagerduty.py similarity index 100% rename from airflow/providers/pagerduty/notifications/pagerduty.py rename to providers/src/airflow/providers/pagerduty/notifications/pagerduty.py diff --git a/airflow/providers/pagerduty/provider.yaml b/providers/src/airflow/providers/pagerduty/provider.yaml similarity index 100% rename from airflow/providers/pagerduty/provider.yaml rename to providers/src/airflow/providers/pagerduty/provider.yaml diff --git a/airflow/providers/papermill/.latest-doc-only-change.txt b/providers/src/airflow/providers/papermill/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/papermill/.latest-doc-only-change.txt rename to providers/src/airflow/providers/papermill/.latest-doc-only-change.txt diff --git a/airflow/providers/papermill/CHANGELOG.rst b/providers/src/airflow/providers/papermill/CHANGELOG.rst similarity index 100% rename from airflow/providers/papermill/CHANGELOG.rst rename to providers/src/airflow/providers/papermill/CHANGELOG.rst diff --git a/airflow/providers/papermill/__init__.py b/providers/src/airflow/providers/papermill/__init__.py similarity index 100% rename from airflow/providers/papermill/__init__.py rename to providers/src/airflow/providers/papermill/__init__.py diff --git a/airflow/providers/oracle/operators/__init__.py b/providers/src/airflow/providers/papermill/hooks/__init__.py similarity index 100% rename from airflow/providers/oracle/operators/__init__.py rename to providers/src/airflow/providers/papermill/hooks/__init__.py diff --git a/airflow/providers/papermill/hooks/kernel.py b/providers/src/airflow/providers/papermill/hooks/kernel.py similarity index 100% rename from airflow/providers/papermill/hooks/kernel.py rename to providers/src/airflow/providers/papermill/hooks/kernel.py diff --git a/airflow/providers/pagerduty/hooks/__init__.py b/providers/src/airflow/providers/papermill/operators/__init__.py similarity index 100% rename from airflow/providers/pagerduty/hooks/__init__.py rename to providers/src/airflow/providers/papermill/operators/__init__.py diff --git a/airflow/providers/papermill/operators/papermill.py b/providers/src/airflow/providers/papermill/operators/papermill.py similarity index 100% rename from airflow/providers/papermill/operators/papermill.py rename to providers/src/airflow/providers/papermill/operators/papermill.py diff --git a/airflow/providers/papermill/provider.yaml b/providers/src/airflow/providers/papermill/provider.yaml similarity index 100% rename from airflow/providers/papermill/provider.yaml rename to providers/src/airflow/providers/papermill/provider.yaml diff --git a/airflow/providers/pgvector/.latest-doc-only-change.txt b/providers/src/airflow/providers/pgvector/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/pgvector/.latest-doc-only-change.txt rename to providers/src/airflow/providers/pgvector/.latest-doc-only-change.txt diff --git a/airflow/providers/pgvector/CHANGELOG.rst b/providers/src/airflow/providers/pgvector/CHANGELOG.rst similarity index 100% rename from airflow/providers/pgvector/CHANGELOG.rst rename to providers/src/airflow/providers/pgvector/CHANGELOG.rst diff --git a/airflow/providers/pgvector/__init__.py b/providers/src/airflow/providers/pgvector/__init__.py similarity index 100% rename from airflow/providers/pgvector/__init__.py rename to providers/src/airflow/providers/pgvector/__init__.py diff --git a/airflow/providers/pgvector/hooks/__init__.py b/providers/src/airflow/providers/pgvector/hooks/__init__.py similarity index 100% rename from airflow/providers/pgvector/hooks/__init__.py rename to providers/src/airflow/providers/pgvector/hooks/__init__.py diff --git a/airflow/providers/pgvector/hooks/pgvector.py b/providers/src/airflow/providers/pgvector/hooks/pgvector.py similarity index 100% rename from airflow/providers/pgvector/hooks/pgvector.py rename to providers/src/airflow/providers/pgvector/hooks/pgvector.py diff --git a/airflow/providers/pgvector/operators/__init__.py b/providers/src/airflow/providers/pgvector/operators/__init__.py similarity index 100% rename from airflow/providers/pgvector/operators/__init__.py rename to providers/src/airflow/providers/pgvector/operators/__init__.py diff --git a/airflow/providers/pgvector/operators/pgvector.py b/providers/src/airflow/providers/pgvector/operators/pgvector.py similarity index 100% rename from airflow/providers/pgvector/operators/pgvector.py rename to providers/src/airflow/providers/pgvector/operators/pgvector.py diff --git a/airflow/providers/pgvector/provider.yaml b/providers/src/airflow/providers/pgvector/provider.yaml similarity index 100% rename from airflow/providers/pgvector/provider.yaml rename to providers/src/airflow/providers/pgvector/provider.yaml diff --git a/airflow/providers/pinecone/.latest-doc-only-change.txt b/providers/src/airflow/providers/pinecone/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/pinecone/.latest-doc-only-change.txt rename to providers/src/airflow/providers/pinecone/.latest-doc-only-change.txt diff --git a/airflow/providers/pinecone/CHANGELOG.rst b/providers/src/airflow/providers/pinecone/CHANGELOG.rst similarity index 100% rename from airflow/providers/pinecone/CHANGELOG.rst rename to providers/src/airflow/providers/pinecone/CHANGELOG.rst diff --git a/airflow/providers/pinecone/__init__.py b/providers/src/airflow/providers/pinecone/__init__.py similarity index 100% rename from airflow/providers/pinecone/__init__.py rename to providers/src/airflow/providers/pinecone/__init__.py diff --git a/airflow/providers/pinecone/hooks/__init__.py b/providers/src/airflow/providers/pinecone/hooks/__init__.py similarity index 100% rename from airflow/providers/pinecone/hooks/__init__.py rename to providers/src/airflow/providers/pinecone/hooks/__init__.py diff --git a/airflow/providers/pinecone/hooks/pinecone.py b/providers/src/airflow/providers/pinecone/hooks/pinecone.py similarity index 100% rename from airflow/providers/pinecone/hooks/pinecone.py rename to providers/src/airflow/providers/pinecone/hooks/pinecone.py diff --git a/airflow/providers/pinecone/operators/__init__.py b/providers/src/airflow/providers/pinecone/operators/__init__.py similarity index 100% rename from airflow/providers/pinecone/operators/__init__.py rename to providers/src/airflow/providers/pinecone/operators/__init__.py diff --git a/airflow/providers/pinecone/operators/pinecone.py b/providers/src/airflow/providers/pinecone/operators/pinecone.py similarity index 100% rename from airflow/providers/pinecone/operators/pinecone.py rename to providers/src/airflow/providers/pinecone/operators/pinecone.py diff --git a/airflow/providers/pinecone/provider.yaml b/providers/src/airflow/providers/pinecone/provider.yaml similarity index 100% rename from airflow/providers/pinecone/provider.yaml rename to providers/src/airflow/providers/pinecone/provider.yaml diff --git a/airflow/providers/postgres/.latest-doc-only-change.txt b/providers/src/airflow/providers/postgres/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/postgres/.latest-doc-only-change.txt rename to providers/src/airflow/providers/postgres/.latest-doc-only-change.txt diff --git a/airflow/providers/postgres/CHANGELOG.rst b/providers/src/airflow/providers/postgres/CHANGELOG.rst similarity index 100% rename from airflow/providers/postgres/CHANGELOG.rst rename to providers/src/airflow/providers/postgres/CHANGELOG.rst diff --git a/airflow/providers/postgres/__init__.py b/providers/src/airflow/providers/postgres/__init__.py similarity index 100% rename from airflow/providers/postgres/__init__.py rename to providers/src/airflow/providers/postgres/__init__.py diff --git a/airflow/providers/postgres/assets/__init__.py b/providers/src/airflow/providers/postgres/assets/__init__.py similarity index 100% rename from airflow/providers/postgres/assets/__init__.py rename to providers/src/airflow/providers/postgres/assets/__init__.py diff --git a/airflow/providers/postgres/assets/postgres.py b/providers/src/airflow/providers/postgres/assets/postgres.py similarity index 100% rename from airflow/providers/postgres/assets/postgres.py rename to providers/src/airflow/providers/postgres/assets/postgres.py diff --git a/airflow/providers/papermill/hooks/__init__.py b/providers/src/airflow/providers/postgres/hooks/__init__.py similarity index 100% rename from airflow/providers/papermill/hooks/__init__.py rename to providers/src/airflow/providers/postgres/hooks/__init__.py diff --git a/airflow/providers/postgres/hooks/postgres.py b/providers/src/airflow/providers/postgres/hooks/postgres.py similarity index 100% rename from airflow/providers/postgres/hooks/postgres.py rename to providers/src/airflow/providers/postgres/hooks/postgres.py diff --git a/airflow/providers/papermill/operators/__init__.py b/providers/src/airflow/providers/postgres/operators/__init__.py similarity index 100% rename from airflow/providers/papermill/operators/__init__.py rename to providers/src/airflow/providers/postgres/operators/__init__.py diff --git a/airflow/providers/postgres/operators/postgres.py b/providers/src/airflow/providers/postgres/operators/postgres.py similarity index 100% rename from airflow/providers/postgres/operators/postgres.py rename to providers/src/airflow/providers/postgres/operators/postgres.py diff --git a/airflow/providers/postgres/provider.yaml b/providers/src/airflow/providers/postgres/provider.yaml similarity index 100% rename from airflow/providers/postgres/provider.yaml rename to providers/src/airflow/providers/postgres/provider.yaml diff --git a/airflow/providers/presto/.latest-doc-only-change.txt b/providers/src/airflow/providers/presto/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/presto/.latest-doc-only-change.txt rename to providers/src/airflow/providers/presto/.latest-doc-only-change.txt diff --git a/airflow/providers/presto/CHANGELOG.rst b/providers/src/airflow/providers/presto/CHANGELOG.rst similarity index 100% rename from airflow/providers/presto/CHANGELOG.rst rename to providers/src/airflow/providers/presto/CHANGELOG.rst diff --git a/airflow/providers/presto/__init__.py b/providers/src/airflow/providers/presto/__init__.py similarity index 100% rename from airflow/providers/presto/__init__.py rename to providers/src/airflow/providers/presto/__init__.py diff --git a/airflow/providers/postgres/hooks/__init__.py b/providers/src/airflow/providers/presto/hooks/__init__.py similarity index 100% rename from airflow/providers/postgres/hooks/__init__.py rename to providers/src/airflow/providers/presto/hooks/__init__.py diff --git a/airflow/providers/presto/hooks/presto.py b/providers/src/airflow/providers/presto/hooks/presto.py similarity index 100% rename from airflow/providers/presto/hooks/presto.py rename to providers/src/airflow/providers/presto/hooks/presto.py diff --git a/airflow/providers/presto/provider.yaml b/providers/src/airflow/providers/presto/provider.yaml similarity index 100% rename from airflow/providers/presto/provider.yaml rename to providers/src/airflow/providers/presto/provider.yaml diff --git a/airflow/providers/presto/transfers/__init__.py b/providers/src/airflow/providers/presto/transfers/__init__.py similarity index 100% rename from airflow/providers/presto/transfers/__init__.py rename to providers/src/airflow/providers/presto/transfers/__init__.py diff --git a/airflow/providers/presto/transfers/gcs_to_presto.py b/providers/src/airflow/providers/presto/transfers/gcs_to_presto.py similarity index 100% rename from airflow/providers/presto/transfers/gcs_to_presto.py rename to providers/src/airflow/providers/presto/transfers/gcs_to_presto.py diff --git a/airflow/providers/qdrant/.latest-doc-only-change.txt b/providers/src/airflow/providers/qdrant/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/qdrant/.latest-doc-only-change.txt rename to providers/src/airflow/providers/qdrant/.latest-doc-only-change.txt diff --git a/airflow/providers/qdrant/CHANGELOG.rst b/providers/src/airflow/providers/qdrant/CHANGELOG.rst similarity index 100% rename from airflow/providers/qdrant/CHANGELOG.rst rename to providers/src/airflow/providers/qdrant/CHANGELOG.rst diff --git a/airflow/providers/qdrant/__init__.py b/providers/src/airflow/providers/qdrant/__init__.py similarity index 100% rename from airflow/providers/qdrant/__init__.py rename to providers/src/airflow/providers/qdrant/__init__.py diff --git a/airflow/providers/qdrant/hooks/__init__.py b/providers/src/airflow/providers/qdrant/hooks/__init__.py similarity index 100% rename from airflow/providers/qdrant/hooks/__init__.py rename to providers/src/airflow/providers/qdrant/hooks/__init__.py diff --git a/airflow/providers/qdrant/hooks/qdrant.py b/providers/src/airflow/providers/qdrant/hooks/qdrant.py similarity index 100% rename from airflow/providers/qdrant/hooks/qdrant.py rename to providers/src/airflow/providers/qdrant/hooks/qdrant.py diff --git a/airflow/providers/qdrant/operators/__init__.py b/providers/src/airflow/providers/qdrant/operators/__init__.py similarity index 100% rename from airflow/providers/qdrant/operators/__init__.py rename to providers/src/airflow/providers/qdrant/operators/__init__.py diff --git a/airflow/providers/qdrant/operators/qdrant.py b/providers/src/airflow/providers/qdrant/operators/qdrant.py similarity index 100% rename from airflow/providers/qdrant/operators/qdrant.py rename to providers/src/airflow/providers/qdrant/operators/qdrant.py diff --git a/airflow/providers/qdrant/provider.yaml b/providers/src/airflow/providers/qdrant/provider.yaml similarity index 100% rename from airflow/providers/qdrant/provider.yaml rename to providers/src/airflow/providers/qdrant/provider.yaml diff --git a/airflow/providers/redis/.latest-doc-only-change.txt b/providers/src/airflow/providers/redis/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/redis/.latest-doc-only-change.txt rename to providers/src/airflow/providers/redis/.latest-doc-only-change.txt diff --git a/airflow/providers/redis/CHANGELOG.rst b/providers/src/airflow/providers/redis/CHANGELOG.rst similarity index 100% rename from airflow/providers/redis/CHANGELOG.rst rename to providers/src/airflow/providers/redis/CHANGELOG.rst diff --git a/airflow/providers/redis/__init__.py b/providers/src/airflow/providers/redis/__init__.py similarity index 100% rename from airflow/providers/redis/__init__.py rename to providers/src/airflow/providers/redis/__init__.py diff --git a/airflow/providers/postgres/operators/__init__.py b/providers/src/airflow/providers/redis/hooks/__init__.py similarity index 100% rename from airflow/providers/postgres/operators/__init__.py rename to providers/src/airflow/providers/redis/hooks/__init__.py diff --git a/airflow/providers/redis/hooks/redis.py b/providers/src/airflow/providers/redis/hooks/redis.py similarity index 100% rename from airflow/providers/redis/hooks/redis.py rename to providers/src/airflow/providers/redis/hooks/redis.py diff --git a/airflow/providers/presto/hooks/__init__.py b/providers/src/airflow/providers/redis/log/__init__.py similarity index 100% rename from airflow/providers/presto/hooks/__init__.py rename to providers/src/airflow/providers/redis/log/__init__.py diff --git a/airflow/providers/redis/log/redis_task_handler.py b/providers/src/airflow/providers/redis/log/redis_task_handler.py similarity index 100% rename from airflow/providers/redis/log/redis_task_handler.py rename to providers/src/airflow/providers/redis/log/redis_task_handler.py diff --git a/airflow/providers/redis/hooks/__init__.py b/providers/src/airflow/providers/redis/operators/__init__.py similarity index 100% rename from airflow/providers/redis/hooks/__init__.py rename to providers/src/airflow/providers/redis/operators/__init__.py diff --git a/airflow/providers/redis/operators/redis_publish.py b/providers/src/airflow/providers/redis/operators/redis_publish.py similarity index 100% rename from airflow/providers/redis/operators/redis_publish.py rename to providers/src/airflow/providers/redis/operators/redis_publish.py diff --git a/airflow/providers/redis/provider.yaml b/providers/src/airflow/providers/redis/provider.yaml similarity index 100% rename from airflow/providers/redis/provider.yaml rename to providers/src/airflow/providers/redis/provider.yaml diff --git a/airflow/providers/redis/log/__init__.py b/providers/src/airflow/providers/redis/sensors/__init__.py similarity index 100% rename from airflow/providers/redis/log/__init__.py rename to providers/src/airflow/providers/redis/sensors/__init__.py diff --git a/airflow/providers/redis/sensors/redis_key.py b/providers/src/airflow/providers/redis/sensors/redis_key.py similarity index 100% rename from airflow/providers/redis/sensors/redis_key.py rename to providers/src/airflow/providers/redis/sensors/redis_key.py diff --git a/airflow/providers/redis/sensors/redis_pub_sub.py b/providers/src/airflow/providers/redis/sensors/redis_pub_sub.py similarity index 100% rename from airflow/providers/redis/sensors/redis_pub_sub.py rename to providers/src/airflow/providers/redis/sensors/redis_pub_sub.py diff --git a/airflow/providers/salesforce/.latest-doc-only-change.txt b/providers/src/airflow/providers/salesforce/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/salesforce/.latest-doc-only-change.txt rename to providers/src/airflow/providers/salesforce/.latest-doc-only-change.txt diff --git a/airflow/providers/salesforce/CHANGELOG.rst b/providers/src/airflow/providers/salesforce/CHANGELOG.rst similarity index 100% rename from airflow/providers/salesforce/CHANGELOG.rst rename to providers/src/airflow/providers/salesforce/CHANGELOG.rst diff --git a/airflow/providers/salesforce/__init__.py b/providers/src/airflow/providers/salesforce/__init__.py similarity index 100% rename from airflow/providers/salesforce/__init__.py rename to providers/src/airflow/providers/salesforce/__init__.py diff --git a/airflow/providers/redis/operators/__init__.py b/providers/src/airflow/providers/salesforce/hooks/__init__.py similarity index 100% rename from airflow/providers/redis/operators/__init__.py rename to providers/src/airflow/providers/salesforce/hooks/__init__.py diff --git a/airflow/providers/salesforce/hooks/salesforce.py b/providers/src/airflow/providers/salesforce/hooks/salesforce.py similarity index 100% rename from airflow/providers/salesforce/hooks/salesforce.py rename to providers/src/airflow/providers/salesforce/hooks/salesforce.py diff --git a/airflow/providers/salesforce/operators/__init__.py b/providers/src/airflow/providers/salesforce/operators/__init__.py similarity index 100% rename from airflow/providers/salesforce/operators/__init__.py rename to providers/src/airflow/providers/salesforce/operators/__init__.py diff --git a/airflow/providers/salesforce/operators/bulk.py b/providers/src/airflow/providers/salesforce/operators/bulk.py similarity index 100% rename from airflow/providers/salesforce/operators/bulk.py rename to providers/src/airflow/providers/salesforce/operators/bulk.py diff --git a/airflow/providers/salesforce/operators/salesforce_apex_rest.py b/providers/src/airflow/providers/salesforce/operators/salesforce_apex_rest.py similarity index 100% rename from airflow/providers/salesforce/operators/salesforce_apex_rest.py rename to providers/src/airflow/providers/salesforce/operators/salesforce_apex_rest.py diff --git a/airflow/providers/salesforce/provider.yaml b/providers/src/airflow/providers/salesforce/provider.yaml similarity index 100% rename from airflow/providers/salesforce/provider.yaml rename to providers/src/airflow/providers/salesforce/provider.yaml diff --git a/airflow/providers/samba/.latest-doc-only-change.txt b/providers/src/airflow/providers/samba/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/samba/.latest-doc-only-change.txt rename to providers/src/airflow/providers/samba/.latest-doc-only-change.txt diff --git a/airflow/providers/samba/CHANGELOG.rst b/providers/src/airflow/providers/samba/CHANGELOG.rst similarity index 100% rename from airflow/providers/samba/CHANGELOG.rst rename to providers/src/airflow/providers/samba/CHANGELOG.rst diff --git a/airflow/providers/samba/__init__.py b/providers/src/airflow/providers/samba/__init__.py similarity index 100% rename from airflow/providers/samba/__init__.py rename to providers/src/airflow/providers/samba/__init__.py diff --git a/airflow/providers/redis/sensors/__init__.py b/providers/src/airflow/providers/samba/hooks/__init__.py similarity index 100% rename from airflow/providers/redis/sensors/__init__.py rename to providers/src/airflow/providers/samba/hooks/__init__.py diff --git a/airflow/providers/samba/hooks/samba.py b/providers/src/airflow/providers/samba/hooks/samba.py similarity index 100% rename from airflow/providers/samba/hooks/samba.py rename to providers/src/airflow/providers/samba/hooks/samba.py diff --git a/airflow/providers/samba/provider.yaml b/providers/src/airflow/providers/samba/provider.yaml similarity index 100% rename from airflow/providers/samba/provider.yaml rename to providers/src/airflow/providers/samba/provider.yaml diff --git a/airflow/providers/samba/transfers/__init__.py b/providers/src/airflow/providers/samba/transfers/__init__.py similarity index 100% rename from airflow/providers/samba/transfers/__init__.py rename to providers/src/airflow/providers/samba/transfers/__init__.py diff --git a/airflow/providers/samba/transfers/gcs_to_samba.py b/providers/src/airflow/providers/samba/transfers/gcs_to_samba.py similarity index 100% rename from airflow/providers/samba/transfers/gcs_to_samba.py rename to providers/src/airflow/providers/samba/transfers/gcs_to_samba.py diff --git a/airflow/providers/segment/.latest-doc-only-change.txt b/providers/src/airflow/providers/segment/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/segment/.latest-doc-only-change.txt rename to providers/src/airflow/providers/segment/.latest-doc-only-change.txt diff --git a/airflow/providers/segment/CHANGELOG.rst b/providers/src/airflow/providers/segment/CHANGELOG.rst similarity index 100% rename from airflow/providers/segment/CHANGELOG.rst rename to providers/src/airflow/providers/segment/CHANGELOG.rst diff --git a/airflow/providers/segment/__init__.py b/providers/src/airflow/providers/segment/__init__.py similarity index 100% rename from airflow/providers/segment/__init__.py rename to providers/src/airflow/providers/segment/__init__.py diff --git a/airflow/providers/salesforce/hooks/__init__.py b/providers/src/airflow/providers/segment/hooks/__init__.py similarity index 100% rename from airflow/providers/salesforce/hooks/__init__.py rename to providers/src/airflow/providers/segment/hooks/__init__.py diff --git a/airflow/providers/segment/hooks/segment.py b/providers/src/airflow/providers/segment/hooks/segment.py similarity index 100% rename from airflow/providers/segment/hooks/segment.py rename to providers/src/airflow/providers/segment/hooks/segment.py diff --git a/airflow/providers/samba/hooks/__init__.py b/providers/src/airflow/providers/segment/operators/__init__.py similarity index 100% rename from airflow/providers/samba/hooks/__init__.py rename to providers/src/airflow/providers/segment/operators/__init__.py diff --git a/airflow/providers/segment/operators/segment_track_event.py b/providers/src/airflow/providers/segment/operators/segment_track_event.py similarity index 100% rename from airflow/providers/segment/operators/segment_track_event.py rename to providers/src/airflow/providers/segment/operators/segment_track_event.py diff --git a/airflow/providers/segment/provider.yaml b/providers/src/airflow/providers/segment/provider.yaml similarity index 100% rename from airflow/providers/segment/provider.yaml rename to providers/src/airflow/providers/segment/provider.yaml diff --git a/airflow/providers/sendgrid/.latest-doc-only-change.txt b/providers/src/airflow/providers/sendgrid/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/sendgrid/.latest-doc-only-change.txt rename to providers/src/airflow/providers/sendgrid/.latest-doc-only-change.txt diff --git a/airflow/providers/sendgrid/CHANGELOG.rst b/providers/src/airflow/providers/sendgrid/CHANGELOG.rst similarity index 100% rename from airflow/providers/sendgrid/CHANGELOG.rst rename to providers/src/airflow/providers/sendgrid/CHANGELOG.rst diff --git a/airflow/providers/sendgrid/__init__.py b/providers/src/airflow/providers/sendgrid/__init__.py similarity index 100% rename from airflow/providers/sendgrid/__init__.py rename to providers/src/airflow/providers/sendgrid/__init__.py diff --git a/airflow/providers/sendgrid/provider.yaml b/providers/src/airflow/providers/sendgrid/provider.yaml similarity index 100% rename from airflow/providers/sendgrid/provider.yaml rename to providers/src/airflow/providers/sendgrid/provider.yaml diff --git a/airflow/providers/sendgrid/utils/__init__.py b/providers/src/airflow/providers/sendgrid/utils/__init__.py similarity index 100% rename from airflow/providers/sendgrid/utils/__init__.py rename to providers/src/airflow/providers/sendgrid/utils/__init__.py diff --git a/airflow/providers/sendgrid/utils/emailer.py b/providers/src/airflow/providers/sendgrid/utils/emailer.py similarity index 100% rename from airflow/providers/sendgrid/utils/emailer.py rename to providers/src/airflow/providers/sendgrid/utils/emailer.py diff --git a/airflow/providers/sftp/.latest-doc-only-change.txt b/providers/src/airflow/providers/sftp/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/sftp/.latest-doc-only-change.txt rename to providers/src/airflow/providers/sftp/.latest-doc-only-change.txt diff --git a/airflow/providers/sftp/CHANGELOG.rst b/providers/src/airflow/providers/sftp/CHANGELOG.rst similarity index 100% rename from airflow/providers/sftp/CHANGELOG.rst rename to providers/src/airflow/providers/sftp/CHANGELOG.rst diff --git a/airflow/providers/sftp/__init__.py b/providers/src/airflow/providers/sftp/__init__.py similarity index 100% rename from airflow/providers/sftp/__init__.py rename to providers/src/airflow/providers/sftp/__init__.py diff --git a/airflow/providers/sftp/decorators/__init__.py b/providers/src/airflow/providers/sftp/decorators/__init__.py similarity index 100% rename from airflow/providers/sftp/decorators/__init__.py rename to providers/src/airflow/providers/sftp/decorators/__init__.py diff --git a/airflow/providers/sftp/decorators/sensors/__init__.py b/providers/src/airflow/providers/sftp/decorators/sensors/__init__.py similarity index 100% rename from airflow/providers/sftp/decorators/sensors/__init__.py rename to providers/src/airflow/providers/sftp/decorators/sensors/__init__.py diff --git a/airflow/providers/sftp/decorators/sensors/sftp.py b/providers/src/airflow/providers/sftp/decorators/sensors/sftp.py similarity index 100% rename from airflow/providers/sftp/decorators/sensors/sftp.py rename to providers/src/airflow/providers/sftp/decorators/sensors/sftp.py diff --git a/airflow/providers/sftp/hooks/__init__.py b/providers/src/airflow/providers/sftp/hooks/__init__.py similarity index 100% rename from airflow/providers/sftp/hooks/__init__.py rename to providers/src/airflow/providers/sftp/hooks/__init__.py diff --git a/airflow/providers/sftp/hooks/sftp.py b/providers/src/airflow/providers/sftp/hooks/sftp.py similarity index 100% rename from airflow/providers/sftp/hooks/sftp.py rename to providers/src/airflow/providers/sftp/hooks/sftp.py diff --git a/airflow/providers/sftp/operators/__init__.py b/providers/src/airflow/providers/sftp/operators/__init__.py similarity index 100% rename from airflow/providers/sftp/operators/__init__.py rename to providers/src/airflow/providers/sftp/operators/__init__.py diff --git a/airflow/providers/sftp/operators/sftp.py b/providers/src/airflow/providers/sftp/operators/sftp.py similarity index 100% rename from airflow/providers/sftp/operators/sftp.py rename to providers/src/airflow/providers/sftp/operators/sftp.py diff --git a/airflow/providers/sftp/provider.yaml b/providers/src/airflow/providers/sftp/provider.yaml similarity index 100% rename from airflow/providers/sftp/provider.yaml rename to providers/src/airflow/providers/sftp/provider.yaml diff --git a/airflow/providers/sftp/sensors/__init__.py b/providers/src/airflow/providers/sftp/sensors/__init__.py similarity index 100% rename from airflow/providers/sftp/sensors/__init__.py rename to providers/src/airflow/providers/sftp/sensors/__init__.py diff --git a/airflow/providers/sftp/sensors/sftp.py b/providers/src/airflow/providers/sftp/sensors/sftp.py similarity index 100% rename from airflow/providers/sftp/sensors/sftp.py rename to providers/src/airflow/providers/sftp/sensors/sftp.py diff --git a/airflow/providers/sftp/triggers/__init__.py b/providers/src/airflow/providers/sftp/triggers/__init__.py similarity index 100% rename from airflow/providers/sftp/triggers/__init__.py rename to providers/src/airflow/providers/sftp/triggers/__init__.py diff --git a/airflow/providers/sftp/triggers/sftp.py b/providers/src/airflow/providers/sftp/triggers/sftp.py similarity index 100% rename from airflow/providers/sftp/triggers/sftp.py rename to providers/src/airflow/providers/sftp/triggers/sftp.py diff --git a/airflow/providers/singularity/.latest-doc-only-change.txt b/providers/src/airflow/providers/singularity/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/singularity/.latest-doc-only-change.txt rename to providers/src/airflow/providers/singularity/.latest-doc-only-change.txt diff --git a/airflow/providers/singularity/CHANGELOG.rst b/providers/src/airflow/providers/singularity/CHANGELOG.rst similarity index 100% rename from airflow/providers/singularity/CHANGELOG.rst rename to providers/src/airflow/providers/singularity/CHANGELOG.rst diff --git a/airflow/providers/singularity/__init__.py b/providers/src/airflow/providers/singularity/__init__.py similarity index 100% rename from airflow/providers/singularity/__init__.py rename to providers/src/airflow/providers/singularity/__init__.py diff --git a/airflow/providers/segment/hooks/__init__.py b/providers/src/airflow/providers/singularity/operators/__init__.py similarity index 100% rename from airflow/providers/segment/hooks/__init__.py rename to providers/src/airflow/providers/singularity/operators/__init__.py diff --git a/airflow/providers/singularity/operators/singularity.py b/providers/src/airflow/providers/singularity/operators/singularity.py similarity index 100% rename from airflow/providers/singularity/operators/singularity.py rename to providers/src/airflow/providers/singularity/operators/singularity.py diff --git a/airflow/providers/singularity/provider.yaml b/providers/src/airflow/providers/singularity/provider.yaml similarity index 100% rename from airflow/providers/singularity/provider.yaml rename to providers/src/airflow/providers/singularity/provider.yaml diff --git a/airflow/providers/slack/.latest-doc-only-change.txt b/providers/src/airflow/providers/slack/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/slack/.latest-doc-only-change.txt rename to providers/src/airflow/providers/slack/.latest-doc-only-change.txt diff --git a/airflow/providers/slack/CHANGELOG.rst b/providers/src/airflow/providers/slack/CHANGELOG.rst similarity index 100% rename from airflow/providers/slack/CHANGELOG.rst rename to providers/src/airflow/providers/slack/CHANGELOG.rst diff --git a/airflow/providers/slack/__init__.py b/providers/src/airflow/providers/slack/__init__.py similarity index 100% rename from airflow/providers/slack/__init__.py rename to providers/src/airflow/providers/slack/__init__.py diff --git a/airflow/providers/segment/operators/__init__.py b/providers/src/airflow/providers/slack/hooks/__init__.py similarity index 100% rename from airflow/providers/segment/operators/__init__.py rename to providers/src/airflow/providers/slack/hooks/__init__.py diff --git a/airflow/providers/slack/hooks/slack.py b/providers/src/airflow/providers/slack/hooks/slack.py similarity index 100% rename from airflow/providers/slack/hooks/slack.py rename to providers/src/airflow/providers/slack/hooks/slack.py diff --git a/airflow/providers/slack/hooks/slack_webhook.py b/providers/src/airflow/providers/slack/hooks/slack_webhook.py similarity index 100% rename from airflow/providers/slack/hooks/slack_webhook.py rename to providers/src/airflow/providers/slack/hooks/slack_webhook.py diff --git a/airflow/providers/slack/notifications/__init__.py b/providers/src/airflow/providers/slack/notifications/__init__.py similarity index 100% rename from airflow/providers/slack/notifications/__init__.py rename to providers/src/airflow/providers/slack/notifications/__init__.py diff --git a/airflow/providers/slack/notifications/slack.py b/providers/src/airflow/providers/slack/notifications/slack.py similarity index 100% rename from airflow/providers/slack/notifications/slack.py rename to providers/src/airflow/providers/slack/notifications/slack.py diff --git a/airflow/providers/slack/notifications/slack_notifier.py b/providers/src/airflow/providers/slack/notifications/slack_notifier.py similarity index 100% rename from airflow/providers/slack/notifications/slack_notifier.py rename to providers/src/airflow/providers/slack/notifications/slack_notifier.py diff --git a/airflow/providers/slack/notifications/slack_webhook.py b/providers/src/airflow/providers/slack/notifications/slack_webhook.py similarity index 100% rename from airflow/providers/slack/notifications/slack_webhook.py rename to providers/src/airflow/providers/slack/notifications/slack_webhook.py diff --git a/airflow/providers/singularity/operators/__init__.py b/providers/src/airflow/providers/slack/operators/__init__.py similarity index 100% rename from airflow/providers/singularity/operators/__init__.py rename to providers/src/airflow/providers/slack/operators/__init__.py diff --git a/airflow/providers/slack/operators/slack.py b/providers/src/airflow/providers/slack/operators/slack.py similarity index 100% rename from airflow/providers/slack/operators/slack.py rename to providers/src/airflow/providers/slack/operators/slack.py diff --git a/airflow/providers/slack/operators/slack_webhook.py b/providers/src/airflow/providers/slack/operators/slack_webhook.py similarity index 100% rename from airflow/providers/slack/operators/slack_webhook.py rename to providers/src/airflow/providers/slack/operators/slack_webhook.py diff --git a/airflow/providers/slack/provider.yaml b/providers/src/airflow/providers/slack/provider.yaml similarity index 100% rename from airflow/providers/slack/provider.yaml rename to providers/src/airflow/providers/slack/provider.yaml diff --git a/airflow/providers/slack/transfers/__init__.py b/providers/src/airflow/providers/slack/transfers/__init__.py similarity index 100% rename from airflow/providers/slack/transfers/__init__.py rename to providers/src/airflow/providers/slack/transfers/__init__.py diff --git a/airflow/providers/slack/transfers/base_sql_to_slack.py b/providers/src/airflow/providers/slack/transfers/base_sql_to_slack.py similarity index 100% rename from airflow/providers/slack/transfers/base_sql_to_slack.py rename to providers/src/airflow/providers/slack/transfers/base_sql_to_slack.py diff --git a/airflow/providers/slack/transfers/sql_to_slack.py b/providers/src/airflow/providers/slack/transfers/sql_to_slack.py similarity index 100% rename from airflow/providers/slack/transfers/sql_to_slack.py rename to providers/src/airflow/providers/slack/transfers/sql_to_slack.py diff --git a/airflow/providers/slack/transfers/sql_to_slack_webhook.py b/providers/src/airflow/providers/slack/transfers/sql_to_slack_webhook.py similarity index 100% rename from airflow/providers/slack/transfers/sql_to_slack_webhook.py rename to providers/src/airflow/providers/slack/transfers/sql_to_slack_webhook.py diff --git a/airflow/providers/slack/utils/__init__.py b/providers/src/airflow/providers/slack/utils/__init__.py similarity index 100% rename from airflow/providers/slack/utils/__init__.py rename to providers/src/airflow/providers/slack/utils/__init__.py diff --git a/airflow/providers/smtp/.latest-doc-only-change.txt b/providers/src/airflow/providers/smtp/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/smtp/.latest-doc-only-change.txt rename to providers/src/airflow/providers/smtp/.latest-doc-only-change.txt diff --git a/airflow/providers/smtp/CHANGELOG.rst b/providers/src/airflow/providers/smtp/CHANGELOG.rst similarity index 100% rename from airflow/providers/smtp/CHANGELOG.rst rename to providers/src/airflow/providers/smtp/CHANGELOG.rst diff --git a/airflow/providers/smtp/__init__.py b/providers/src/airflow/providers/smtp/__init__.py similarity index 100% rename from airflow/providers/smtp/__init__.py rename to providers/src/airflow/providers/smtp/__init__.py diff --git a/airflow/providers/slack/hooks/__init__.py b/providers/src/airflow/providers/smtp/hooks/__init__.py similarity index 100% rename from airflow/providers/slack/hooks/__init__.py rename to providers/src/airflow/providers/smtp/hooks/__init__.py diff --git a/airflow/providers/smtp/hooks/smtp.py b/providers/src/airflow/providers/smtp/hooks/smtp.py similarity index 100% rename from airflow/providers/smtp/hooks/smtp.py rename to providers/src/airflow/providers/smtp/hooks/smtp.py diff --git a/airflow/providers/smtp/notifications/__init__.py b/providers/src/airflow/providers/smtp/notifications/__init__.py similarity index 100% rename from airflow/providers/smtp/notifications/__init__.py rename to providers/src/airflow/providers/smtp/notifications/__init__.py diff --git a/airflow/providers/smtp/notifications/smtp.py b/providers/src/airflow/providers/smtp/notifications/smtp.py similarity index 100% rename from airflow/providers/smtp/notifications/smtp.py rename to providers/src/airflow/providers/smtp/notifications/smtp.py diff --git a/airflow/providers/smtp/notifications/templates/__init__.py b/providers/src/airflow/providers/smtp/notifications/templates/__init__.py similarity index 100% rename from airflow/providers/smtp/notifications/templates/__init__.py rename to providers/src/airflow/providers/smtp/notifications/templates/__init__.py diff --git a/airflow/providers/smtp/notifications/templates/email.html b/providers/src/airflow/providers/smtp/notifications/templates/email.html similarity index 100% rename from airflow/providers/smtp/notifications/templates/email.html rename to providers/src/airflow/providers/smtp/notifications/templates/email.html diff --git a/airflow/providers/smtp/notifications/templates/email_subject.jinja2 b/providers/src/airflow/providers/smtp/notifications/templates/email_subject.jinja2 similarity index 100% rename from airflow/providers/smtp/notifications/templates/email_subject.jinja2 rename to providers/src/airflow/providers/smtp/notifications/templates/email_subject.jinja2 diff --git a/airflow/providers/slack/operators/__init__.py b/providers/src/airflow/providers/smtp/operators/__init__.py similarity index 100% rename from airflow/providers/slack/operators/__init__.py rename to providers/src/airflow/providers/smtp/operators/__init__.py diff --git a/airflow/providers/smtp/operators/smtp.py b/providers/src/airflow/providers/smtp/operators/smtp.py similarity index 100% rename from airflow/providers/smtp/operators/smtp.py rename to providers/src/airflow/providers/smtp/operators/smtp.py diff --git a/airflow/providers/smtp/provider.yaml b/providers/src/airflow/providers/smtp/provider.yaml similarity index 100% rename from airflow/providers/smtp/provider.yaml rename to providers/src/airflow/providers/smtp/provider.yaml diff --git a/airflow/providers/snowflake/.latest-doc-only-change.txt b/providers/src/airflow/providers/snowflake/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/snowflake/.latest-doc-only-change.txt rename to providers/src/airflow/providers/snowflake/.latest-doc-only-change.txt diff --git a/airflow/providers/snowflake/CHANGELOG.rst b/providers/src/airflow/providers/snowflake/CHANGELOG.rst similarity index 100% rename from airflow/providers/snowflake/CHANGELOG.rst rename to providers/src/airflow/providers/snowflake/CHANGELOG.rst diff --git a/airflow/providers/snowflake/__init__.py b/providers/src/airflow/providers/snowflake/__init__.py similarity index 100% rename from airflow/providers/snowflake/__init__.py rename to providers/src/airflow/providers/snowflake/__init__.py diff --git a/airflow/providers/snowflake/decorators/__init__.py b/providers/src/airflow/providers/snowflake/decorators/__init__.py similarity index 100% rename from airflow/providers/snowflake/decorators/__init__.py rename to providers/src/airflow/providers/snowflake/decorators/__init__.py diff --git a/airflow/providers/snowflake/decorators/snowpark.py b/providers/src/airflow/providers/snowflake/decorators/snowpark.py similarity index 100% rename from airflow/providers/snowflake/decorators/snowpark.py rename to providers/src/airflow/providers/snowflake/decorators/snowpark.py diff --git a/airflow/providers/snowflake/hooks/__init__.py b/providers/src/airflow/providers/snowflake/hooks/__init__.py similarity index 100% rename from airflow/providers/snowflake/hooks/__init__.py rename to providers/src/airflow/providers/snowflake/hooks/__init__.py diff --git a/airflow/providers/snowflake/hooks/snowflake.py b/providers/src/airflow/providers/snowflake/hooks/snowflake.py similarity index 100% rename from airflow/providers/snowflake/hooks/snowflake.py rename to providers/src/airflow/providers/snowflake/hooks/snowflake.py diff --git a/airflow/providers/snowflake/hooks/snowflake_sql_api.py b/providers/src/airflow/providers/snowflake/hooks/snowflake_sql_api.py similarity index 99% rename from airflow/providers/snowflake/hooks/snowflake_sql_api.py rename to providers/src/airflow/providers/snowflake/hooks/snowflake_sql_api.py index 5143f573c2637..8770492d06ec1 100644 --- a/airflow/providers/snowflake/hooks/snowflake_sql_api.py +++ b/providers/src/airflow/providers/snowflake/hooks/snowflake_sql_api.py @@ -327,9 +327,10 @@ async def get_sql_api_query_status_async(self, query_id: str) -> dict[str, str | """ self.log.info("Retrieving status for query id %s", query_id) header, params, url = self.get_request_url_header_params(query_id) - async with aiohttp.ClientSession(headers=header) as session, session.get( - url, params=params - ) as response: + async with ( + aiohttp.ClientSession(headers=header) as session, + session.get(url, params=params) as response, + ): status_code = response.status resp = await response.json() return self._process_response(status_code, resp) diff --git a/airflow/providers/snowflake/operators/__init__.py b/providers/src/airflow/providers/snowflake/operators/__init__.py similarity index 100% rename from airflow/providers/snowflake/operators/__init__.py rename to providers/src/airflow/providers/snowflake/operators/__init__.py diff --git a/airflow/providers/snowflake/operators/snowflake.py b/providers/src/airflow/providers/snowflake/operators/snowflake.py similarity index 100% rename from airflow/providers/snowflake/operators/snowflake.py rename to providers/src/airflow/providers/snowflake/operators/snowflake.py diff --git a/airflow/providers/snowflake/operators/snowpark.py b/providers/src/airflow/providers/snowflake/operators/snowpark.py similarity index 100% rename from airflow/providers/snowflake/operators/snowpark.py rename to providers/src/airflow/providers/snowflake/operators/snowpark.py diff --git a/airflow/providers/snowflake/provider.yaml b/providers/src/airflow/providers/snowflake/provider.yaml similarity index 100% rename from airflow/providers/snowflake/provider.yaml rename to providers/src/airflow/providers/snowflake/provider.yaml diff --git a/airflow/providers/snowflake/transfers/__init__.py b/providers/src/airflow/providers/snowflake/transfers/__init__.py similarity index 100% rename from airflow/providers/snowflake/transfers/__init__.py rename to providers/src/airflow/providers/snowflake/transfers/__init__.py diff --git a/airflow/providers/snowflake/transfers/copy_into_snowflake.py b/providers/src/airflow/providers/snowflake/transfers/copy_into_snowflake.py similarity index 100% rename from airflow/providers/snowflake/transfers/copy_into_snowflake.py rename to providers/src/airflow/providers/snowflake/transfers/copy_into_snowflake.py diff --git a/airflow/providers/snowflake/triggers/__init__.py b/providers/src/airflow/providers/snowflake/triggers/__init__.py similarity index 100% rename from airflow/providers/snowflake/triggers/__init__.py rename to providers/src/airflow/providers/snowflake/triggers/__init__.py diff --git a/airflow/providers/snowflake/triggers/snowflake_trigger.py b/providers/src/airflow/providers/snowflake/triggers/snowflake_trigger.py similarity index 100% rename from airflow/providers/snowflake/triggers/snowflake_trigger.py rename to providers/src/airflow/providers/snowflake/triggers/snowflake_trigger.py diff --git a/airflow/providers/snowflake/utils/__init__.py b/providers/src/airflow/providers/snowflake/utils/__init__.py similarity index 100% rename from airflow/providers/snowflake/utils/__init__.py rename to providers/src/airflow/providers/snowflake/utils/__init__.py diff --git a/airflow/providers/snowflake/utils/common.py b/providers/src/airflow/providers/snowflake/utils/common.py similarity index 100% rename from airflow/providers/snowflake/utils/common.py rename to providers/src/airflow/providers/snowflake/utils/common.py diff --git a/airflow/providers/snowflake/utils/openlineage.py b/providers/src/airflow/providers/snowflake/utils/openlineage.py similarity index 100% rename from airflow/providers/snowflake/utils/openlineage.py rename to providers/src/airflow/providers/snowflake/utils/openlineage.py diff --git a/airflow/providers/snowflake/utils/snowpark.py b/providers/src/airflow/providers/snowflake/utils/snowpark.py similarity index 100% rename from airflow/providers/snowflake/utils/snowpark.py rename to providers/src/airflow/providers/snowflake/utils/snowpark.py diff --git a/airflow/providers/snowflake/utils/sql_api_generate_jwt.py b/providers/src/airflow/providers/snowflake/utils/sql_api_generate_jwt.py similarity index 100% rename from airflow/providers/snowflake/utils/sql_api_generate_jwt.py rename to providers/src/airflow/providers/snowflake/utils/sql_api_generate_jwt.py diff --git a/airflow/providers/sqlite/.latest-doc-only-change.txt b/providers/src/airflow/providers/sqlite/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/sqlite/.latest-doc-only-change.txt rename to providers/src/airflow/providers/sqlite/.latest-doc-only-change.txt diff --git a/airflow/providers/sqlite/CHANGELOG.rst b/providers/src/airflow/providers/sqlite/CHANGELOG.rst similarity index 100% rename from airflow/providers/sqlite/CHANGELOG.rst rename to providers/src/airflow/providers/sqlite/CHANGELOG.rst diff --git a/airflow/providers/sqlite/__init__.py b/providers/src/airflow/providers/sqlite/__init__.py similarity index 100% rename from airflow/providers/sqlite/__init__.py rename to providers/src/airflow/providers/sqlite/__init__.py diff --git a/airflow/providers/smtp/hooks/__init__.py b/providers/src/airflow/providers/sqlite/hooks/__init__.py similarity index 100% rename from airflow/providers/smtp/hooks/__init__.py rename to providers/src/airflow/providers/sqlite/hooks/__init__.py diff --git a/airflow/providers/sqlite/hooks/sqlite.py b/providers/src/airflow/providers/sqlite/hooks/sqlite.py similarity index 100% rename from airflow/providers/sqlite/hooks/sqlite.py rename to providers/src/airflow/providers/sqlite/hooks/sqlite.py diff --git a/airflow/providers/smtp/operators/__init__.py b/providers/src/airflow/providers/sqlite/operators/__init__.py similarity index 100% rename from airflow/providers/smtp/operators/__init__.py rename to providers/src/airflow/providers/sqlite/operators/__init__.py diff --git a/airflow/providers/sqlite/operators/sqlite.py b/providers/src/airflow/providers/sqlite/operators/sqlite.py similarity index 100% rename from airflow/providers/sqlite/operators/sqlite.py rename to providers/src/airflow/providers/sqlite/operators/sqlite.py diff --git a/airflow/providers/sqlite/provider.yaml b/providers/src/airflow/providers/sqlite/provider.yaml similarity index 100% rename from airflow/providers/sqlite/provider.yaml rename to providers/src/airflow/providers/sqlite/provider.yaml diff --git a/airflow/providers/ssh/.latest-doc-only-change.txt b/providers/src/airflow/providers/ssh/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/ssh/.latest-doc-only-change.txt rename to providers/src/airflow/providers/ssh/.latest-doc-only-change.txt diff --git a/airflow/providers/ssh/CHANGELOG.rst b/providers/src/airflow/providers/ssh/CHANGELOG.rst similarity index 100% rename from airflow/providers/ssh/CHANGELOG.rst rename to providers/src/airflow/providers/ssh/CHANGELOG.rst diff --git a/airflow/providers/ssh/__init__.py b/providers/src/airflow/providers/ssh/__init__.py similarity index 100% rename from airflow/providers/ssh/__init__.py rename to providers/src/airflow/providers/ssh/__init__.py diff --git a/airflow/providers/sqlite/hooks/__init__.py b/providers/src/airflow/providers/ssh/hooks/__init__.py similarity index 100% rename from airflow/providers/sqlite/hooks/__init__.py rename to providers/src/airflow/providers/ssh/hooks/__init__.py diff --git a/airflow/providers/ssh/hooks/ssh.py b/providers/src/airflow/providers/ssh/hooks/ssh.py similarity index 100% rename from airflow/providers/ssh/hooks/ssh.py rename to providers/src/airflow/providers/ssh/hooks/ssh.py diff --git a/airflow/providers/sqlite/operators/__init__.py b/providers/src/airflow/providers/ssh/operators/__init__.py similarity index 100% rename from airflow/providers/sqlite/operators/__init__.py rename to providers/src/airflow/providers/ssh/operators/__init__.py diff --git a/airflow/providers/ssh/operators/ssh.py b/providers/src/airflow/providers/ssh/operators/ssh.py similarity index 100% rename from airflow/providers/ssh/operators/ssh.py rename to providers/src/airflow/providers/ssh/operators/ssh.py diff --git a/airflow/providers/ssh/provider.yaml b/providers/src/airflow/providers/ssh/provider.yaml similarity index 100% rename from airflow/providers/ssh/provider.yaml rename to providers/src/airflow/providers/ssh/provider.yaml diff --git a/airflow/providers/standard/CHANGELOG.rst b/providers/src/airflow/providers/standard/CHANGELOG.rst similarity index 100% rename from airflow/providers/standard/CHANGELOG.rst rename to providers/src/airflow/providers/standard/CHANGELOG.rst diff --git a/airflow/providers/ssh/hooks/__init__.py b/providers/src/airflow/providers/standard/__init__.py similarity index 100% rename from airflow/providers/ssh/hooks/__init__.py rename to providers/src/airflow/providers/standard/__init__.py diff --git a/airflow/providers/standard/operators/__init__.py b/providers/src/airflow/providers/standard/operators/__init__.py similarity index 100% rename from airflow/providers/standard/operators/__init__.py rename to providers/src/airflow/providers/standard/operators/__init__.py diff --git a/airflow/providers/standard/operators/bash.py b/providers/src/airflow/providers/standard/operators/bash.py similarity index 100% rename from airflow/providers/standard/operators/bash.py rename to providers/src/airflow/providers/standard/operators/bash.py diff --git a/airflow/providers/standard/operators/datetime.py b/providers/src/airflow/providers/standard/operators/datetime.py similarity index 100% rename from airflow/providers/standard/operators/datetime.py rename to providers/src/airflow/providers/standard/operators/datetime.py diff --git a/airflow/providers/standard/operators/weekday.py b/providers/src/airflow/providers/standard/operators/weekday.py similarity index 100% rename from airflow/providers/standard/operators/weekday.py rename to providers/src/airflow/providers/standard/operators/weekday.py diff --git a/airflow/providers/standard/provider.yaml b/providers/src/airflow/providers/standard/provider.yaml similarity index 100% rename from airflow/providers/standard/provider.yaml rename to providers/src/airflow/providers/standard/provider.yaml diff --git a/airflow/providers/standard/sensors/__init__.py b/providers/src/airflow/providers/standard/sensors/__init__.py similarity index 100% rename from airflow/providers/standard/sensors/__init__.py rename to providers/src/airflow/providers/standard/sensors/__init__.py diff --git a/airflow/providers/standard/sensors/bash.py b/providers/src/airflow/providers/standard/sensors/bash.py similarity index 100% rename from airflow/providers/standard/sensors/bash.py rename to providers/src/airflow/providers/standard/sensors/bash.py diff --git a/airflow/providers/standard/sensors/date_time.py b/providers/src/airflow/providers/standard/sensors/date_time.py similarity index 100% rename from airflow/providers/standard/sensors/date_time.py rename to providers/src/airflow/providers/standard/sensors/date_time.py diff --git a/airflow/providers/standard/sensors/time.py b/providers/src/airflow/providers/standard/sensors/time.py similarity index 100% rename from airflow/providers/standard/sensors/time.py rename to providers/src/airflow/providers/standard/sensors/time.py diff --git a/airflow/providers/standard/sensors/time_delta.py b/providers/src/airflow/providers/standard/sensors/time_delta.py similarity index 100% rename from airflow/providers/standard/sensors/time_delta.py rename to providers/src/airflow/providers/standard/sensors/time_delta.py diff --git a/airflow/providers/standard/sensors/weekday.py b/providers/src/airflow/providers/standard/sensors/weekday.py similarity index 100% rename from airflow/providers/standard/sensors/weekday.py rename to providers/src/airflow/providers/standard/sensors/weekday.py diff --git a/airflow/providers/tableau/.latest-doc-only-change.txt b/providers/src/airflow/providers/tableau/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/tableau/.latest-doc-only-change.txt rename to providers/src/airflow/providers/tableau/.latest-doc-only-change.txt diff --git a/airflow/providers/tableau/CHANGELOG.rst b/providers/src/airflow/providers/tableau/CHANGELOG.rst similarity index 100% rename from airflow/providers/tableau/CHANGELOG.rst rename to providers/src/airflow/providers/tableau/CHANGELOG.rst diff --git a/airflow/providers/tableau/__init__.py b/providers/src/airflow/providers/tableau/__init__.py similarity index 100% rename from airflow/providers/tableau/__init__.py rename to providers/src/airflow/providers/tableau/__init__.py diff --git a/airflow/providers/ssh/operators/__init__.py b/providers/src/airflow/providers/tableau/hooks/__init__.py similarity index 100% rename from airflow/providers/ssh/operators/__init__.py rename to providers/src/airflow/providers/tableau/hooks/__init__.py diff --git a/airflow/providers/tableau/hooks/tableau.py b/providers/src/airflow/providers/tableau/hooks/tableau.py similarity index 100% rename from airflow/providers/tableau/hooks/tableau.py rename to providers/src/airflow/providers/tableau/hooks/tableau.py diff --git a/airflow/providers/tableau/operators/__init__.py b/providers/src/airflow/providers/tableau/operators/__init__.py similarity index 100% rename from airflow/providers/tableau/operators/__init__.py rename to providers/src/airflow/providers/tableau/operators/__init__.py diff --git a/airflow/providers/tableau/operators/tableau.py b/providers/src/airflow/providers/tableau/operators/tableau.py similarity index 100% rename from airflow/providers/tableau/operators/tableau.py rename to providers/src/airflow/providers/tableau/operators/tableau.py diff --git a/airflow/providers/tableau/provider.yaml b/providers/src/airflow/providers/tableau/provider.yaml similarity index 100% rename from airflow/providers/tableau/provider.yaml rename to providers/src/airflow/providers/tableau/provider.yaml diff --git a/airflow/providers/tableau/sensors/__init__.py b/providers/src/airflow/providers/tableau/sensors/__init__.py similarity index 100% rename from airflow/providers/tableau/sensors/__init__.py rename to providers/src/airflow/providers/tableau/sensors/__init__.py diff --git a/airflow/providers/tableau/sensors/tableau.py b/providers/src/airflow/providers/tableau/sensors/tableau.py similarity index 100% rename from airflow/providers/tableau/sensors/tableau.py rename to providers/src/airflow/providers/tableau/sensors/tableau.py diff --git a/airflow/providers/telegram/.latest-doc-only-change.txt b/providers/src/airflow/providers/telegram/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/telegram/.latest-doc-only-change.txt rename to providers/src/airflow/providers/telegram/.latest-doc-only-change.txt diff --git a/airflow/providers/telegram/CHANGELOG.rst b/providers/src/airflow/providers/telegram/CHANGELOG.rst similarity index 100% rename from airflow/providers/telegram/CHANGELOG.rst rename to providers/src/airflow/providers/telegram/CHANGELOG.rst diff --git a/airflow/providers/telegram/__init__.py b/providers/src/airflow/providers/telegram/__init__.py similarity index 100% rename from airflow/providers/telegram/__init__.py rename to providers/src/airflow/providers/telegram/__init__.py diff --git a/airflow/providers/standard/__init__.py b/providers/src/airflow/providers/telegram/hooks/__init__.py similarity index 100% rename from airflow/providers/standard/__init__.py rename to providers/src/airflow/providers/telegram/hooks/__init__.py diff --git a/airflow/providers/telegram/hooks/telegram.py b/providers/src/airflow/providers/telegram/hooks/telegram.py similarity index 100% rename from airflow/providers/telegram/hooks/telegram.py rename to providers/src/airflow/providers/telegram/hooks/telegram.py diff --git a/airflow/providers/tableau/hooks/__init__.py b/providers/src/airflow/providers/telegram/operators/__init__.py similarity index 100% rename from airflow/providers/tableau/hooks/__init__.py rename to providers/src/airflow/providers/telegram/operators/__init__.py diff --git a/airflow/providers/telegram/operators/telegram.py b/providers/src/airflow/providers/telegram/operators/telegram.py similarity index 100% rename from airflow/providers/telegram/operators/telegram.py rename to providers/src/airflow/providers/telegram/operators/telegram.py diff --git a/airflow/providers/telegram/provider.yaml b/providers/src/airflow/providers/telegram/provider.yaml similarity index 100% rename from airflow/providers/telegram/provider.yaml rename to providers/src/airflow/providers/telegram/provider.yaml diff --git a/airflow/providers/teradata/.latest-doc-only-change.txt b/providers/src/airflow/providers/teradata/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/teradata/.latest-doc-only-change.txt rename to providers/src/airflow/providers/teradata/.latest-doc-only-change.txt diff --git a/airflow/providers/teradata/CHANGELOG.rst b/providers/src/airflow/providers/teradata/CHANGELOG.rst similarity index 100% rename from airflow/providers/teradata/CHANGELOG.rst rename to providers/src/airflow/providers/teradata/CHANGELOG.rst diff --git a/airflow/providers/teradata/__init__.py b/providers/src/airflow/providers/teradata/__init__.py similarity index 100% rename from airflow/providers/teradata/__init__.py rename to providers/src/airflow/providers/teradata/__init__.py diff --git a/airflow/providers/telegram/hooks/__init__.py b/providers/src/airflow/providers/teradata/hooks/__init__.py similarity index 100% rename from airflow/providers/telegram/hooks/__init__.py rename to providers/src/airflow/providers/teradata/hooks/__init__.py diff --git a/airflow/providers/teradata/hooks/teradata.py b/providers/src/airflow/providers/teradata/hooks/teradata.py similarity index 100% rename from airflow/providers/teradata/hooks/teradata.py rename to providers/src/airflow/providers/teradata/hooks/teradata.py diff --git a/airflow/providers/telegram/operators/__init__.py b/providers/src/airflow/providers/teradata/operators/__init__.py similarity index 100% rename from airflow/providers/telegram/operators/__init__.py rename to providers/src/airflow/providers/teradata/operators/__init__.py diff --git a/airflow/providers/teradata/operators/teradata.py b/providers/src/airflow/providers/teradata/operators/teradata.py similarity index 100% rename from airflow/providers/teradata/operators/teradata.py rename to providers/src/airflow/providers/teradata/operators/teradata.py diff --git a/airflow/providers/teradata/operators/teradata_compute_cluster.py b/providers/src/airflow/providers/teradata/operators/teradata_compute_cluster.py similarity index 100% rename from airflow/providers/teradata/operators/teradata_compute_cluster.py rename to providers/src/airflow/providers/teradata/operators/teradata_compute_cluster.py diff --git a/airflow/providers/teradata/provider.yaml b/providers/src/airflow/providers/teradata/provider.yaml similarity index 100% rename from airflow/providers/teradata/provider.yaml rename to providers/src/airflow/providers/teradata/provider.yaml diff --git a/airflow/providers/teradata/transfers/__init__.py b/providers/src/airflow/providers/teradata/transfers/__init__.py similarity index 100% rename from airflow/providers/teradata/transfers/__init__.py rename to providers/src/airflow/providers/teradata/transfers/__init__.py diff --git a/airflow/providers/teradata/transfers/azure_blob_to_teradata.py b/providers/src/airflow/providers/teradata/transfers/azure_blob_to_teradata.py similarity index 100% rename from airflow/providers/teradata/transfers/azure_blob_to_teradata.py rename to providers/src/airflow/providers/teradata/transfers/azure_blob_to_teradata.py diff --git a/airflow/providers/teradata/transfers/s3_to_teradata.py b/providers/src/airflow/providers/teradata/transfers/s3_to_teradata.py similarity index 100% rename from airflow/providers/teradata/transfers/s3_to_teradata.py rename to providers/src/airflow/providers/teradata/transfers/s3_to_teradata.py diff --git a/airflow/providers/teradata/transfers/teradata_to_teradata.py b/providers/src/airflow/providers/teradata/transfers/teradata_to_teradata.py similarity index 100% rename from airflow/providers/teradata/transfers/teradata_to_teradata.py rename to providers/src/airflow/providers/teradata/transfers/teradata_to_teradata.py diff --git a/airflow/providers/teradata/triggers/__init__.py b/providers/src/airflow/providers/teradata/triggers/__init__.py similarity index 100% rename from airflow/providers/teradata/triggers/__init__.py rename to providers/src/airflow/providers/teradata/triggers/__init__.py diff --git a/airflow/providers/teradata/triggers/teradata_compute_cluster.py b/providers/src/airflow/providers/teradata/triggers/teradata_compute_cluster.py similarity index 100% rename from airflow/providers/teradata/triggers/teradata_compute_cluster.py rename to providers/src/airflow/providers/teradata/triggers/teradata_compute_cluster.py diff --git a/airflow/providers/teradata/utils/__init__.py b/providers/src/airflow/providers/teradata/utils/__init__.py similarity index 100% rename from airflow/providers/teradata/utils/__init__.py rename to providers/src/airflow/providers/teradata/utils/__init__.py diff --git a/airflow/providers/teradata/utils/constants.py b/providers/src/airflow/providers/teradata/utils/constants.py similarity index 100% rename from airflow/providers/teradata/utils/constants.py rename to providers/src/airflow/providers/teradata/utils/constants.py diff --git a/airflow/providers/trino/.latest-doc-only-change.txt b/providers/src/airflow/providers/trino/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/trino/.latest-doc-only-change.txt rename to providers/src/airflow/providers/trino/.latest-doc-only-change.txt diff --git a/airflow/providers/trino/CHANGELOG.rst b/providers/src/airflow/providers/trino/CHANGELOG.rst similarity index 100% rename from airflow/providers/trino/CHANGELOG.rst rename to providers/src/airflow/providers/trino/CHANGELOG.rst diff --git a/airflow/providers/trino/__init__.py b/providers/src/airflow/providers/trino/__init__.py similarity index 100% rename from airflow/providers/trino/__init__.py rename to providers/src/airflow/providers/trino/__init__.py diff --git a/airflow/providers/trino/assets/__init__.py b/providers/src/airflow/providers/trino/assets/__init__.py similarity index 100% rename from airflow/providers/trino/assets/__init__.py rename to providers/src/airflow/providers/trino/assets/__init__.py diff --git a/airflow/providers/trino/assets/trino.py b/providers/src/airflow/providers/trino/assets/trino.py similarity index 100% rename from airflow/providers/trino/assets/trino.py rename to providers/src/airflow/providers/trino/assets/trino.py diff --git a/airflow/providers/teradata/hooks/__init__.py b/providers/src/airflow/providers/trino/hooks/__init__.py similarity index 100% rename from airflow/providers/teradata/hooks/__init__.py rename to providers/src/airflow/providers/trino/hooks/__init__.py diff --git a/airflow/providers/trino/hooks/trino.py b/providers/src/airflow/providers/trino/hooks/trino.py similarity index 100% rename from airflow/providers/trino/hooks/trino.py rename to providers/src/airflow/providers/trino/hooks/trino.py diff --git a/airflow/providers/trino/operators/__init__.py b/providers/src/airflow/providers/trino/operators/__init__.py similarity index 100% rename from airflow/providers/trino/operators/__init__.py rename to providers/src/airflow/providers/trino/operators/__init__.py diff --git a/airflow/providers/trino/operators/trino.py b/providers/src/airflow/providers/trino/operators/trino.py similarity index 100% rename from airflow/providers/trino/operators/trino.py rename to providers/src/airflow/providers/trino/operators/trino.py diff --git a/airflow/providers/trino/provider.yaml b/providers/src/airflow/providers/trino/provider.yaml similarity index 100% rename from airflow/providers/trino/provider.yaml rename to providers/src/airflow/providers/trino/provider.yaml diff --git a/airflow/providers/teradata/operators/__init__.py b/providers/src/airflow/providers/trino/transfers/__init__.py similarity index 100% rename from airflow/providers/teradata/operators/__init__.py rename to providers/src/airflow/providers/trino/transfers/__init__.py diff --git a/airflow/providers/trino/transfers/gcs_to_trino.py b/providers/src/airflow/providers/trino/transfers/gcs_to_trino.py similarity index 100% rename from airflow/providers/trino/transfers/gcs_to_trino.py rename to providers/src/airflow/providers/trino/transfers/gcs_to_trino.py diff --git a/airflow/providers/vertica/.latest-doc-only-change.txt b/providers/src/airflow/providers/vertica/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/vertica/.latest-doc-only-change.txt rename to providers/src/airflow/providers/vertica/.latest-doc-only-change.txt diff --git a/airflow/providers/vertica/CHANGELOG.rst b/providers/src/airflow/providers/vertica/CHANGELOG.rst similarity index 100% rename from airflow/providers/vertica/CHANGELOG.rst rename to providers/src/airflow/providers/vertica/CHANGELOG.rst diff --git a/airflow/providers/vertica/__init__.py b/providers/src/airflow/providers/vertica/__init__.py similarity index 100% rename from airflow/providers/vertica/__init__.py rename to providers/src/airflow/providers/vertica/__init__.py diff --git a/airflow/providers/trino/hooks/__init__.py b/providers/src/airflow/providers/vertica/hooks/__init__.py similarity index 100% rename from airflow/providers/trino/hooks/__init__.py rename to providers/src/airflow/providers/vertica/hooks/__init__.py diff --git a/airflow/providers/vertica/hooks/vertica.py b/providers/src/airflow/providers/vertica/hooks/vertica.py similarity index 100% rename from airflow/providers/vertica/hooks/vertica.py rename to providers/src/airflow/providers/vertica/hooks/vertica.py diff --git a/airflow/providers/trino/transfers/__init__.py b/providers/src/airflow/providers/vertica/operators/__init__.py similarity index 100% rename from airflow/providers/trino/transfers/__init__.py rename to providers/src/airflow/providers/vertica/operators/__init__.py diff --git a/airflow/providers/vertica/operators/vertica.py b/providers/src/airflow/providers/vertica/operators/vertica.py similarity index 100% rename from airflow/providers/vertica/operators/vertica.py rename to providers/src/airflow/providers/vertica/operators/vertica.py diff --git a/airflow/providers/vertica/provider.yaml b/providers/src/airflow/providers/vertica/provider.yaml similarity index 100% rename from airflow/providers/vertica/provider.yaml rename to providers/src/airflow/providers/vertica/provider.yaml diff --git a/airflow/providers/weaviate/.latest-doc-only-change.txt b/providers/src/airflow/providers/weaviate/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/weaviate/.latest-doc-only-change.txt rename to providers/src/airflow/providers/weaviate/.latest-doc-only-change.txt diff --git a/airflow/providers/weaviate/CHANGELOG.rst b/providers/src/airflow/providers/weaviate/CHANGELOG.rst similarity index 100% rename from airflow/providers/weaviate/CHANGELOG.rst rename to providers/src/airflow/providers/weaviate/CHANGELOG.rst diff --git a/airflow/providers/weaviate/__init__.py b/providers/src/airflow/providers/weaviate/__init__.py similarity index 100% rename from airflow/providers/weaviate/__init__.py rename to providers/src/airflow/providers/weaviate/__init__.py diff --git a/airflow/providers/weaviate/hooks/__init__.py b/providers/src/airflow/providers/weaviate/hooks/__init__.py similarity index 100% rename from airflow/providers/weaviate/hooks/__init__.py rename to providers/src/airflow/providers/weaviate/hooks/__init__.py diff --git a/airflow/providers/weaviate/hooks/weaviate.py b/providers/src/airflow/providers/weaviate/hooks/weaviate.py similarity index 100% rename from airflow/providers/weaviate/hooks/weaviate.py rename to providers/src/airflow/providers/weaviate/hooks/weaviate.py diff --git a/airflow/providers/weaviate/operators/__init__.py b/providers/src/airflow/providers/weaviate/operators/__init__.py similarity index 100% rename from airflow/providers/weaviate/operators/__init__.py rename to providers/src/airflow/providers/weaviate/operators/__init__.py diff --git a/airflow/providers/weaviate/operators/weaviate.py b/providers/src/airflow/providers/weaviate/operators/weaviate.py similarity index 100% rename from airflow/providers/weaviate/operators/weaviate.py rename to providers/src/airflow/providers/weaviate/operators/weaviate.py diff --git a/airflow/providers/weaviate/provider.yaml b/providers/src/airflow/providers/weaviate/provider.yaml similarity index 100% rename from airflow/providers/weaviate/provider.yaml rename to providers/src/airflow/providers/weaviate/provider.yaml diff --git a/airflow/providers/yandex/.latest-doc-only-change.txt b/providers/src/airflow/providers/yandex/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/yandex/.latest-doc-only-change.txt rename to providers/src/airflow/providers/yandex/.latest-doc-only-change.txt diff --git a/airflow/providers/yandex/CHANGELOG.rst b/providers/src/airflow/providers/yandex/CHANGELOG.rst similarity index 100% rename from airflow/providers/yandex/CHANGELOG.rst rename to providers/src/airflow/providers/yandex/CHANGELOG.rst diff --git a/airflow/providers/yandex/__init__.py b/providers/src/airflow/providers/yandex/__init__.py similarity index 100% rename from airflow/providers/yandex/__init__.py rename to providers/src/airflow/providers/yandex/__init__.py diff --git a/airflow/providers/yandex/hooks/__init__.py b/providers/src/airflow/providers/yandex/hooks/__init__.py similarity index 100% rename from airflow/providers/yandex/hooks/__init__.py rename to providers/src/airflow/providers/yandex/hooks/__init__.py diff --git a/airflow/providers/yandex/hooks/dataproc.py b/providers/src/airflow/providers/yandex/hooks/dataproc.py similarity index 100% rename from airflow/providers/yandex/hooks/dataproc.py rename to providers/src/airflow/providers/yandex/hooks/dataproc.py diff --git a/airflow/providers/yandex/hooks/yandex.py b/providers/src/airflow/providers/yandex/hooks/yandex.py similarity index 100% rename from airflow/providers/yandex/hooks/yandex.py rename to providers/src/airflow/providers/yandex/hooks/yandex.py diff --git a/airflow/providers/yandex/hooks/yandexcloud_dataproc.py b/providers/src/airflow/providers/yandex/hooks/yandexcloud_dataproc.py similarity index 100% rename from airflow/providers/yandex/hooks/yandexcloud_dataproc.py rename to providers/src/airflow/providers/yandex/hooks/yandexcloud_dataproc.py diff --git a/airflow/providers/yandex/hooks/yq.py b/providers/src/airflow/providers/yandex/hooks/yq.py similarity index 100% rename from airflow/providers/yandex/hooks/yq.py rename to providers/src/airflow/providers/yandex/hooks/yq.py diff --git a/airflow/providers/yandex/links/__init__.py b/providers/src/airflow/providers/yandex/links/__init__.py similarity index 100% rename from airflow/providers/yandex/links/__init__.py rename to providers/src/airflow/providers/yandex/links/__init__.py diff --git a/airflow/providers/yandex/links/yq.py b/providers/src/airflow/providers/yandex/links/yq.py similarity index 100% rename from airflow/providers/yandex/links/yq.py rename to providers/src/airflow/providers/yandex/links/yq.py diff --git a/airflow/providers/yandex/operators/__init__.py b/providers/src/airflow/providers/yandex/operators/__init__.py similarity index 100% rename from airflow/providers/yandex/operators/__init__.py rename to providers/src/airflow/providers/yandex/operators/__init__.py diff --git a/airflow/providers/yandex/operators/dataproc.py b/providers/src/airflow/providers/yandex/operators/dataproc.py similarity index 100% rename from airflow/providers/yandex/operators/dataproc.py rename to providers/src/airflow/providers/yandex/operators/dataproc.py diff --git a/airflow/providers/yandex/operators/yandexcloud_dataproc.py b/providers/src/airflow/providers/yandex/operators/yandexcloud_dataproc.py similarity index 100% rename from airflow/providers/yandex/operators/yandexcloud_dataproc.py rename to providers/src/airflow/providers/yandex/operators/yandexcloud_dataproc.py diff --git a/airflow/providers/yandex/operators/yq.py b/providers/src/airflow/providers/yandex/operators/yq.py similarity index 100% rename from airflow/providers/yandex/operators/yq.py rename to providers/src/airflow/providers/yandex/operators/yq.py diff --git a/airflow/providers/yandex/provider.yaml b/providers/src/airflow/providers/yandex/provider.yaml similarity index 100% rename from airflow/providers/yandex/provider.yaml rename to providers/src/airflow/providers/yandex/provider.yaml diff --git a/airflow/providers/yandex/secrets/__init__.py b/providers/src/airflow/providers/yandex/secrets/__init__.py similarity index 100% rename from airflow/providers/yandex/secrets/__init__.py rename to providers/src/airflow/providers/yandex/secrets/__init__.py diff --git a/airflow/providers/yandex/secrets/lockbox.py b/providers/src/airflow/providers/yandex/secrets/lockbox.py similarity index 100% rename from airflow/providers/yandex/secrets/lockbox.py rename to providers/src/airflow/providers/yandex/secrets/lockbox.py diff --git a/airflow/providers/yandex/utils/__init__.py b/providers/src/airflow/providers/yandex/utils/__init__.py similarity index 100% rename from airflow/providers/yandex/utils/__init__.py rename to providers/src/airflow/providers/yandex/utils/__init__.py diff --git a/airflow/providers/yandex/utils/credentials.py b/providers/src/airflow/providers/yandex/utils/credentials.py similarity index 100% rename from airflow/providers/yandex/utils/credentials.py rename to providers/src/airflow/providers/yandex/utils/credentials.py diff --git a/airflow/providers/yandex/utils/defaults.py b/providers/src/airflow/providers/yandex/utils/defaults.py similarity index 100% rename from airflow/providers/yandex/utils/defaults.py rename to providers/src/airflow/providers/yandex/utils/defaults.py diff --git a/airflow/providers/yandex/utils/fields.py b/providers/src/airflow/providers/yandex/utils/fields.py similarity index 100% rename from airflow/providers/yandex/utils/fields.py rename to providers/src/airflow/providers/yandex/utils/fields.py diff --git a/airflow/providers/yandex/utils/user_agent.py b/providers/src/airflow/providers/yandex/utils/user_agent.py similarity index 100% rename from airflow/providers/yandex/utils/user_agent.py rename to providers/src/airflow/providers/yandex/utils/user_agent.py diff --git a/airflow/providers/ydb/CHANGELOG.rst b/providers/src/airflow/providers/ydb/CHANGELOG.rst similarity index 100% rename from airflow/providers/ydb/CHANGELOG.rst rename to providers/src/airflow/providers/ydb/CHANGELOG.rst diff --git a/airflow/providers/ydb/__init__.py b/providers/src/airflow/providers/ydb/__init__.py similarity index 100% rename from airflow/providers/ydb/__init__.py rename to providers/src/airflow/providers/ydb/__init__.py diff --git a/airflow/providers/ydb/hooks/__init__.py b/providers/src/airflow/providers/ydb/hooks/__init__.py similarity index 100% rename from airflow/providers/ydb/hooks/__init__.py rename to providers/src/airflow/providers/ydb/hooks/__init__.py diff --git a/airflow/providers/ydb/hooks/_vendor/__init__.py b/providers/src/airflow/providers/ydb/hooks/_vendor/__init__.py similarity index 100% rename from airflow/providers/ydb/hooks/_vendor/__init__.py rename to providers/src/airflow/providers/ydb/hooks/_vendor/__init__.py diff --git a/airflow/providers/ydb/hooks/_vendor/dbapi/__init__.py b/providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/__init__.py similarity index 100% rename from airflow/providers/ydb/hooks/_vendor/dbapi/__init__.py rename to providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/__init__.py diff --git a/airflow/providers/ydb/hooks/_vendor/dbapi/connection.py b/providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/connection.py similarity index 100% rename from airflow/providers/ydb/hooks/_vendor/dbapi/connection.py rename to providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/connection.py diff --git a/airflow/providers/ydb/hooks/_vendor/dbapi/constants.py b/providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/constants.py similarity index 100% rename from airflow/providers/ydb/hooks/_vendor/dbapi/constants.py rename to providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/constants.py diff --git a/airflow/providers/ydb/hooks/_vendor/dbapi/cursor.py b/providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/cursor.py similarity index 100% rename from airflow/providers/ydb/hooks/_vendor/dbapi/cursor.py rename to providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/cursor.py diff --git a/airflow/providers/ydb/hooks/_vendor/dbapi/errors.py b/providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/errors.py similarity index 100% rename from airflow/providers/ydb/hooks/_vendor/dbapi/errors.py rename to providers/src/airflow/providers/ydb/hooks/_vendor/dbapi/errors.py diff --git a/airflow/providers/ydb/hooks/_vendor/readme.md b/providers/src/airflow/providers/ydb/hooks/_vendor/readme.md similarity index 100% rename from airflow/providers/ydb/hooks/_vendor/readme.md rename to providers/src/airflow/providers/ydb/hooks/_vendor/readme.md diff --git a/airflow/providers/ydb/hooks/ydb.py b/providers/src/airflow/providers/ydb/hooks/ydb.py similarity index 100% rename from airflow/providers/ydb/hooks/ydb.py rename to providers/src/airflow/providers/ydb/hooks/ydb.py diff --git a/airflow/providers/ydb/operators/__init__.py b/providers/src/airflow/providers/ydb/operators/__init__.py similarity index 100% rename from airflow/providers/ydb/operators/__init__.py rename to providers/src/airflow/providers/ydb/operators/__init__.py diff --git a/airflow/providers/ydb/operators/ydb.py b/providers/src/airflow/providers/ydb/operators/ydb.py similarity index 100% rename from airflow/providers/ydb/operators/ydb.py rename to providers/src/airflow/providers/ydb/operators/ydb.py diff --git a/airflow/providers/ydb/provider.yaml b/providers/src/airflow/providers/ydb/provider.yaml similarity index 100% rename from airflow/providers/ydb/provider.yaml rename to providers/src/airflow/providers/ydb/provider.yaml diff --git a/airflow/providers/ydb/utils/__init__.py b/providers/src/airflow/providers/ydb/utils/__init__.py similarity index 100% rename from airflow/providers/ydb/utils/__init__.py rename to providers/src/airflow/providers/ydb/utils/__init__.py diff --git a/airflow/providers/ydb/utils/credentials.py b/providers/src/airflow/providers/ydb/utils/credentials.py similarity index 100% rename from airflow/providers/ydb/utils/credentials.py rename to providers/src/airflow/providers/ydb/utils/credentials.py diff --git a/airflow/providers/ydb/utils/defaults.py b/providers/src/airflow/providers/ydb/utils/defaults.py similarity index 100% rename from airflow/providers/ydb/utils/defaults.py rename to providers/src/airflow/providers/ydb/utils/defaults.py diff --git a/airflow/providers/zendesk/.latest-doc-only-change.txt b/providers/src/airflow/providers/zendesk/.latest-doc-only-change.txt similarity index 100% rename from airflow/providers/zendesk/.latest-doc-only-change.txt rename to providers/src/airflow/providers/zendesk/.latest-doc-only-change.txt diff --git a/airflow/providers/zendesk/CHANGELOG.rst b/providers/src/airflow/providers/zendesk/CHANGELOG.rst similarity index 100% rename from airflow/providers/zendesk/CHANGELOG.rst rename to providers/src/airflow/providers/zendesk/CHANGELOG.rst diff --git a/airflow/providers/zendesk/__init__.py b/providers/src/airflow/providers/zendesk/__init__.py similarity index 100% rename from airflow/providers/zendesk/__init__.py rename to providers/src/airflow/providers/zendesk/__init__.py diff --git a/airflow/providers/vertica/hooks/__init__.py b/providers/src/airflow/providers/zendesk/hooks/__init__.py similarity index 100% rename from airflow/providers/vertica/hooks/__init__.py rename to providers/src/airflow/providers/zendesk/hooks/__init__.py diff --git a/airflow/providers/zendesk/hooks/zendesk.py b/providers/src/airflow/providers/zendesk/hooks/zendesk.py similarity index 100% rename from airflow/providers/zendesk/hooks/zendesk.py rename to providers/src/airflow/providers/zendesk/hooks/zendesk.py diff --git a/airflow/providers/zendesk/provider.yaml b/providers/src/airflow/providers/zendesk/provider.yaml similarity index 100% rename from airflow/providers/zendesk/provider.yaml rename to providers/src/airflow/providers/zendesk/provider.yaml diff --git a/tests/_internals/__init__.py b/providers/tests/__init__.py similarity index 100% rename from tests/_internals/__init__.py rename to providers/tests/__init__.py diff --git a/tests/integration/providers/__init__.py b/providers/tests/airbyte/__init__.py similarity index 100% rename from tests/integration/providers/__init__.py rename to providers/tests/airbyte/__init__.py diff --git a/tests/integration/providers/apache/__init__.py b/providers/tests/airbyte/hooks/__init__.py similarity index 100% rename from tests/integration/providers/apache/__init__.py rename to providers/tests/airbyte/hooks/__init__.py diff --git a/tests/providers/airbyte/hooks/test_airbyte.py b/providers/tests/airbyte/hooks/test_airbyte.py similarity index 100% rename from tests/providers/airbyte/hooks/test_airbyte.py rename to providers/tests/airbyte/hooks/test_airbyte.py diff --git a/tests/integration/providers/apache/cassandra/__init__.py b/providers/tests/airbyte/operators/__init__.py similarity index 100% rename from tests/integration/providers/apache/cassandra/__init__.py rename to providers/tests/airbyte/operators/__init__.py diff --git a/tests/providers/airbyte/operators/test_airbyte.py b/providers/tests/airbyte/operators/test_airbyte.py similarity index 100% rename from tests/providers/airbyte/operators/test_airbyte.py rename to providers/tests/airbyte/operators/test_airbyte.py diff --git a/tests/integration/providers/apache/cassandra/hooks/__init__.py b/providers/tests/airbyte/sensors/__init__.py similarity index 100% rename from tests/integration/providers/apache/cassandra/hooks/__init__.py rename to providers/tests/airbyte/sensors/__init__.py diff --git a/tests/providers/airbyte/sensors/test_airbyte.py b/providers/tests/airbyte/sensors/test_airbyte.py similarity index 100% rename from tests/providers/airbyte/sensors/test_airbyte.py rename to providers/tests/airbyte/sensors/test_airbyte.py diff --git a/tests/integration/providers/apache/drill/__init__.py b/providers/tests/airbyte/triggers/__init__.py similarity index 100% rename from tests/integration/providers/apache/drill/__init__.py rename to providers/tests/airbyte/triggers/__init__.py diff --git a/tests/providers/airbyte/triggers/test_airbyte.py b/providers/tests/airbyte/triggers/test_airbyte.py similarity index 100% rename from tests/providers/airbyte/triggers/test_airbyte.py rename to providers/tests/airbyte/triggers/test_airbyte.py diff --git a/tests/integration/providers/apache/drill/hooks/__init__.py b/providers/tests/alibaba/__init__.py similarity index 100% rename from tests/integration/providers/apache/drill/hooks/__init__.py rename to providers/tests/alibaba/__init__.py diff --git a/tests/integration/providers/apache/drill/operators/__init__.py b/providers/tests/alibaba/cloud/__init__.py similarity index 100% rename from tests/integration/providers/apache/drill/operators/__init__.py rename to providers/tests/alibaba/cloud/__init__.py diff --git a/tests/integration/providers/apache/hive/__init__.py b/providers/tests/alibaba/cloud/hooks/__init__.py similarity index 100% rename from tests/integration/providers/apache/hive/__init__.py rename to providers/tests/alibaba/cloud/hooks/__init__.py diff --git a/tests/providers/alibaba/cloud/hooks/test_analyticdb_spark.py b/providers/tests/alibaba/cloud/hooks/test_analyticdb_spark.py similarity index 99% rename from tests/providers/alibaba/cloud/hooks/test_analyticdb_spark.py rename to providers/tests/alibaba/cloud/hooks/test_analyticdb_spark.py index bf38a3f7ca666..b3e9100e2e59d 100644 --- a/tests/providers/alibaba/cloud/hooks/test_analyticdb_spark.py +++ b/providers/tests/alibaba/cloud/hooks/test_analyticdb_spark.py @@ -34,7 +34,8 @@ ) from airflow.providers.alibaba.cloud.hooks.analyticdb_spark import AnalyticDBSparkHook -from tests.providers.alibaba.cloud.utils.analyticdb_spark_mock import mock_adb_spark_hook_default_project_id + +from providers.tests.alibaba.cloud.utils.analyticdb_spark_mock import mock_adb_spark_hook_default_project_id ADB_SPARK_STRING = "airflow.providers.alibaba.cloud.hooks.analyticdb_spark.{}" MOCK_ADB_SPARK_CONN_ID = "mock_id" diff --git a/tests/providers/alibaba/cloud/hooks/test_oss.py b/providers/tests/alibaba/cloud/hooks/test_oss.py similarity index 99% rename from tests/providers/alibaba/cloud/hooks/test_oss.py rename to providers/tests/alibaba/cloud/hooks/test_oss.py index 1c47aa10c9741..2bb53dddece5c 100644 --- a/tests/providers/alibaba/cloud/hooks/test_oss.py +++ b/providers/tests/alibaba/cloud/hooks/test_oss.py @@ -20,7 +20,8 @@ from unittest import mock from airflow.providers.alibaba.cloud.hooks.oss import OSSHook -from tests.providers.alibaba.cloud.utils.oss_mock import mock_oss_hook_default_project_id + +from providers.tests.alibaba.cloud.utils.oss_mock import mock_oss_hook_default_project_id OSS_STRING = "airflow.providers.alibaba.cloud.hooks.oss.{}" MOCK_OSS_CONN_ID = "mock_id" diff --git a/tests/integration/providers/apache/hive/transfers/__init__.py b/providers/tests/alibaba/cloud/log/__init__.py similarity index 100% rename from tests/integration/providers/apache/hive/transfers/__init__.py rename to providers/tests/alibaba/cloud/log/__init__.py diff --git a/tests/providers/alibaba/cloud/log/test_oss_task_handler.py b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py similarity index 98% rename from tests/providers/alibaba/cloud/log/test_oss_task_handler.py rename to providers/tests/alibaba/cloud/log/test_oss_task_handler.py index b17d953410117..18abe57aa09b7 100644 --- a/tests/providers/alibaba/cloud/log/test_oss_task_handler.py +++ b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py @@ -26,8 +26,9 @@ from airflow.providers.alibaba.cloud.log.oss_task_handler import OSSTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/integration/providers/apache/kafka/__init__.py b/providers/tests/alibaba/cloud/operators/__init__.py similarity index 100% rename from tests/integration/providers/apache/kafka/__init__.py rename to providers/tests/alibaba/cloud/operators/__init__.py diff --git a/tests/providers/alibaba/cloud/operators/test_analyticdb_spark.py b/providers/tests/alibaba/cloud/operators/test_analyticdb_spark.py similarity index 100% rename from tests/providers/alibaba/cloud/operators/test_analyticdb_spark.py rename to providers/tests/alibaba/cloud/operators/test_analyticdb_spark.py diff --git a/tests/providers/alibaba/cloud/operators/test_oss.py b/providers/tests/alibaba/cloud/operators/test_oss.py similarity index 100% rename from tests/providers/alibaba/cloud/operators/test_oss.py rename to providers/tests/alibaba/cloud/operators/test_oss.py diff --git a/tests/integration/providers/apache/kafka/hooks/__init__.py b/providers/tests/alibaba/cloud/sensors/__init__.py similarity index 100% rename from tests/integration/providers/apache/kafka/hooks/__init__.py rename to providers/tests/alibaba/cloud/sensors/__init__.py diff --git a/tests/providers/alibaba/cloud/sensors/test_analyticdb_spark.py b/providers/tests/alibaba/cloud/sensors/test_analyticdb_spark.py similarity index 100% rename from tests/providers/alibaba/cloud/sensors/test_analyticdb_spark.py rename to providers/tests/alibaba/cloud/sensors/test_analyticdb_spark.py diff --git a/tests/providers/alibaba/cloud/sensors/test_oss_key.py b/providers/tests/alibaba/cloud/sensors/test_oss_key.py similarity index 100% rename from tests/providers/alibaba/cloud/sensors/test_oss_key.py rename to providers/tests/alibaba/cloud/sensors/test_oss_key.py diff --git a/tests/integration/providers/apache/kafka/operators/__init__.py b/providers/tests/alibaba/cloud/utils/__init__.py similarity index 100% rename from tests/integration/providers/apache/kafka/operators/__init__.py rename to providers/tests/alibaba/cloud/utils/__init__.py diff --git a/tests/providers/alibaba/cloud/utils/analyticdb_spark_mock.py b/providers/tests/alibaba/cloud/utils/analyticdb_spark_mock.py similarity index 100% rename from tests/providers/alibaba/cloud/utils/analyticdb_spark_mock.py rename to providers/tests/alibaba/cloud/utils/analyticdb_spark_mock.py diff --git a/tests/providers/alibaba/cloud/utils/oss_mock.py b/providers/tests/alibaba/cloud/utils/oss_mock.py similarity index 100% rename from tests/providers/alibaba/cloud/utils/oss_mock.py rename to providers/tests/alibaba/cloud/utils/oss_mock.py diff --git a/tests/providers/alibaba/cloud/utils/test_utils.py b/providers/tests/alibaba/cloud/utils/test_utils.py similarity index 100% rename from tests/providers/alibaba/cloud/utils/test_utils.py rename to providers/tests/alibaba/cloud/utils/test_utils.py diff --git a/tests/integration/providers/apache/kafka/sensors/__init__.py b/providers/tests/amazon/__init__.py similarity index 100% rename from tests/integration/providers/apache/kafka/sensors/__init__.py rename to providers/tests/amazon/__init__.py diff --git a/tests/providers/amazon/aws/.gitignore b/providers/tests/amazon/aws/.gitignore similarity index 100% rename from tests/providers/amazon/aws/.gitignore rename to providers/tests/amazon/aws/.gitignore diff --git a/tests/integration/providers/apache/kafka/triggers/__init__.py b/providers/tests/amazon/aws/__init__.py similarity index 100% rename from tests/integration/providers/apache/kafka/triggers/__init__.py rename to providers/tests/amazon/aws/__init__.py diff --git a/tests/integration/providers/apache/pinot/__init__.py b/providers/tests/amazon/aws/assets/__init__.py similarity index 100% rename from tests/integration/providers/apache/pinot/__init__.py rename to providers/tests/amazon/aws/assets/__init__.py diff --git a/tests/providers/amazon/aws/assets/test_s3.py b/providers/tests/amazon/aws/assets/test_s3.py similarity index 100% rename from tests/providers/amazon/aws/assets/test_s3.py rename to providers/tests/amazon/aws/assets/test_s3.py diff --git a/tests/integration/providers/apache/pinot/hooks/__init__.py b/providers/tests/amazon/aws/auth_manager/__init__.py similarity index 100% rename from tests/integration/providers/apache/pinot/hooks/__init__.py rename to providers/tests/amazon/aws/auth_manager/__init__.py diff --git a/tests/integration/providers/google/__init__.py b/providers/tests/amazon/aws/auth_manager/avp/__init__.py similarity index 100% rename from tests/integration/providers/google/__init__.py rename to providers/tests/amazon/aws/auth_manager/avp/__init__.py diff --git a/tests/providers/amazon/aws/auth_manager/avp/test_entities.py b/providers/tests/amazon/aws/auth_manager/avp/test_entities.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/avp/test_entities.py rename to providers/tests/amazon/aws/auth_manager/avp/test_entities.py diff --git a/tests/providers/amazon/aws/auth_manager/avp/test_facade.py b/providers/tests/amazon/aws/auth_manager/avp/test_facade.py similarity index 98% rename from tests/providers/amazon/aws/auth_manager/avp/test_facade.py rename to providers/tests/amazon/aws/auth_manager/avp/test_facade.py index 5c632ac1ba8e7..3d2a0195039df 100644 --- a/tests/providers/amazon/aws/auth_manager/avp/test_facade.py +++ b/providers/tests/amazon/aws/auth_manager/avp/test_facade.py @@ -27,7 +27,8 @@ from airflow.providers.amazon.aws.auth_manager.avp.facade import AwsAuthManagerAmazonVerifiedPermissionsFacade from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser from airflow.utils.helpers import prune_dict -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars if TYPE_CHECKING: from airflow.auth.managers.base_auth_manager import ResourceMethod @@ -311,8 +312,8 @@ def test_get_batch_is_authorized_single_result_unsuccessful(self, facade): user=test_user, ) - def test_is_policy_store_schema_up_to_date_when_schema_up_to_date(self, facade, airflow_root_path): - schema_path = airflow_root_path.joinpath( + def test_is_policy_store_schema_up_to_date_when_schema_up_to_date(self, facade, providers_src_folder): + schema_path = providers_src_folder.joinpath( "airflow", "providers", "amazon", "aws", "auth_manager", "avp", "schema.json" ).resolve() with open(schema_path) as schema_file: @@ -322,8 +323,8 @@ def test_is_policy_store_schema_up_to_date_when_schema_up_to_date(self, facade, assert facade.is_policy_store_schema_up_to_date() - def test_is_policy_store_schema_up_to_date_when_schema_is_modified(self, facade, airflow_root_path): - schema_path = airflow_root_path.joinpath( + def test_is_policy_store_schema_up_to_date_when_schema_is_modified(self, facade, providers_src_folder): + schema_path = providers_src_folder.joinpath( "airflow", "providers", "amazon", "aws", "auth_manager", "avp", "schema.json" ).resolve() with open(schema_path) as schema_file: diff --git a/tests/integration/providers/google/cloud/__init__.py b/providers/tests/amazon/aws/auth_manager/cli/__init__.py similarity index 100% rename from tests/integration/providers/google/cloud/__init__.py rename to providers/tests/amazon/aws/auth_manager/cli/__init__.py diff --git a/tests/providers/amazon/aws/auth_manager/cli/test_avp_commands.py b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py similarity index 97% rename from tests/providers/amazon/aws/auth_manager/cli/test_avp_commands.py rename to providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py index f285beca8af1b..6122079fee4cb 100644 --- a/tests/providers/amazon/aws/auth_manager/cli/test_avp_commands.py +++ b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py @@ -23,8 +23,9 @@ from airflow.cli import cli_parser from airflow.providers.amazon.aws.auth_manager.cli.avp_commands import init_avp, update_schema -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS +from dev.tests_common.test_utils.config import conf_vars mock_boto3 = Mock() diff --git a/tests/providers/amazon/aws/auth_manager/cli/test_definition.py b/providers/tests/amazon/aws/auth_manager/cli/test_definition.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/cli/test_definition.py rename to providers/tests/amazon/aws/auth_manager/cli/test_definition.py diff --git a/tests/integration/providers/google/cloud/transfers/__init__.py b/providers/tests/amazon/aws/auth_manager/security_manager/__init__.py similarity index 100% rename from tests/integration/providers/google/cloud/transfers/__init__.py rename to providers/tests/amazon/aws/auth_manager/security_manager/__init__.py diff --git a/tests/providers/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py b/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py similarity index 96% rename from tests/providers/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py rename to providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py index ebb452fb1afb5..b6071aac955c6 100644 --- a/tests/providers/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py +++ b/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py @@ -21,7 +21,7 @@ import pytest from flask import Flask -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error python3_saml = pytest.importorskip("python3-saml") diff --git a/tests/providers/amazon/aws/auth_manager/test_aws_auth_manager.py b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py similarity index 97% rename from tests/providers/amazon/aws/auth_manager/test_aws_auth_manager.py rename to providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py index d827ba3ff0e6d..47e8a4cbcb26d 100644 --- a/tests/providers/amazon/aws/auth_manager/test_aws_auth_manager.py +++ b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py @@ -38,9 +38,10 @@ ) from airflow.www import app as application from airflow.www.extensions.init_appbuilder import init_appbuilder -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.www import check_content_in_response + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.www import check_content_in_response try: from airflow.auth.managers.models.resource_details import ( @@ -61,6 +62,7 @@ else: raise + if TYPE_CHECKING: from airflow.auth.managers.base_auth_manager import ResourceMethod from airflow.auth.managers.models.resource_details import AssetDetails @@ -146,13 +148,17 @@ def client_admin(): ("aws_auth_manager", "avp_policy_store_id"): "avp_policy_store_id", } ): - with patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" - ) as mock_parser, patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" - ) as mock_init_saml_auth, patch( - "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" - ) as mock_is_policy_store_schema_up_to_date: + with ( + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" + ) as mock_parser, + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" + ) as mock_init_saml_auth, + patch( + "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" + ) as mock_is_policy_store_schema_up_to_date, + ): mock_parser.parse_remote.return_value = SAML_METADATA_PARSED mock_is_policy_store_schema_up_to_date.return_value = True diff --git a/tests/providers/amazon/aws/auth_manager/test_constants.py b/providers/tests/amazon/aws/auth_manager/test_constants.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/test_constants.py rename to providers/tests/amazon/aws/auth_manager/test_constants.py diff --git a/tests/providers/amazon/aws/auth_manager/test_user.py b/providers/tests/amazon/aws/auth_manager/test_user.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/test_user.py rename to providers/tests/amazon/aws/auth_manager/test_user.py diff --git a/tests/integration/providers/microsoft/__init__.py b/providers/tests/amazon/aws/auth_manager/views/__init__.py similarity index 100% rename from tests/integration/providers/microsoft/__init__.py rename to providers/tests/amazon/aws/auth_manager/views/__init__.py diff --git a/tests/providers/amazon/aws/auth_manager/views/test_auth.py b/providers/tests/amazon/aws/auth_manager/views/test_auth.py similarity index 80% rename from tests/providers/amazon/aws/auth_manager/views/test_auth.py rename to providers/tests/amazon/aws/auth_manager/views/test_auth.py index 05d2fb84b51cf..9b2eec69188be 100644 --- a/tests/providers/amazon/aws/auth_manager/views/test_auth.py +++ b/providers/tests/amazon/aws/auth_manager/views/test_auth.py @@ -23,8 +23,9 @@ from airflow.exceptions import AirflowException from airflow.www import app as application -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.config import conf_vars pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Test requires Airflow 2.9+"), @@ -62,11 +63,14 @@ def aws_app(): ("aws_auth_manager", "saml_metadata_url"): SAML_METADATA_URL, } ): - with patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" - ) as mock_parser, patch( - "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" - ) as mock_is_policy_store_schema_up_to_date: + with ( + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" + ) as mock_parser, + patch( + "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" + ) as mock_is_policy_store_schema_up_to_date, + ): mock_is_policy_store_schema_up_to_date.return_value = True mock_parser.parse_remote.return_value = SAML_METADATA_PARSED return application.create_app(testing=True, config={"WTF_CSRF_ENABLED": False}) @@ -102,13 +106,17 @@ def test_login_callback_set_user_in_session(self): ("aws_auth_manager", "saml_metadata_url"): SAML_METADATA_URL, } ): - with patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" - ) as mock_parser, patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" - ) as mock_init_saml_auth, patch( - "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" - ) as mock_is_policy_store_schema_up_to_date: + with ( + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" + ) as mock_parser, + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" + ) as mock_init_saml_auth, + patch( + "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" + ) as mock_is_policy_store_schema_up_to_date, + ): mock_is_policy_store_schema_up_to_date.return_value = True mock_parser.parse_remote.return_value = SAML_METADATA_PARSED @@ -140,13 +148,17 @@ def test_login_callback_raise_exception_if_errors(self): ("aws_auth_manager", "saml_metadata_url"): SAML_METADATA_URL, } ): - with patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" - ) as mock_parser, patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" - ) as mock_init_saml_auth, patch( - "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" - ) as mock_is_policy_store_schema_up_to_date: + with ( + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" + ) as mock_parser, + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" + ) as mock_init_saml_auth, + patch( + "airflow.providers.amazon.aws.auth_manager.avp.facade.AwsAuthManagerAmazonVerifiedPermissionsFacade.is_policy_store_schema_up_to_date" + ) as mock_is_policy_store_schema_up_to_date, + ): mock_is_policy_store_schema_up_to_date.return_value = True mock_parser.parse_remote.return_value = SAML_METADATA_PARSED diff --git a/tests/integration/providers/microsoft/mssql/__init__.py b/providers/tests/amazon/aws/config_templates/__init__.py similarity index 100% rename from tests/integration/providers/microsoft/mssql/__init__.py rename to providers/tests/amazon/aws/config_templates/__init__.py diff --git a/tests/providers/amazon/aws/config_templates/args.json b/providers/tests/amazon/aws/config_templates/args.json similarity index 100% rename from tests/providers/amazon/aws/config_templates/args.json rename to providers/tests/amazon/aws/config_templates/args.json diff --git a/tests/providers/amazon/aws/config_templates/job.j2.json b/providers/tests/amazon/aws/config_templates/job.j2.json similarity index 100% rename from tests/providers/amazon/aws/config_templates/job.j2.json rename to providers/tests/amazon/aws/config_templates/job.j2.json diff --git a/tests/providers/amazon/aws/config_templates/steps.j2.json b/providers/tests/amazon/aws/config_templates/steps.j2.json similarity index 100% rename from tests/providers/amazon/aws/config_templates/steps.j2.json rename to providers/tests/amazon/aws/config_templates/steps.j2.json diff --git a/tests/providers/amazon/aws/config_templates/steps.json b/providers/tests/amazon/aws/config_templates/steps.json similarity index 100% rename from tests/providers/amazon/aws/config_templates/steps.json rename to providers/tests/amazon/aws/config_templates/steps.json diff --git a/tests/integration/providers/microsoft/mssql/hooks/__init__.py b/providers/tests/amazon/aws/deferrable/__init__.py similarity index 100% rename from tests/integration/providers/microsoft/mssql/hooks/__init__.py rename to providers/tests/amazon/aws/deferrable/__init__.py diff --git a/tests/integration/providers/mongo/__init__.py b/providers/tests/amazon/aws/deferrable/hooks/__init__.py similarity index 100% rename from tests/integration/providers/mongo/__init__.py rename to providers/tests/amazon/aws/deferrable/hooks/__init__.py diff --git a/tests/integration/providers/mongo/sensors/__init__.py b/providers/tests/amazon/aws/executors/__init__.py similarity index 100% rename from tests/integration/providers/mongo/sensors/__init__.py rename to providers/tests/amazon/aws/executors/__init__.py diff --git a/tests/integration/providers/openlineage/__init__.py b/providers/tests/amazon/aws/executors/batch/__init__.py similarity index 100% rename from tests/integration/providers/openlineage/__init__.py rename to providers/tests/amazon/aws/executors/batch/__init__.py diff --git a/tests/providers/amazon/aws/executors/batch/test_batch_executor.py b/providers/tests/amazon/aws/executors/batch/test_batch_executor.py similarity index 99% rename from tests/providers/amazon/aws/executors/batch/test_batch_executor.py rename to providers/tests/amazon/aws/executors/batch/test_batch_executor.py index 8b81e4a966a45..e7037bd16c85e 100644 --- a/tests/providers/amazon/aws/executors/batch/test_batch_executor.py +++ b/providers/tests/amazon/aws/executors/batch/test_batch_executor.py @@ -45,8 +45,9 @@ from airflow.utils.helpers import convert_camel_to_snake from airflow.utils.state import State from airflow.version import version as airflow_version_str -from tests.conftest import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from tests.test_utils.config import conf_vars + +from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from dev.tests_common.test_utils.config import conf_vars airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3])) ARN1 = "arn1" @@ -652,9 +653,9 @@ def teardown_method(self) -> None: ) def test_validate_config_defaults(self): """Assert that the defaults stated in the config.yml file match those in utils.CONFIG_DEFAULTS.""" - curr_dir = os.path.dirname(os.path.abspath(__file__)) - executor_path = "aws/executors/batch" - config_filename = curr_dir.replace("tests", "airflow").replace(executor_path, "provider.yaml") + from airflow.providers.amazon import __file__ as provider_path + + config_filename = os.path.join(os.path.dirname(provider_path), "provider.yaml") with open(config_filename) as config: options = yaml.safe_load(config)["config"][CONFIG_GROUP_NAME]["options"] diff --git a/tests/integration/providers/openlineage/operators/__init__.py b/providers/tests/amazon/aws/executors/ecs/__init__.py similarity index 100% rename from tests/integration/providers/openlineage/operators/__init__.py rename to providers/tests/amazon/aws/executors/ecs/__init__.py diff --git a/tests/providers/amazon/aws/executors/ecs/test_ecs_executor.py b/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py similarity index 99% rename from tests/providers/amazon/aws/executors/ecs/test_ecs_executor.py rename to providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py index 51e3c17934900..50cdb580382f3 100644 --- a/tests/providers/amazon/aws/executors/ecs/test_ecs_executor.py +++ b/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py @@ -56,9 +56,10 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import utcnow from airflow.version import version as airflow_version_str -from tests.conftest import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test @@ -1219,9 +1220,9 @@ def test_validate_config_defaults(self): airflow sources, not when airflow is installed from packages, because airflow installed from packages will not have the provider.yml file. """ - curr_dir = os.path.dirname(os.path.abspath(__file__)) - executor_path = "aws/executors/ecs" - config_filename = curr_dir.replace("tests", "airflow").replace(executor_path, "provider.yaml") + from airflow.providers.amazon import __file__ as provider_path + + config_filename = os.path.join(os.path.dirname(provider_path), "provider.yaml") with open(config_filename) as config: options = yaml.safe_load(config)["config"][CONFIG_GROUP_NAME]["options"] diff --git a/tests/integration/providers/qdrant/__init__.py b/providers/tests/amazon/aws/executors/utils/__init__.py similarity index 100% rename from tests/integration/providers/qdrant/__init__.py rename to providers/tests/amazon/aws/executors/utils/__init__.py diff --git a/tests/providers/amazon/aws/executors/utils/test_exponential_backoff_retry.py b/providers/tests/amazon/aws/executors/utils/test_exponential_backoff_retry.py similarity index 100% rename from tests/providers/amazon/aws/executors/utils/test_exponential_backoff_retry.py rename to providers/tests/amazon/aws/executors/utils/test_exponential_backoff_retry.py diff --git a/tests/integration/providers/qdrant/hooks/__init__.py b/providers/tests/amazon/aws/fs/__init__.py similarity index 100% rename from tests/integration/providers/qdrant/hooks/__init__.py rename to providers/tests/amazon/aws/fs/__init__.py diff --git a/tests/providers/amazon/aws/fs/test_s3.py b/providers/tests/amazon/aws/fs/test_s3.py similarity index 100% rename from tests/providers/amazon/aws/fs/test_s3.py rename to providers/tests/amazon/aws/fs/test_s3.py diff --git a/tests/integration/providers/qdrant/operators/__init__.py b/providers/tests/amazon/aws/hooks/__init__.py similarity index 100% rename from tests/integration/providers/qdrant/operators/__init__.py rename to providers/tests/amazon/aws/hooks/__init__.py diff --git a/tests/providers/amazon/aws/hooks/test_appflow.py b/providers/tests/amazon/aws/hooks/test_appflow.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_appflow.py rename to providers/tests/amazon/aws/hooks/test_appflow.py diff --git a/tests/providers/amazon/aws/hooks/test_athena.py b/providers/tests/amazon/aws/hooks/test_athena.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_athena.py rename to providers/tests/amazon/aws/hooks/test_athena.py diff --git a/tests/providers/amazon/aws/hooks/test_athena_sql.py b/providers/tests/amazon/aws/hooks/test_athena_sql.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_athena_sql.py rename to providers/tests/amazon/aws/hooks/test_athena_sql.py diff --git a/tests/providers/amazon/aws/hooks/test_base_aws.py b/providers/tests/amazon/aws/hooks/test_base_aws.py similarity index 98% rename from tests/providers/amazon/aws/hooks/test_base_aws.py rename to providers/tests/amazon/aws/hooks/test_base_aws.py index 0957e6a928aae..c58993d748f2d 100644 --- a/tests/providers/amazon/aws/hooks/test_base_aws.py +++ b/providers/tests/amazon/aws/hooks/test_base_aws.py @@ -49,7 +49,8 @@ resolve_session_factory, ) from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytest.importorskip("aiobotocore") @@ -140,7 +141,7 @@ def mock_conn(request): class TestSessionFactory: @conf_vars( - {("aws", "session_factory"): "tests.providers.amazon.aws.hooks.test_base_aws.CustomSessionFactory"} + {("aws", "session_factory"): "providers.tests.amazon.aws.hooks.test_base_aws.CustomSessionFactory"} ) def test_resolve_session_factory_class(self): cls = resolve_session_factory() @@ -545,13 +546,13 @@ def import_mock(name, *args): return mock_id_token_credentials return orig_import(name, *args) - with mock.patch("builtins.__import__", side_effect=import_mock), mock.patch.dict( - "os.environ", AIRFLOW_CONN_AWS_DEFAULT=mock_connection.get_uri() - ), mock.patch("airflow.providers.amazon.aws.hooks.base_aws.boto3") as mock_boto3, mock.patch( - "airflow.providers.amazon.aws.hooks.base_aws.botocore" - ) as mock_botocore, mock.patch( - "airflow.providers.amazon.aws.hooks.base_aws.botocore.session" - ) as mock_session: + with ( + mock.patch("builtins.__import__", side_effect=import_mock), + mock.patch.dict("os.environ", AIRFLOW_CONN_AWS_DEFAULT=mock_connection.get_uri()), + mock.patch("airflow.providers.amazon.aws.hooks.base_aws.boto3") as mock_boto3, + mock.patch("airflow.providers.amazon.aws.hooks.base_aws.botocore") as mock_botocore, + mock.patch("airflow.providers.amazon.aws.hooks.base_aws.botocore.session") as mock_session, + ): hook = AwsBaseHook(aws_conn_id="aws_default", client_type="airflow_test") credentials_from_hook = hook.get_credentials() @@ -701,12 +702,14 @@ def mock_assume_role_with_saml(**kwargs): } return sts_response - with mock.patch("builtins.__import__", side_effect=import_mock), mock.patch( - "airflow.providers.amazon.aws.hooks.base_aws.requests.Session.get" - ) as mock_get, mock.patch( - "airflow.providers.amazon.aws.hooks.base_aws.BaseSessionFactory._create_basic_session", - spec=boto3.session.Session, - ) as mocked_basic_session: + with ( + mock.patch("builtins.__import__", side_effect=import_mock), + mock.patch("airflow.providers.amazon.aws.hooks.base_aws.requests.Session.get") as mock_get, + mock.patch( + "airflow.providers.amazon.aws.hooks.base_aws.BaseSessionFactory._create_basic_session", + spec=boto3.session.Session, + ) as mocked_basic_session, + ): mocked_basic_session.return_value.region_name = "us-east-2" mock_client = mocked_basic_session.return_value.client mock_client.return_value.assume_role_with_saml.side_effect = mock_assume_role_with_saml diff --git a/tests/providers/amazon/aws/hooks/test_batch_client.py b/providers/tests/amazon/aws/hooks/test_batch_client.py similarity index 98% rename from tests/providers/amazon/aws/hooks/test_batch_client.py rename to providers/tests/amazon/aws/hooks/test_batch_client.py index 83385878b57fe..98b89de0a60b4 100644 --- a/tests/providers/amazon/aws/hooks/test_batch_client.py +++ b/providers/tests/amazon/aws/hooks/test_batch_client.py @@ -126,13 +126,11 @@ def test_wait_for_job_with_logs(self): thread_stop = mock.Mock(side_effect=lambda: time.sleep(2)) thread_join = mock.Mock(side_effect=lambda: time.sleep(2)) - with mock.patch.object( - batch_log_fetcher, "start", thread_start - ) as mock_fetcher_start, mock.patch.object( - batch_log_fetcher, "stop", thread_stop - ) as mock_fetcher_stop, mock.patch.object( - batch_log_fetcher, "join", thread_join - ) as mock_fetcher_join: + with ( + mock.patch.object(batch_log_fetcher, "start", thread_start) as mock_fetcher_start, + mock.patch.object(batch_log_fetcher, "stop", thread_stop) as mock_fetcher_stop, + mock.patch.object(batch_log_fetcher, "join", thread_join) as mock_fetcher_join, + ): self.batch_client.wait_for_job(JOB_ID, get_batch_log_fetcher=mock_get_batch_log_fetcher) mock_get_batch_log_fetcher.assert_called_with(JOB_ID) mock_fetcher_start.assert_called_once() diff --git a/tests/providers/amazon/aws/hooks/test_batch_waiters.py b/providers/tests/amazon/aws/hooks/test_batch_waiters.py similarity index 97% rename from tests/providers/amazon/aws/hooks/test_batch_waiters.py rename to providers/tests/amazon/aws/hooks/test_batch_waiters.py index 72f2061b902ca..d67b25d07f2a6 100644 --- a/tests/providers/amazon/aws/hooks/test_batch_waiters.py +++ b/providers/tests/amazon/aws/hooks/test_batch_waiters.py @@ -157,13 +157,12 @@ def test_wait_for_job_with_cloudwatch_logs(self): thread_stop = mock.Mock(side_effect=lambda: time.sleep(2)) thread_join = mock.Mock(side_effect=lambda: time.sleep(2)) - with mock.patch.object(self.batch_waiters, "get_waiter") as mock_get_waiter, mock.patch.object( - batch_log_fetcher, "start", thread_start - ) as mock_fetcher_start, mock.patch.object( - batch_log_fetcher, "stop", thread_stop - ) as mock_fetcher_stop, mock.patch.object( - batch_log_fetcher, "join", thread_join - ) as mock_fetcher_join: + with ( + mock.patch.object(self.batch_waiters, "get_waiter") as mock_get_waiter, + mock.patch.object(batch_log_fetcher, "start", thread_start) as mock_fetcher_start, + mock.patch.object(batch_log_fetcher, "stop", thread_stop) as mock_fetcher_stop, + mock.patch.object(batch_log_fetcher, "join", thread_join) as mock_fetcher_join, + ): # Run the wait_for_job method self.batch_waiters.wait_for_job(self.job_id, get_batch_log_fetcher=mock_get_batch_log_fetcher) diff --git a/tests/providers/amazon/aws/hooks/test_bedrock.py b/providers/tests/amazon/aws/hooks/test_bedrock.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_bedrock.py rename to providers/tests/amazon/aws/hooks/test_bedrock.py diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/providers/tests/amazon/aws/hooks/test_chime.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_chime.py rename to providers/tests/amazon/aws/hooks/test_chime.py diff --git a/tests/providers/amazon/aws/hooks/test_cloud_formation.py b/providers/tests/amazon/aws/hooks/test_cloud_formation.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_cloud_formation.py rename to providers/tests/amazon/aws/hooks/test_cloud_formation.py diff --git a/tests/providers/amazon/aws/hooks/test_comprehend.py b/providers/tests/amazon/aws/hooks/test_comprehend.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_comprehend.py rename to providers/tests/amazon/aws/hooks/test_comprehend.py diff --git a/tests/providers/amazon/aws/hooks/test_datasync.py b/providers/tests/amazon/aws/hooks/test_datasync.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_datasync.py rename to providers/tests/amazon/aws/hooks/test_datasync.py diff --git a/tests/providers/amazon/aws/hooks/test_dms.py b/providers/tests/amazon/aws/hooks/test_dms.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_dms.py rename to providers/tests/amazon/aws/hooks/test_dms.py diff --git a/tests/providers/amazon/aws/hooks/test_dynamodb.py b/providers/tests/amazon/aws/hooks/test_dynamodb.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_dynamodb.py rename to providers/tests/amazon/aws/hooks/test_dynamodb.py diff --git a/tests/providers/amazon/aws/hooks/test_ec2.py b/providers/tests/amazon/aws/hooks/test_ec2.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_ec2.py rename to providers/tests/amazon/aws/hooks/test_ec2.py diff --git a/tests/providers/amazon/aws/hooks/test_ecr.py b/providers/tests/amazon/aws/hooks/test_ecr.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_ecr.py rename to providers/tests/amazon/aws/hooks/test_ecr.py diff --git a/tests/providers/amazon/aws/hooks/test_ecs.py b/providers/tests/amazon/aws/hooks/test_ecs.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_ecs.py rename to providers/tests/amazon/aws/hooks/test_ecs.py diff --git a/tests/providers/amazon/aws/hooks/test_eks.py b/providers/tests/amazon/aws/hooks/test_eks.py similarity index 99% rename from tests/providers/amazon/aws/hooks/test_eks.py rename to providers/tests/amazon/aws/hooks/test_eks.py index ae0f0e19949fa..cf3638d30d37d 100644 --- a/tests/providers/amazon/aws/hooks/test_eks.py +++ b/providers/tests/amazon/aws/hooks/test_eks.py @@ -53,7 +53,8 @@ ) from airflow.providers.amazon.aws.hooks.eks import COMMAND, EksHook -from tests.providers.amazon.aws.utils.eks_test_constants import ( + +from providers.tests.amazon.aws.utils.eks_test_constants import ( DEFAULT_CONN_ID, DEFAULT_NAMESPACE, DISK_SIZE, @@ -82,7 +83,7 @@ RegExTemplates, ResponseAttributes, ) -from tests.providers.amazon.aws.utils.eks_test_utils import ( +from providers.tests.amazon.aws.utils.eks_test_utils import ( attributes_to_test, generate_clusters, generate_dict, diff --git a/tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py b/providers/tests/amazon/aws/hooks/test_elasticache_replication_group.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py rename to providers/tests/amazon/aws/hooks/test_elasticache_replication_group.py diff --git a/tests/providers/amazon/aws/hooks/test_emr.py b/providers/tests/amazon/aws/hooks/test_emr.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_emr.py rename to providers/tests/amazon/aws/hooks/test_emr.py diff --git a/tests/providers/amazon/aws/hooks/test_emr_containers.py b/providers/tests/amazon/aws/hooks/test_emr_containers.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_emr_containers.py rename to providers/tests/amazon/aws/hooks/test_emr_containers.py diff --git a/tests/providers/amazon/aws/hooks/test_emr_serverless.py b/providers/tests/amazon/aws/hooks/test_emr_serverless.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_emr_serverless.py rename to providers/tests/amazon/aws/hooks/test_emr_serverless.py diff --git a/tests/providers/amazon/aws/hooks/test_eventbridge.py b/providers/tests/amazon/aws/hooks/test_eventbridge.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_eventbridge.py rename to providers/tests/amazon/aws/hooks/test_eventbridge.py diff --git a/tests/providers/amazon/aws/hooks/test_glacier.py b/providers/tests/amazon/aws/hooks/test_glacier.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_glacier.py rename to providers/tests/amazon/aws/hooks/test_glacier.py diff --git a/tests/providers/amazon/aws/hooks/test_glue.py b/providers/tests/amazon/aws/hooks/test_glue.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_glue.py rename to providers/tests/amazon/aws/hooks/test_glue.py diff --git a/tests/providers/amazon/aws/hooks/test_glue_catalog.py b/providers/tests/amazon/aws/hooks/test_glue_catalog.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_glue_catalog.py rename to providers/tests/amazon/aws/hooks/test_glue_catalog.py diff --git a/tests/providers/amazon/aws/hooks/test_glue_crawler.py b/providers/tests/amazon/aws/hooks/test_glue_crawler.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_glue_crawler.py rename to providers/tests/amazon/aws/hooks/test_glue_crawler.py diff --git a/tests/providers/amazon/aws/hooks/test_glue_databrew.py b/providers/tests/amazon/aws/hooks/test_glue_databrew.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_glue_databrew.py rename to providers/tests/amazon/aws/hooks/test_glue_databrew.py diff --git a/tests/providers/amazon/aws/hooks/test_hooks_signature.py b/providers/tests/amazon/aws/hooks/test_hooks_signature.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_hooks_signature.py rename to providers/tests/amazon/aws/hooks/test_hooks_signature.py diff --git a/tests/providers/amazon/aws/hooks/test_kinesis.py b/providers/tests/amazon/aws/hooks/test_kinesis.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_kinesis.py rename to providers/tests/amazon/aws/hooks/test_kinesis.py diff --git a/tests/providers/amazon/aws/hooks/test_kinesis_analytics.py b/providers/tests/amazon/aws/hooks/test_kinesis_analytics.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_kinesis_analytics.py rename to providers/tests/amazon/aws/hooks/test_kinesis_analytics.py diff --git a/tests/providers/amazon/aws/hooks/test_lambda_function.py b/providers/tests/amazon/aws/hooks/test_lambda_function.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_lambda_function.py rename to providers/tests/amazon/aws/hooks/test_lambda_function.py diff --git a/tests/providers/amazon/aws/hooks/test_logs.py b/providers/tests/amazon/aws/hooks/test_logs.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_logs.py rename to providers/tests/amazon/aws/hooks/test_logs.py diff --git a/tests/providers/amazon/aws/hooks/test_neptune.py b/providers/tests/amazon/aws/hooks/test_neptune.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_neptune.py rename to providers/tests/amazon/aws/hooks/test_neptune.py diff --git a/tests/providers/amazon/aws/hooks/test_opensearch_serverless.py b/providers/tests/amazon/aws/hooks/test_opensearch_serverless.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_opensearch_serverless.py rename to providers/tests/amazon/aws/hooks/test_opensearch_serverless.py diff --git a/tests/providers/amazon/aws/hooks/test_quicksight.py b/providers/tests/amazon/aws/hooks/test_quicksight.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_quicksight.py rename to providers/tests/amazon/aws/hooks/test_quicksight.py diff --git a/tests/providers/amazon/aws/hooks/test_rds.py b/providers/tests/amazon/aws/hooks/test_rds.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_rds.py rename to providers/tests/amazon/aws/hooks/test_rds.py diff --git a/tests/providers/amazon/aws/hooks/test_redshift_cluster.py b/providers/tests/amazon/aws/hooks/test_redshift_cluster.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_redshift_cluster.py rename to providers/tests/amazon/aws/hooks/test_redshift_cluster.py diff --git a/tests/providers/amazon/aws/hooks/test_redshift_data.py b/providers/tests/amazon/aws/hooks/test_redshift_data.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_redshift_data.py rename to providers/tests/amazon/aws/hooks/test_redshift_data.py diff --git a/tests/providers/amazon/aws/hooks/test_redshift_sql.py b/providers/tests/amazon/aws/hooks/test_redshift_sql.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_redshift_sql.py rename to providers/tests/amazon/aws/hooks/test_redshift_sql.py diff --git a/tests/providers/amazon/aws/hooks/test_s3.py b/providers/tests/amazon/aws/hooks/test_s3.py similarity index 99% rename from tests/providers/amazon/aws/hooks/test_s3.py rename to providers/tests/amazon/aws/hooks/test_s3.py index 43c4b94445b6f..2e9a010006bab 100644 --- a/tests/providers/amazon/aws/hooks/test_s3.py +++ b/providers/tests/amazon/aws/hooks/test_s3.py @@ -42,7 +42,8 @@ unify_bucket_name_and_key, ) from airflow.utils.timezone import datetime -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS @pytest.fixture diff --git a/tests/providers/amazon/aws/hooks/test_sagemaker.py b/providers/tests/amazon/aws/hooks/test_sagemaker.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_sagemaker.py rename to providers/tests/amazon/aws/hooks/test_sagemaker.py diff --git a/tests/providers/amazon/aws/hooks/test_secrets_manager.py b/providers/tests/amazon/aws/hooks/test_secrets_manager.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_secrets_manager.py rename to providers/tests/amazon/aws/hooks/test_secrets_manager.py diff --git a/tests/providers/amazon/aws/hooks/test_ses.py b/providers/tests/amazon/aws/hooks/test_ses.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_ses.py rename to providers/tests/amazon/aws/hooks/test_ses.py diff --git a/tests/providers/amazon/aws/hooks/test_sns.py b/providers/tests/amazon/aws/hooks/test_sns.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_sns.py rename to providers/tests/amazon/aws/hooks/test_sns.py diff --git a/tests/providers/amazon/aws/hooks/test_sqs.py b/providers/tests/amazon/aws/hooks/test_sqs.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_sqs.py rename to providers/tests/amazon/aws/hooks/test_sqs.py diff --git a/tests/providers/amazon/aws/hooks/test_ssm.py b/providers/tests/amazon/aws/hooks/test_ssm.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_ssm.py rename to providers/tests/amazon/aws/hooks/test_ssm.py diff --git a/tests/providers/amazon/aws/hooks/test_step_function.py b/providers/tests/amazon/aws/hooks/test_step_function.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_step_function.py rename to providers/tests/amazon/aws/hooks/test_step_function.py diff --git a/tests/providers/amazon/aws/hooks/test_sts.py b/providers/tests/amazon/aws/hooks/test_sts.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_sts.py rename to providers/tests/amazon/aws/hooks/test_sts.py diff --git a/tests/providers/amazon/aws/hooks/test_verified_permissions.py b/providers/tests/amazon/aws/hooks/test_verified_permissions.py similarity index 100% rename from tests/providers/amazon/aws/hooks/test_verified_permissions.py rename to providers/tests/amazon/aws/hooks/test_verified_permissions.py diff --git a/tests/integration/providers/redis/__init__.py b/providers/tests/amazon/aws/infrastructure/__init__.py similarity index 100% rename from tests/integration/providers/redis/__init__.py rename to providers/tests/amazon/aws/infrastructure/__init__.py diff --git a/tests/integration/providers/redis/hooks/__init__.py b/providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py similarity index 100% rename from tests/integration/providers/redis/hooks/__init__.py rename to providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py diff --git a/tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/outputs.tf b/providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/outputs.tf similarity index 100% rename from tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/outputs.tf rename to providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/outputs.tf diff --git a/tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/resources.tf b/providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/resources.tf similarity index 100% rename from tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/resources.tf rename to providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/resources.tf diff --git a/tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/variables.tf b/providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/variables.tf similarity index 100% rename from tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/variables.tf rename to providers/tests/amazon/aws/infrastructure/example_s3_to_redshift/variables.tf diff --git a/tests/integration/providers/redis/operators/__init__.py b/providers/tests/amazon/aws/links/__init__.py similarity index 100% rename from tests/integration/providers/redis/operators/__init__.py rename to providers/tests/amazon/aws/links/__init__.py diff --git a/tests/providers/amazon/aws/links/test_athena.py b/providers/tests/amazon/aws/links/test_athena.py similarity index 95% rename from tests/providers/amazon/aws/links/test_athena.py rename to providers/tests/amazon/aws/links/test_athena.py index 1729fdf4e5c23..2da3f3fa441d1 100644 --- a/tests/providers/amazon/aws/links/test_athena.py +++ b/providers/tests/amazon/aws/links/test_athena.py @@ -17,7 +17,8 @@ from __future__ import annotations from airflow.providers.amazon.aws.links.athena import AthenaQueryResultsLink -from tests.providers.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase + +from providers.tests.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase class TestAthenaQueryResultsLink(BaseAwsLinksTestCase): diff --git a/tests/providers/amazon/aws/links/test_base_aws.py b/providers/tests/amazon/aws/links/test_base_aws.py similarity index 98% rename from tests/providers/amazon/aws/links/test_base_aws.py rename to providers/tests/amazon/aws/links/test_base_aws.py index 1afcfea0a826f..78622870806fb 100644 --- a/tests/providers/amazon/aws/links/test_base_aws.py +++ b/providers/tests/amazon/aws/links/test_base_aws.py @@ -25,8 +25,9 @@ from airflow.models.xcom import XCom from airflow.providers.amazon.aws.links.base_aws import BaseAwsLink from airflow.serialization.serialized_objects import SerializedDAG -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.mock_operators import MockOperator if TYPE_CHECKING: from airflow.models.taskinstance import TaskInstance diff --git a/tests/providers/amazon/aws/links/test_batch.py b/providers/tests/amazon/aws/links/test_batch.py similarity index 97% rename from tests/providers/amazon/aws/links/test_batch.py rename to providers/tests/amazon/aws/links/test_batch.py index 2c8c9b59cb830..eafe49260c9b4 100644 --- a/tests/providers/amazon/aws/links/test_batch.py +++ b/providers/tests/amazon/aws/links/test_batch.py @@ -21,7 +21,8 @@ BatchJobDetailsLink, BatchJobQueueLink, ) -from tests.providers.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase + +from providers.tests.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase class TestBatchJobDefinitionLink(BaseAwsLinksTestCase): diff --git a/tests/providers/amazon/aws/links/test_emr.py b/providers/tests/amazon/aws/links/test_emr.py similarity index 99% rename from tests/providers/amazon/aws/links/test_emr.py rename to providers/tests/amazon/aws/links/test_emr.py index 50db654d4914a..7510b2a2f50d3 100644 --- a/tests/providers/amazon/aws/links/test_emr.py +++ b/providers/tests/amazon/aws/links/test_emr.py @@ -32,7 +32,8 @@ get_log_uri, get_serverless_dashboard_url, ) -from tests.providers.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase + +from providers.tests.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase class TestEmrClusterLink(BaseAwsLinksTestCase): diff --git a/tests/providers/amazon/aws/links/test_glue.py b/providers/tests/amazon/aws/links/test_glue.py similarity index 95% rename from tests/providers/amazon/aws/links/test_glue.py rename to providers/tests/amazon/aws/links/test_glue.py index 5f929cd3e950a..772ac5ee9c773 100644 --- a/tests/providers/amazon/aws/links/test_glue.py +++ b/providers/tests/amazon/aws/links/test_glue.py @@ -17,7 +17,8 @@ from __future__ import annotations from airflow.providers.amazon.aws.links.glue import GlueJobRunDetailsLink -from tests.providers.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase + +from providers.tests.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase class TestGlueJobRunDetailsLink(BaseAwsLinksTestCase): diff --git a/tests/providers/amazon/aws/links/test_logs.py b/providers/tests/amazon/aws/links/test_logs.py similarity index 96% rename from tests/providers/amazon/aws/links/test_logs.py rename to providers/tests/amazon/aws/links/test_logs.py index 991a8bc6f0c19..b596e7517eafb 100644 --- a/tests/providers/amazon/aws/links/test_logs.py +++ b/providers/tests/amazon/aws/links/test_logs.py @@ -17,7 +17,8 @@ from __future__ import annotations from airflow.providers.amazon.aws.links.logs import CloudWatchEventsLink -from tests.providers.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase + +from providers.tests.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase class TestCloudWatchEventsLink(BaseAwsLinksTestCase): diff --git a/tests/providers/amazon/aws/links/test_step_function.py b/providers/tests/amazon/aws/links/test_step_function.py similarity index 97% rename from tests/providers/amazon/aws/links/test_step_function.py rename to providers/tests/amazon/aws/links/test_step_function.py index 3c6c9cc7cdc27..fab1205689dc9 100644 --- a/tests/providers/amazon/aws/links/test_step_function.py +++ b/providers/tests/amazon/aws/links/test_step_function.py @@ -22,7 +22,8 @@ StateMachineDetailsLink, StateMachineExecutionsDetailsLink, ) -from tests.providers.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase + +from providers.tests.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase class TestStateMachineDetailsLink(BaseAwsLinksTestCase): diff --git a/tests/integration/providers/redis/sensors/__init__.py b/providers/tests/amazon/aws/log/__init__.py similarity index 100% rename from tests/integration/providers/redis/sensors/__init__.py rename to providers/tests/amazon/aws/log/__init__.py diff --git a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py similarity index 99% rename from tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py rename to providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py index 801935a5496af..c78fab89e197a 100644 --- a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py +++ b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py @@ -35,7 +35,8 @@ from airflow.providers.amazon.aws.utils import datetime_to_epoch_utc_ms from airflow.utils.state import State from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars def get_time_str(time_in_milliseconds): diff --git a/tests/providers/amazon/aws/log/test_s3_task_handler.py b/providers/tests/amazon/aws/log/test_s3_task_handler.py similarity index 99% rename from tests/providers/amazon/aws/log/test_s3_task_handler.py rename to providers/tests/amazon/aws/log/test_s3_task_handler.py index fcb5d3c7833b4..9819cf95e9cc1 100644 --- a/tests/providers/amazon/aws/log/test_s3_task_handler.py +++ b/providers/tests/amazon/aws/log/test_s3_task_handler.py @@ -33,7 +33,8 @@ from airflow.providers.amazon.aws.log.s3_task_handler import S3TaskHandler from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars @pytest.fixture(autouse=True) diff --git a/tests/integration/providers/trino/__init__.py b/providers/tests/amazon/aws/notifications/__init__.py similarity index 100% rename from tests/integration/providers/trino/__init__.py rename to providers/tests/amazon/aws/notifications/__init__.py diff --git a/tests/providers/amazon/aws/notifications/test_chime.py b/providers/tests/amazon/aws/notifications/test_chime.py similarity index 100% rename from tests/providers/amazon/aws/notifications/test_chime.py rename to providers/tests/amazon/aws/notifications/test_chime.py diff --git a/tests/providers/amazon/aws/notifications/test_sns.py b/providers/tests/amazon/aws/notifications/test_sns.py similarity index 100% rename from tests/providers/amazon/aws/notifications/test_sns.py rename to providers/tests/amazon/aws/notifications/test_sns.py diff --git a/tests/providers/amazon/aws/notifications/test_sqs.py b/providers/tests/amazon/aws/notifications/test_sqs.py similarity index 100% rename from tests/providers/amazon/aws/notifications/test_sqs.py rename to providers/tests/amazon/aws/notifications/test_sqs.py diff --git a/tests/integration/providers/trino/hooks/__init__.py b/providers/tests/amazon/aws/operators/__init__.py similarity index 100% rename from tests/integration/providers/trino/hooks/__init__.py rename to providers/tests/amazon/aws/operators/__init__.py diff --git a/tests/providers/amazon/aws/operators/athena_metadata.json b/providers/tests/amazon/aws/operators/athena_metadata.json similarity index 100% rename from tests/providers/amazon/aws/operators/athena_metadata.json rename to providers/tests/amazon/aws/operators/athena_metadata.json diff --git a/tests/providers/amazon/aws/operators/test_appflow.py b/providers/tests/amazon/aws/operators/test_appflow.py similarity index 100% rename from tests/providers/amazon/aws/operators/test_appflow.py rename to providers/tests/amazon/aws/operators/test_appflow.py diff --git a/tests/providers/amazon/aws/operators/test_athena.py b/providers/tests/amazon/aws/operators/test_athena.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_athena.py rename to providers/tests/amazon/aws/operators/test_athena.py index 102d1fe31e5c1..e4d7d6622a091 100644 --- a/tests/providers/amazon/aws/operators/test_athena.py +++ b/providers/tests/amazon/aws/operators/test_athena.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +import os from unittest import mock import pytest @@ -39,7 +40,8 @@ from airflow.utils import timezone from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields TEST_DAG_ID = "unit_tests" DEFAULT_DATE = datetime(2018, 1, 1) @@ -299,7 +301,7 @@ def test_operator_openlineage_data(self, mock_conn, mock_region_name): mock_region_name.return_value = "eu-west-1" def mock_get_table_metadata(CatalogName, DatabaseName, TableName): - with open("tests/providers/amazon/aws/operators/athena_metadata.json") as f: + with open(os.path.dirname(__file__) + "/athena_metadata.json") as f: return json.load(f)[TableName] mock_conn.return_value.get_table_metadata = mock_get_table_metadata diff --git a/tests/providers/amazon/aws/operators/test_base_aws.py b/providers/tests/amazon/aws/operators/test_base_aws.py similarity index 100% rename from tests/providers/amazon/aws/operators/test_base_aws.py rename to providers/tests/amazon/aws/operators/test_base_aws.py diff --git a/tests/providers/amazon/aws/operators/test_batch.py b/providers/tests/amazon/aws/operators/test_batch.py similarity index 100% rename from tests/providers/amazon/aws/operators/test_batch.py rename to providers/tests/amazon/aws/operators/test_batch.py diff --git a/tests/providers/amazon/aws/operators/test_bedrock.py b/providers/tests/amazon/aws/operators/test_bedrock.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_bedrock.py rename to providers/tests/amazon/aws/operators/test_bedrock.py index 8cbb67d6f50df..5e8cd32e1a4d4 100644 --- a/tests/providers/amazon/aws/operators/test_bedrock.py +++ b/providers/tests/amazon/aws/operators/test_bedrock.py @@ -35,7 +35,8 @@ BedrockInvokeModelOperator, BedrockRaGOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection diff --git a/tests/providers/amazon/aws/operators/test_cloud_formation.py b/providers/tests/amazon/aws/operators/test_cloud_formation.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_cloud_formation.py rename to providers/tests/amazon/aws/operators/test_cloud_formation.py index 4d8fb4d12bd3c..47b1659b426f0 100644 --- a/tests/providers/amazon/aws/operators/test_cloud_formation.py +++ b/providers/tests/amazon/aws/operators/test_cloud_formation.py @@ -28,7 +28,8 @@ CloudFormationDeleteStackOperator, ) from airflow.utils import timezone -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields DEFAULT_DATE = timezone.datetime(2019, 1, 1) DEFAULT_ARGS = {"owner": "airflow", "start_date": DEFAULT_DATE} diff --git a/tests/providers/amazon/aws/operators/test_comprehend.py b/providers/tests/amazon/aws/operators/test_comprehend.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_comprehend.py rename to providers/tests/amazon/aws/operators/test_comprehend.py index a86b779b1d502..170f7b6760823 100644 --- a/tests/providers/amazon/aws/operators/test_comprehend.py +++ b/providers/tests/amazon/aws/operators/test_comprehend.py @@ -29,7 +29,8 @@ ComprehendStartPiiEntitiesDetectionJobOperator, ) from airflow.utils.types import NOTSET -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection diff --git a/tests/providers/amazon/aws/operators/test_datasync.py b/providers/tests/amazon/aws/operators/test_datasync.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_datasync.py rename to providers/tests/amazon/aws/operators/test_datasync.py index 18b0e86103c0b..52c5863b8aa75 100644 --- a/tests/providers/amazon/aws/operators/test_datasync.py +++ b/providers/tests/amazon/aws/operators/test_datasync.py @@ -29,7 +29,8 @@ from airflow.utils import timezone from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields TEST_DAG_ID = "unit_tests" DEFAULT_DATE = datetime(2018, 1, 1) diff --git a/tests/providers/amazon/aws/operators/test_dms.py b/providers/tests/amazon/aws/operators/test_dms.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_dms.py rename to providers/tests/amazon/aws/operators/test_dms.py index 2528edaef9e0a..0c99dc2cd9334 100644 --- a/tests/providers/amazon/aws/operators/test_dms.py +++ b/providers/tests/amazon/aws/operators/test_dms.py @@ -34,7 +34,8 @@ ) from airflow.utils import timezone from airflow.utils.types import DagRunType -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields TASK_ARN = "test_arn" diff --git a/tests/providers/amazon/aws/operators/test_ec2.py b/providers/tests/amazon/aws/operators/test_ec2.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_ec2.py rename to providers/tests/amazon/aws/operators/test_ec2.py index a5ea81ff6ae87..b4b576c6567f6 100644 --- a/tests/providers/amazon/aws/operators/test_ec2.py +++ b/providers/tests/amazon/aws/operators/test_ec2.py @@ -30,7 +30,8 @@ EC2StopInstanceOperator, EC2TerminateInstanceOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields class BaseEc2TestClass: diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/providers/tests/amazon/aws/operators/test_ecs.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_ecs.py rename to providers/tests/amazon/aws/operators/test_ecs.py index d7b6c0d4e8716..ed900acb73644 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/providers/tests/amazon/aws/operators/test_ecs.py @@ -38,7 +38,8 @@ from airflow.providers.amazon.aws.triggers.ecs import TaskDoneTrigger from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher from airflow.utils.types import NOTSET -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CLUSTER_NAME = "test_cluster" CONTAINER_NAME = "e1ed7aac-d9b2-4315-8726-d2432bf11868" diff --git a/tests/providers/amazon/aws/operators/test_eks.py b/providers/tests/amazon/aws/operators/test_eks.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_eks.py rename to providers/tests/amazon/aws/operators/test_eks.py index 2daa484626879..8105614849a3f 100644 --- a/tests/providers/amazon/aws/operators/test_eks.py +++ b/providers/tests/amazon/aws/operators/test_eks.py @@ -41,7 +41,8 @@ ) from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction from airflow.typing_compat import TypedDict -from tests.providers.amazon.aws.utils.eks_test_constants import ( + +from providers.tests.amazon.aws.utils.eks_test_constants import ( NODEROLE_ARN, POD_EXECUTION_ROLE_ARN, RESOURCES_VPC_CONFIG, @@ -50,9 +51,9 @@ SUBNET_IDS, TASK_ID, ) -from tests.providers.amazon.aws.utils.eks_test_utils import convert_keys -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type +from providers.tests.amazon.aws.utils.eks_test_utils import convert_keys +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type CLUSTER_NAME = "cluster1" NODEGROUP_NAME = "nodegroup1" diff --git a/tests/providers/amazon/aws/operators/test_emr_add_steps.py b/providers/tests/amazon/aws/operators/test_emr_add_steps.py similarity index 97% rename from tests/providers/amazon/aws/operators/test_emr_add_steps.py rename to providers/tests/amazon/aws/operators/test_emr_add_steps.py index d5a999349aa53..4414ae4327070 100644 --- a/tests/providers/amazon/aws/operators/test_emr_add_steps.py +++ b/providers/tests/amazon/aws/operators/test_emr_add_steps.py @@ -18,8 +18,8 @@ from __future__ import annotations import json -import os from datetime import timedelta +from pathlib import Path from unittest.mock import MagicMock, call, patch import pytest @@ -31,16 +31,14 @@ from airflow.providers.amazon.aws.triggers.emr import EmrAddStepsTrigger from airflow.utils import timezone from airflow.utils.types import DagRunType -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields -from tests.test_utils import AIRFLOW_MAIN_FOLDER + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields DEFAULT_DATE = timezone.datetime(2017, 1, 1) ADD_STEPS_SUCCESS_RETURN = {"ResponseMetadata": {"HTTPStatusCode": 200}, "StepIds": ["s-2LH3R5GW3A53T"]} -TEMPLATE_SEARCHPATH = os.path.join( - AIRFLOW_MAIN_FOLDER, "tests", "providers", "amazon", "aws", "config_templates" -) +TEMPLATE_SEARCHPATH = Path(__file__).parents[1].joinpath("config_templates").as_posix() @pytest.fixture diff --git a/tests/providers/amazon/aws/operators/test_emr_containers.py b/providers/tests/amazon/aws/operators/test_emr_containers.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_emr_containers.py rename to providers/tests/amazon/aws/operators/test_emr_containers.py index 52306864f3597..b31364bb02f50 100644 --- a/tests/providers/amazon/aws/operators/test_emr_containers.py +++ b/providers/tests/amazon/aws/operators/test_emr_containers.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.operators.emr import EmrContainerOperator, EmrEksCreateClusterOperator from airflow.providers.amazon.aws.triggers.emr import EmrContainerTrigger -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields SUBMIT_JOB_SUCCESS_RETURN = { "ResponseMetadata": {"HTTPStatusCode": 200}, diff --git a/tests/providers/amazon/aws/operators/test_emr_create_job_flow.py b/providers/tests/amazon/aws/operators/test_emr_create_job_flow.py similarity index 95% rename from tests/providers/amazon/aws/operators/test_emr_create_job_flow.py rename to providers/tests/amazon/aws/operators/test_emr_create_job_flow.py index 860df8c7219ac..b2f7c8eb48f7b 100644 --- a/tests/providers/amazon/aws/operators/test_emr_create_job_flow.py +++ b/providers/tests/amazon/aws/operators/test_emr_create_job_flow.py @@ -17,8 +17,8 @@ # under the License. from __future__ import annotations -import os from datetime import timedelta +from pathlib import Path from unittest import mock from unittest.mock import MagicMock, patch @@ -32,9 +32,9 @@ from airflow.providers.amazon.aws.triggers.emr import EmrCreateJobFlowTrigger from airflow.utils import timezone from airflow.utils.types import DagRunType -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type -from tests.test_utils import AIRFLOW_MAIN_FOLDER + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type TASK_ID = "test_task" @@ -45,9 +45,7 @@ JOB_FLOW_ID = "j-8989898989" RUN_JOB_FLOW_SUCCESS_RETURN = {"ResponseMetadata": {"HTTPStatusCode": 200}, "JobFlowId": JOB_FLOW_ID} -TEMPLATE_SEARCHPATH = os.path.join( - AIRFLOW_MAIN_FOLDER, "tests", "providers", "amazon", "aws", "config_templates" -) +TEMPLATE_SEARCHPATH = Path(__file__).parents[1].joinpath("config_templates").as_posix() @pytest.fixture diff --git a/tests/providers/amazon/aws/operators/test_emr_modify_cluster.py b/providers/tests/amazon/aws/operators/test_emr_modify_cluster.py similarity index 97% rename from tests/providers/amazon/aws/operators/test_emr_modify_cluster.py rename to providers/tests/amazon/aws/operators/test_emr_modify_cluster.py index 6f257288760c3..4c7aae9b4c272 100644 --- a/tests/providers/amazon/aws/operators/test_emr_modify_cluster.py +++ b/providers/tests/amazon/aws/operators/test_emr_modify_cluster.py @@ -25,7 +25,8 @@ from airflow.models.dag import DAG from airflow.providers.amazon.aws.operators.emr import EmrModifyClusterOperator from airflow.utils import timezone -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields DEFAULT_DATE = timezone.datetime(2017, 1, 1) MODIFY_CLUSTER_SUCCESS_RETURN = {"ResponseMetadata": {"HTTPStatusCode": 200}, "StepConcurrencyLevel": 1} diff --git a/tests/providers/amazon/aws/operators/test_emr_notebook_execution.py b/providers/tests/amazon/aws/operators/test_emr_notebook_execution.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_emr_notebook_execution.py rename to providers/tests/amazon/aws/operators/test_emr_notebook_execution.py index 6fcd4eeb74629..42ce47ea6b699 100644 --- a/tests/providers/amazon/aws/operators/test_emr_notebook_execution.py +++ b/providers/tests/amazon/aws/operators/test_emr_notebook_execution.py @@ -28,8 +28,9 @@ EmrStartNotebookExecutionOperator, EmrStopNotebookExecutionOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type PARAMS = { "EditorId": "test_editor", diff --git a/tests/providers/amazon/aws/operators/test_emr_serverless.py b/providers/tests/amazon/aws/operators/test_emr_serverless.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_emr_serverless.py rename to providers/tests/amazon/aws/operators/test_emr_serverless.py index c84d1032bcf71..bde6e9895f5c9 100644 --- a/tests/providers/amazon/aws/operators/test_emr_serverless.py +++ b/providers/tests/amazon/aws/operators/test_emr_serverless.py @@ -32,7 +32,8 @@ EmrServerlessStopApplicationOperator, ) from airflow.utils.types import NOTSET -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from unittest.mock import MagicMock diff --git a/tests/providers/amazon/aws/operators/test_emr_terminate_job_flow.py b/providers/tests/amazon/aws/operators/test_emr_terminate_job_flow.py similarity index 97% rename from tests/providers/amazon/aws/operators/test_emr_terminate_job_flow.py rename to providers/tests/amazon/aws/operators/test_emr_terminate_job_flow.py index 06ab35e4510ba..6ce3fb29cd6b2 100644 --- a/tests/providers/amazon/aws/operators/test_emr_terminate_job_flow.py +++ b/providers/tests/amazon/aws/operators/test_emr_terminate_job_flow.py @@ -24,7 +24,8 @@ from airflow.exceptions import TaskDeferred from airflow.providers.amazon.aws.operators.emr import EmrTerminateJobFlowOperator from airflow.providers.amazon.aws.triggers.emr import EmrTerminateJobFlowTrigger -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields TERMINATE_SUCCESS_RETURN = {"ResponseMetadata": {"HTTPStatusCode": 200}} diff --git a/tests/providers/amazon/aws/operators/test_eventbridge.py b/providers/tests/amazon/aws/operators/test_eventbridge.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_eventbridge.py rename to providers/tests/amazon/aws/operators/test_eventbridge.py index 3c682f1477e03..129edf9ff6247 100644 --- a/tests/providers/amazon/aws/operators/test_eventbridge.py +++ b/providers/tests/amazon/aws/operators/test_eventbridge.py @@ -29,7 +29,8 @@ EventBridgePutEventsOperator, EventBridgePutRuleOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from unittest.mock import MagicMock diff --git a/tests/providers/amazon/aws/operators/test_glacier.py b/providers/tests/amazon/aws/operators/test_glacier.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_glacier.py rename to providers/tests/amazon/aws/operators/test_glacier.py index f46b0bc929fe8..ff4d220076f89 100644 --- a/tests/providers/amazon/aws/operators/test_glacier.py +++ b/providers/tests/amazon/aws/operators/test_glacier.py @@ -26,7 +26,8 @@ GlacierCreateJobOperator, GlacierUploadArchiveOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator diff --git a/tests/providers/amazon/aws/operators/test_glue.py b/providers/tests/amazon/aws/operators/test_glue.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_glue.py rename to providers/tests/amazon/aws/operators/test_glue.py index e1adcee7d639f..8243940e695fd 100644 --- a/tests/providers/amazon/aws/operators/test_glue.py +++ b/providers/tests/amazon/aws/operators/test_glue.py @@ -34,7 +34,8 @@ GlueDataQualityRuleSetEvaluationRunOperator, GlueJobOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from airflow.models import TaskInstance diff --git a/tests/providers/amazon/aws/operators/test_glue_crawler.py b/providers/tests/amazon/aws/operators/test_glue_crawler.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_glue_crawler.py rename to providers/tests/amazon/aws/operators/test_glue_crawler.py index 1e5a3f2177b5b..1df63a4e2ad05 100644 --- a/tests/providers/amazon/aws/operators/test_glue_crawler.py +++ b/providers/tests/amazon/aws/operators/test_glue_crawler.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.hooks.glue_crawler import GlueCrawlerHook from airflow.providers.amazon.aws.hooks.sts import StsHook from airflow.providers.amazon.aws.operators.glue_crawler import GlueCrawlerOperator -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection diff --git a/tests/providers/amazon/aws/operators/test_glue_databrew.py b/providers/tests/amazon/aws/operators/test_glue_databrew.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_glue_databrew.py rename to providers/tests/amazon/aws/operators/test_glue_databrew.py index 698b206acfb1c..1b46549df4545 100644 --- a/tests/providers/amazon/aws/operators/test_glue_databrew.py +++ b/providers/tests/amazon/aws/operators/test_glue_databrew.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.hooks.glue_databrew import GlueDataBrewHook from airflow.providers.amazon.aws.operators.glue_databrew import GlueDataBrewStartJobOperator -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields JOB_NAME = "test_job" diff --git a/tests/providers/amazon/aws/operators/test_kinesis_analytics.py b/providers/tests/amazon/aws/operators/test_kinesis_analytics.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_kinesis_analytics.py rename to providers/tests/amazon/aws/operators/test_kinesis_analytics.py index ab8bb3123008d..b6b92e4c0b134 100644 --- a/tests/providers/amazon/aws/operators/test_kinesis_analytics.py +++ b/providers/tests/amazon/aws/operators/test_kinesis_analytics.py @@ -30,7 +30,8 @@ KinesisAnalyticsV2StartApplicationOperator, KinesisAnalyticsV2StopApplicationOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection diff --git a/tests/providers/amazon/aws/operators/test_lambda_function.py b/providers/tests/amazon/aws/operators/test_lambda_function.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_lambda_function.py rename to providers/tests/amazon/aws/operators/test_lambda_function.py index e3a5b8cad6201..7ec081969103a 100644 --- a/tests/providers/amazon/aws/operators/test_lambda_function.py +++ b/providers/tests/amazon/aws/operators/test_lambda_function.py @@ -29,7 +29,8 @@ LambdaCreateFunctionOperator, LambdaInvokeFunctionOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields FUNCTION_NAME = "function_name" PAYLOADS = [ diff --git a/tests/providers/amazon/aws/operators/test_neptune.py b/providers/tests/amazon/aws/operators/test_neptune.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_neptune.py rename to providers/tests/amazon/aws/operators/test_neptune.py index 146effaaa20f7..0606f06e053a1 100644 --- a/tests/providers/amazon/aws/operators/test_neptune.py +++ b/providers/tests/amazon/aws/operators/test_neptune.py @@ -30,7 +30,8 @@ NeptuneStartDbClusterOperator, NeptuneStopDbClusterOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CLUSTER_ID = "test_cluster" diff --git a/tests/providers/amazon/aws/operators/test_quicksight.py b/providers/tests/amazon/aws/operators/test_quicksight.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_quicksight.py rename to providers/tests/amazon/aws/operators/test_quicksight.py index f2d23c7b81793..063c5fc44a464 100644 --- a/tests/providers/amazon/aws/operators/test_quicksight.py +++ b/providers/tests/amazon/aws/operators/test_quicksight.py @@ -21,7 +21,8 @@ from airflow.providers.amazon.aws.hooks.quicksight import QuickSightHook from airflow.providers.amazon.aws.operators.quicksight import QuickSightCreateIngestionOperator -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields DATA_SET_ID = "DemoDataSet" INGESTION_ID = "DemoDataSet_Ingestion" diff --git a/tests/providers/amazon/aws/operators/test_rds.py b/providers/tests/amazon/aws/operators/test_rds.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_rds.py rename to providers/tests/amazon/aws/operators/test_rds.py index 0574d4b553b48..3d0fbcb84096d 100644 --- a/tests/providers/amazon/aws/operators/test_rds.py +++ b/providers/tests/amazon/aws/operators/test_rds.py @@ -44,7 +44,8 @@ ) from airflow.providers.amazon.aws.triggers.rds import RdsDbAvailableTrigger, RdsDbStoppedTrigger from airflow.utils import timezone -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook diff --git a/tests/providers/amazon/aws/operators/test_redshift_cluster.py b/providers/tests/amazon/aws/operators/test_redshift_cluster.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_redshift_cluster.py rename to providers/tests/amazon/aws/operators/test_redshift_cluster.py index e48f7b2ed96ea..e67bff3390c08 100644 --- a/tests/providers/amazon/aws/operators/test_redshift_cluster.py +++ b/providers/tests/amazon/aws/operators/test_redshift_cluster.py @@ -38,7 +38,8 @@ RedshiftPauseClusterTrigger, RedshiftResumeClusterTrigger, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields class TestRedshiftCreateClusterOperator: diff --git a/tests/providers/amazon/aws/operators/test_redshift_data.py b/providers/tests/amazon/aws/operators/test_redshift_data.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_redshift_data.py rename to providers/tests/amazon/aws/operators/test_redshift_data.py index c4972e4c42e7e..d367fc3ca9596 100644 --- a/tests/providers/amazon/aws/operators/test_redshift_data.py +++ b/providers/tests/amazon/aws/operators/test_redshift_data.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.hooks.redshift_data import QueryExecutionOutput from airflow.providers.amazon.aws.operators.redshift_data import RedshiftDataOperator from airflow.providers.amazon.aws.triggers.redshift_data import RedshiftDataTrigger -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CONN_ID = "aws_conn_test" TASK_ID = "task_id" diff --git a/tests/providers/amazon/aws/operators/test_redshift_sql.py b/providers/tests/amazon/aws/operators/test_redshift_sql.py similarity index 100% rename from tests/providers/amazon/aws/operators/test_redshift_sql.py rename to providers/tests/amazon/aws/operators/test_redshift_sql.py diff --git a/tests/providers/amazon/aws/operators/test_s3.py b/providers/tests/amazon/aws/operators/test_s3.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_s3.py rename to providers/tests/amazon/aws/operators/test_s3.py index 937baefde59ea..8c6dbd2df50fc 100644 --- a/tests/providers/amazon/aws/operators/test_s3.py +++ b/providers/tests/amazon/aws/operators/test_s3.py @@ -57,7 +57,8 @@ from airflow.providers.openlineage.extractors import OperatorLineage from airflow.utils.timezone import datetime, utcnow from airflow.utils.types import DagRunType -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields BUCKET_NAME = os.environ.get("BUCKET_NAME", "test-airflow-bucket") S3_KEY = "test-airflow-key" diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_base.py b/providers/tests/amazon/aws/operators/test_sagemaker_base.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_sagemaker_base.py rename to providers/tests/amazon/aws/operators/test_sagemaker_base.py index 5de40708d5158..e6e1fcd5bd90c 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_base.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_base.py @@ -32,7 +32,8 @@ ) from airflow.utils import timezone from airflow.utils.types import DagRunType -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CONFIG: dict = { "key1": "1", diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_endpoint.py b/providers/tests/amazon/aws/operators/test_sagemaker_endpoint.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_sagemaker_endpoint.py rename to providers/tests/amazon/aws/operators/test_sagemaker_endpoint.py index 24cf944f8db6a..c24cf39e79c3a 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_endpoint.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_endpoint.py @@ -27,7 +27,8 @@ from airflow.providers.amazon.aws.operators import sagemaker from airflow.providers.amazon.aws.operators.sagemaker import SageMakerEndpointOperator from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CREATE_MODEL_PARAMS: dict = { "ModelName": "model_name", diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_endpoint_config.py b/providers/tests/amazon/aws/operators/test_sagemaker_endpoint_config.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_sagemaker_endpoint_config.py rename to providers/tests/amazon/aws/operators/test_sagemaker_endpoint_config.py index 1169f09d9141d..c62721fcf3fe9 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_endpoint_config.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_endpoint_config.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook from airflow.providers.amazon.aws.operators import sagemaker from airflow.providers.amazon.aws.operators.sagemaker import SageMakerEndpointConfigOperator -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CREATE_ENDPOINT_CONFIG_PARAMS: dict = { "EndpointConfigName": "config_name", diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_model.py b/providers/tests/amazon/aws/operators/test_sagemaker_model.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_sagemaker_model.py rename to providers/tests/amazon/aws/operators/test_sagemaker_model.py index 33d1f5b4d1f6f..095c6990ce51a 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_model.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_model.py @@ -31,7 +31,8 @@ SageMakerModelOperator, SageMakerRegisterModelVersionOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CREATE_MODEL_PARAMS: dict = { "ModelName": "model_name", diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_notebook.py b/providers/tests/amazon/aws/operators/test_sagemaker_notebook.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_sagemaker_notebook.py rename to providers/tests/amazon/aws/operators/test_sagemaker_notebook.py index 093e264a3e454..0593b0f798429 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_notebook.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_notebook.py @@ -30,7 +30,8 @@ SageMakerStartNoteBookOperator, SageMakerStopNotebookOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields INSTANCE_NAME = "notebook" INSTANCE_TYPE = "ml.t3.medium" diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_pipeline.py b/providers/tests/amazon/aws/operators/test_sagemaker_pipeline.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_sagemaker_pipeline.py rename to providers/tests/amazon/aws/operators/test_sagemaker_pipeline.py index e7334de98df03..0a2d4d00ad85d 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_pipeline.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_pipeline.py @@ -29,7 +29,8 @@ SageMakerStopPipelineOperator, ) from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerPipelineTrigger -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields if TYPE_CHECKING: from unittest.mock import MagicMock diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_processing.py b/providers/tests/amazon/aws/operators/test_sagemaker_processing.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_sagemaker_processing.py rename to providers/tests/amazon/aws/operators/test_sagemaker_processing.py index b1ca2b62adbb9..898aa68a65100 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_processing.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_processing.py @@ -31,7 +31,8 @@ from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger from airflow.providers.common.compat.openlineage.facet import Dataset from airflow.providers.openlineage.extractors import OperatorLineage -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields CREATE_PROCESSING_PARAMS: dict = { "AppSpecification": { diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_training.py b/providers/tests/amazon/aws/operators/test_sagemaker_training.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_sagemaker_training.py rename to providers/tests/amazon/aws/operators/test_sagemaker_training.py index 85c6954ac1ac5..f37a41967b9cc 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_training.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_training.py @@ -31,7 +31,8 @@ ) from airflow.providers.common.compat.openlineage.facet import Dataset from airflow.providers.openlineage.extractors import OperatorLineage -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields EXPECTED_INTEGER_FIELDS: list[list[str]] = [ ["ResourceConfig", "InstanceCount"], diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_transform.py b/providers/tests/amazon/aws/operators/test_sagemaker_transform.py similarity index 99% rename from tests/providers/amazon/aws/operators/test_sagemaker_transform.py rename to providers/tests/amazon/aws/operators/test_sagemaker_transform.py index 7804ddde20400..2452558ec4228 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_transform.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_transform.py @@ -30,7 +30,8 @@ from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger from airflow.providers.common.compat.openlineage.facet import Dataset from airflow.providers.openlineage.extractors import OperatorLineage -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields EXPECTED_INTEGER_FIELDS: list[list[str]] = [ ["Transform", "TransformResources", "InstanceCount"], diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_tuning.py b/providers/tests/amazon/aws/operators/test_sagemaker_tuning.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_sagemaker_tuning.py rename to providers/tests/amazon/aws/operators/test_sagemaker_tuning.py index 78058c771a28d..9048b19b0ec71 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_tuning.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_tuning.py @@ -26,7 +26,8 @@ from airflow.providers.amazon.aws.operators import sagemaker from airflow.providers.amazon.aws.operators.sagemaker import SageMakerTuningOperator from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields EXPECTED_INTEGER_FIELDS: list[list[str]] = [ ["HyperParameterTuningJobConfig", "ResourceLimits", "MaxNumberOfTrainingJobs"], diff --git a/tests/providers/amazon/aws/operators/test_sns.py b/providers/tests/amazon/aws/operators/test_sns.py similarity index 97% rename from tests/providers/amazon/aws/operators/test_sns.py rename to providers/tests/amazon/aws/operators/test_sns.py index 6c5de06822d3a..d2571f6cc1866 100644 --- a/tests/providers/amazon/aws/operators/test_sns.py +++ b/providers/tests/amazon/aws/operators/test_sns.py @@ -22,7 +22,8 @@ import pytest from airflow.providers.amazon.aws.operators.sns import SnsPublishOperator -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields TASK_ID = "sns_publish_job" AWS_CONN_ID = "custom_aws_conn" diff --git a/tests/providers/amazon/aws/operators/test_sqs.py b/providers/tests/amazon/aws/operators/test_sqs.py similarity index 98% rename from tests/providers/amazon/aws/operators/test_sqs.py rename to providers/tests/amazon/aws/operators/test_sqs.py index 2187262fe7c36..4534a16bff7ca 100644 --- a/tests/providers/amazon/aws/operators/test_sqs.py +++ b/providers/tests/amazon/aws/operators/test_sqs.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.hooks.sqs import SqsHook from airflow.providers.amazon.aws.operators.sqs import SqsPublishOperator -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields REGION_NAME = "eu-west-1" QUEUE_NAME = "test-queue" diff --git a/tests/providers/amazon/aws/operators/test_step_function.py b/providers/tests/amazon/aws/operators/test_step_function.py similarity index 95% rename from tests/providers/amazon/aws/operators/test_step_function.py rename to providers/tests/amazon/aws/operators/test_step_function.py index 29d743996af48..cac611381ee74 100644 --- a/tests/providers/amazon/aws/operators/test_step_function.py +++ b/providers/tests/amazon/aws/operators/test_step_function.py @@ -26,7 +26,8 @@ StepFunctionGetExecutionOutputOperator, StepFunctionStartExecutionOperator, ) -from tests.providers.amazon.aws.utils.test_template_fields import validate_template_fields + +from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields EXECUTION_ARN = ( "arn:aws:states:us-east-1:123456789012:execution:" @@ -117,11 +118,14 @@ class TestStepFunctionStartExecutionOperator: @pytest.fixture(autouse=True) def setup_test_cases(self): - with mock.patch( - "airflow.providers.amazon.aws.links.step_function.StateMachineExecutionsDetailsLink.persist" - ) as executions_details_link, mock.patch( - "airflow.providers.amazon.aws.links.step_function.StateMachineDetailsLink.persist" - ) as details_link: + with ( + mock.patch( + "airflow.providers.amazon.aws.links.step_function.StateMachineExecutionsDetailsLink.persist" + ) as executions_details_link, + mock.patch( + "airflow.providers.amazon.aws.links.step_function.StateMachineDetailsLink.persist" + ) as details_link, + ): self.mocked_executions_details_link = executions_details_link self.mocked_details_link = details_link yield diff --git a/tests/integration/providers/ydb/__init__.py b/providers/tests/amazon/aws/secrets/__init__.py similarity index 100% rename from tests/integration/providers/ydb/__init__.py rename to providers/tests/amazon/aws/secrets/__init__.py diff --git a/tests/providers/amazon/aws/secrets/test_secrets_manager.py b/providers/tests/amazon/aws/secrets/test_secrets_manager.py similarity index 100% rename from tests/providers/amazon/aws/secrets/test_secrets_manager.py rename to providers/tests/amazon/aws/secrets/test_secrets_manager.py diff --git a/tests/providers/amazon/aws/secrets/test_systems_manager.py b/providers/tests/amazon/aws/secrets/test_systems_manager.py similarity index 99% rename from tests/providers/amazon/aws/secrets/test_systems_manager.py rename to providers/tests/amazon/aws/secrets/test_systems_manager.py index d4c9c0f9bc136..c1b35a799d3c9 100644 --- a/tests/providers/amazon/aws/secrets/test_systems_manager.py +++ b/providers/tests/amazon/aws/secrets/test_systems_manager.py @@ -24,7 +24,8 @@ from airflow.configuration import initialize_secrets_backends from airflow.providers.amazon.aws.secrets.systems_manager import SystemsManagerParameterStoreBackend -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars URI_CONNECTION = pytest.param( "postgres://my-login:my-pass@my-host:5432/my-schema?param1=val1¶m2=val2", id="uri-connection" diff --git a/tests/integration/providers/ydb/hooks/__init__.py b/providers/tests/amazon/aws/sensors/__init__.py similarity index 100% rename from tests/integration/providers/ydb/hooks/__init__.py rename to providers/tests/amazon/aws/sensors/__init__.py diff --git a/tests/providers/amazon/aws/sensors/test_athena.py b/providers/tests/amazon/aws/sensors/test_athena.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_athena.py rename to providers/tests/amazon/aws/sensors/test_athena.py diff --git a/tests/providers/amazon/aws/sensors/test_base_aws.py b/providers/tests/amazon/aws/sensors/test_base_aws.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_base_aws.py rename to providers/tests/amazon/aws/sensors/test_base_aws.py diff --git a/tests/providers/amazon/aws/sensors/test_batch.py b/providers/tests/amazon/aws/sensors/test_batch.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_batch.py rename to providers/tests/amazon/aws/sensors/test_batch.py diff --git a/tests/providers/amazon/aws/sensors/test_bedrock.py b/providers/tests/amazon/aws/sensors/test_bedrock.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_bedrock.py rename to providers/tests/amazon/aws/sensors/test_bedrock.py diff --git a/tests/providers/amazon/aws/sensors/test_cloud_formation.py b/providers/tests/amazon/aws/sensors/test_cloud_formation.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_cloud_formation.py rename to providers/tests/amazon/aws/sensors/test_cloud_formation.py diff --git a/tests/providers/amazon/aws/sensors/test_comprehend.py b/providers/tests/amazon/aws/sensors/test_comprehend.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_comprehend.py rename to providers/tests/amazon/aws/sensors/test_comprehend.py diff --git a/tests/providers/amazon/aws/sensors/test_dms.py b/providers/tests/amazon/aws/sensors/test_dms.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_dms.py rename to providers/tests/amazon/aws/sensors/test_dms.py diff --git a/tests/providers/amazon/aws/sensors/test_dynamodb.py b/providers/tests/amazon/aws/sensors/test_dynamodb.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_dynamodb.py rename to providers/tests/amazon/aws/sensors/test_dynamodb.py diff --git a/tests/providers/amazon/aws/sensors/test_ec2.py b/providers/tests/amazon/aws/sensors/test_ec2.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_ec2.py rename to providers/tests/amazon/aws/sensors/test_ec2.py diff --git a/tests/providers/amazon/aws/sensors/test_ecs.py b/providers/tests/amazon/aws/sensors/test_ecs.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_ecs.py rename to providers/tests/amazon/aws/sensors/test_ecs.py diff --git a/tests/providers/amazon/aws/sensors/test_eks.py b/providers/tests/amazon/aws/sensors/test_eks.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_eks.py rename to providers/tests/amazon/aws/sensors/test_eks.py diff --git a/tests/providers/amazon/aws/sensors/test_emr_base.py b/providers/tests/amazon/aws/sensors/test_emr_base.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_emr_base.py rename to providers/tests/amazon/aws/sensors/test_emr_base.py diff --git a/tests/providers/amazon/aws/sensors/test_emr_containers.py b/providers/tests/amazon/aws/sensors/test_emr_containers.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_emr_containers.py rename to providers/tests/amazon/aws/sensors/test_emr_containers.py diff --git a/tests/providers/amazon/aws/sensors/test_emr_job_flow.py b/providers/tests/amazon/aws/sensors/test_emr_job_flow.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_emr_job_flow.py rename to providers/tests/amazon/aws/sensors/test_emr_job_flow.py diff --git a/tests/providers/amazon/aws/sensors/test_emr_notebook_execution.py b/providers/tests/amazon/aws/sensors/test_emr_notebook_execution.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_emr_notebook_execution.py rename to providers/tests/amazon/aws/sensors/test_emr_notebook_execution.py diff --git a/tests/providers/amazon/aws/sensors/test_emr_serverless_application.py b/providers/tests/amazon/aws/sensors/test_emr_serverless_application.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_emr_serverless_application.py rename to providers/tests/amazon/aws/sensors/test_emr_serverless_application.py diff --git a/tests/providers/amazon/aws/sensors/test_emr_serverless_job.py b/providers/tests/amazon/aws/sensors/test_emr_serverless_job.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_emr_serverless_job.py rename to providers/tests/amazon/aws/sensors/test_emr_serverless_job.py diff --git a/tests/providers/amazon/aws/sensors/test_emr_step.py b/providers/tests/amazon/aws/sensors/test_emr_step.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_emr_step.py rename to providers/tests/amazon/aws/sensors/test_emr_step.py diff --git a/tests/providers/amazon/aws/sensors/test_glacier.py b/providers/tests/amazon/aws/sensors/test_glacier.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_glacier.py rename to providers/tests/amazon/aws/sensors/test_glacier.py diff --git a/tests/providers/amazon/aws/sensors/test_glue.py b/providers/tests/amazon/aws/sensors/test_glue.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_glue.py rename to providers/tests/amazon/aws/sensors/test_glue.py diff --git a/tests/providers/amazon/aws/sensors/test_glue_catalog_partition.py b/providers/tests/amazon/aws/sensors/test_glue_catalog_partition.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_glue_catalog_partition.py rename to providers/tests/amazon/aws/sensors/test_glue_catalog_partition.py diff --git a/tests/providers/amazon/aws/sensors/test_glue_crawler.py b/providers/tests/amazon/aws/sensors/test_glue_crawler.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_glue_crawler.py rename to providers/tests/amazon/aws/sensors/test_glue_crawler.py diff --git a/tests/providers/amazon/aws/sensors/test_glue_data_quality.py b/providers/tests/amazon/aws/sensors/test_glue_data_quality.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_glue_data_quality.py rename to providers/tests/amazon/aws/sensors/test_glue_data_quality.py diff --git a/tests/providers/amazon/aws/sensors/test_kinesis_analytics.py b/providers/tests/amazon/aws/sensors/test_kinesis_analytics.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_kinesis_analytics.py rename to providers/tests/amazon/aws/sensors/test_kinesis_analytics.py diff --git a/tests/providers/amazon/aws/sensors/test_lambda_function.py b/providers/tests/amazon/aws/sensors/test_lambda_function.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_lambda_function.py rename to providers/tests/amazon/aws/sensors/test_lambda_function.py diff --git a/tests/providers/amazon/aws/sensors/test_opensearch_serverless.py b/providers/tests/amazon/aws/sensors/test_opensearch_serverless.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_opensearch_serverless.py rename to providers/tests/amazon/aws/sensors/test_opensearch_serverless.py diff --git a/tests/providers/amazon/aws/sensors/test_quicksight.py b/providers/tests/amazon/aws/sensors/test_quicksight.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_quicksight.py rename to providers/tests/amazon/aws/sensors/test_quicksight.py diff --git a/tests/providers/amazon/aws/sensors/test_rds.py b/providers/tests/amazon/aws/sensors/test_rds.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_rds.py rename to providers/tests/amazon/aws/sensors/test_rds.py diff --git a/tests/providers/amazon/aws/sensors/test_redshift_cluster.py b/providers/tests/amazon/aws/sensors/test_redshift_cluster.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_redshift_cluster.py rename to providers/tests/amazon/aws/sensors/test_redshift_cluster.py diff --git a/tests/providers/amazon/aws/sensors/test_s3.py b/providers/tests/amazon/aws/sensors/test_s3.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_s3.py rename to providers/tests/amazon/aws/sensors/test_s3.py diff --git a/tests/providers/amazon/aws/sensors/test_sagemaker_automl.py b/providers/tests/amazon/aws/sensors/test_sagemaker_automl.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sagemaker_automl.py rename to providers/tests/amazon/aws/sensors/test_sagemaker_automl.py diff --git a/tests/providers/amazon/aws/sensors/test_sagemaker_base.py b/providers/tests/amazon/aws/sensors/test_sagemaker_base.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sagemaker_base.py rename to providers/tests/amazon/aws/sensors/test_sagemaker_base.py diff --git a/tests/providers/amazon/aws/sensors/test_sagemaker_endpoint.py b/providers/tests/amazon/aws/sensors/test_sagemaker_endpoint.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sagemaker_endpoint.py rename to providers/tests/amazon/aws/sensors/test_sagemaker_endpoint.py diff --git a/tests/providers/amazon/aws/sensors/test_sagemaker_pipeline.py b/providers/tests/amazon/aws/sensors/test_sagemaker_pipeline.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sagemaker_pipeline.py rename to providers/tests/amazon/aws/sensors/test_sagemaker_pipeline.py diff --git a/tests/providers/amazon/aws/sensors/test_sagemaker_training.py b/providers/tests/amazon/aws/sensors/test_sagemaker_training.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sagemaker_training.py rename to providers/tests/amazon/aws/sensors/test_sagemaker_training.py diff --git a/tests/providers/amazon/aws/sensors/test_sagemaker_transform.py b/providers/tests/amazon/aws/sensors/test_sagemaker_transform.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sagemaker_transform.py rename to providers/tests/amazon/aws/sensors/test_sagemaker_transform.py diff --git a/tests/providers/amazon/aws/sensors/test_sagemaker_tuning.py b/providers/tests/amazon/aws/sensors/test_sagemaker_tuning.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sagemaker_tuning.py rename to providers/tests/amazon/aws/sensors/test_sagemaker_tuning.py diff --git a/tests/providers/amazon/aws/sensors/test_sqs.py b/providers/tests/amazon/aws/sensors/test_sqs.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_sqs.py rename to providers/tests/amazon/aws/sensors/test_sqs.py diff --git a/tests/providers/amazon/aws/sensors/test_step_function.py b/providers/tests/amazon/aws/sensors/test_step_function.py similarity index 100% rename from tests/providers/amazon/aws/sensors/test_step_function.py rename to providers/tests/amazon/aws/sensors/test_step_function.py diff --git a/tests/integration/providers/ydb/operators/__init__.py b/providers/tests/amazon/aws/system/__init__.py similarity index 100% rename from tests/integration/providers/ydb/operators/__init__.py rename to providers/tests/amazon/aws/system/__init__.py diff --git a/tests/providers/__init__.py b/providers/tests/amazon/aws/system/utils/__init__.py similarity index 100% rename from tests/providers/__init__.py rename to providers/tests/amazon/aws/system/utils/__init__.py diff --git a/tests/providers/amazon/aws/system/utils/test_helpers.py b/providers/tests/amazon/aws/system/utils/test_helpers.py similarity index 96% rename from tests/providers/amazon/aws/system/utils/test_helpers.py rename to providers/tests/amazon/aws/system/utils/test_helpers.py index f48de1788b74c..20324ebca945f 100644 --- a/tests/providers/amazon/aws/system/utils/test_helpers.py +++ b/providers/tests/amazon/aws/system/utils/test_helpers.py @@ -16,7 +16,7 @@ # under the License. """ This module contains the unit tests for the helper methods included in the Amazon System Tests found at -tests/system/providers/amazon/aws/utils/__init__.py +providers/tests/system/amazon/aws/utils/__init__.py """ from __future__ import annotations @@ -29,8 +29,8 @@ import pytest from moto import mock_aws -from tests.system.providers.amazon.aws import utils -from tests.system.providers.amazon.aws.utils import ( +from providers.tests.system.amazon.aws import utils +from providers.tests.system.amazon.aws.utils import ( DEFAULT_ENV_ID_LEN, DEFAULT_ENV_ID_PREFIX, ENV_ID_ENVIRON_KEY, diff --git a/tests/providers/airbyte/__init__.py b/providers/tests/amazon/aws/transfers/__init__.py similarity index 100% rename from tests/providers/airbyte/__init__.py rename to providers/tests/amazon/aws/transfers/__init__.py diff --git a/tests/providers/amazon/aws/transfers/test_azure_blob_to_s3.py b/providers/tests/amazon/aws/transfers/test_azure_blob_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_azure_blob_to_s3.py rename to providers/tests/amazon/aws/transfers/test_azure_blob_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_base.py b/providers/tests/amazon/aws/transfers/test_base.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_base.py rename to providers/tests/amazon/aws/transfers/test_base.py diff --git a/tests/providers/amazon/aws/transfers/test_dynamodb_to_s3.py b/providers/tests/amazon/aws/transfers/test_dynamodb_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_dynamodb_to_s3.py rename to providers/tests/amazon/aws/transfers/test_dynamodb_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_exasol_to_s3.py b/providers/tests/amazon/aws/transfers/test_exasol_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_exasol_to_s3.py rename to providers/tests/amazon/aws/transfers/test_exasol_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_ftp_to_s3.py b/providers/tests/amazon/aws/transfers/test_ftp_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_ftp_to_s3.py rename to providers/tests/amazon/aws/transfers/test_ftp_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py b/providers/tests/amazon/aws/transfers/test_gcs_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_gcs_to_s3.py rename to providers/tests/amazon/aws/transfers/test_gcs_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_glacier_to_gcs.py b/providers/tests/amazon/aws/transfers/test_glacier_to_gcs.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_glacier_to_gcs.py rename to providers/tests/amazon/aws/transfers/test_glacier_to_gcs.py diff --git a/tests/providers/amazon/aws/transfers/test_google_api_to_s3.py b/providers/tests/amazon/aws/transfers/test_google_api_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_google_api_to_s3.py rename to providers/tests/amazon/aws/transfers/test_google_api_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_hive_to_dynamodb.py b/providers/tests/amazon/aws/transfers/test_hive_to_dynamodb.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_hive_to_dynamodb.py rename to providers/tests/amazon/aws/transfers/test_hive_to_dynamodb.py diff --git a/tests/providers/amazon/aws/transfers/test_http_to_s3.py b/providers/tests/amazon/aws/transfers/test_http_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_http_to_s3.py rename to providers/tests/amazon/aws/transfers/test_http_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_imap_attachment_to_s3.py b/providers/tests/amazon/aws/transfers/test_imap_attachment_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_imap_attachment_to_s3.py rename to providers/tests/amazon/aws/transfers/test_imap_attachment_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_local_to_s3.py b/providers/tests/amazon/aws/transfers/test_local_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_local_to_s3.py rename to providers/tests/amazon/aws/transfers/test_local_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_mongo_to_s3.py b/providers/tests/amazon/aws/transfers/test_mongo_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_mongo_to_s3.py rename to providers/tests/amazon/aws/transfers/test_mongo_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_redshift_to_s3.py b/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py similarity index 99% rename from tests/providers/amazon/aws/transfers/test_redshift_to_s3.py rename to providers/tests/amazon/aws/transfers/test_redshift_to_s3.py index 2d28acd22e7e6..e27ff2d0aa56d 100644 --- a/tests/providers/amazon/aws/transfers/test_redshift_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py @@ -27,7 +27,8 @@ from airflow.models.connection import Connection from airflow.providers.amazon.aws.transfers.redshift_to_s3 import RedshiftToS3Operator from airflow.providers.amazon.aws.utils.redshift import build_credentials_block -from tests.test_utils.asserts import assert_equal_ignore_multiple_spaces + +from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces class TestRedshiftToS3Transfer: diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_dynamodb.py b/providers/tests/amazon/aws/transfers/test_s3_to_dynamodb.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_s3_to_dynamodb.py rename to providers/tests/amazon/aws/transfers/test_s3_to_dynamodb.py diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/providers/tests/amazon/aws/transfers/test_s3_to_ftp.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_s3_to_ftp.py rename to providers/tests/amazon/aws/transfers/test_s3_to_ftp.py diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py b/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py similarity index 99% rename from tests/providers/amazon/aws/transfers/test_s3_to_redshift.py rename to providers/tests/amazon/aws/transfers/test_s3_to_redshift.py index b80c5991626c0..6e300791e0935 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py +++ b/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py @@ -33,7 +33,8 @@ SchemaDatasetFacet, SchemaDatasetFacetFields, ) -from tests.test_utils.asserts import assert_equal_ignore_multiple_spaces + +from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces class TestS3ToRedshiftTransfer: diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_sftp.py b/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py similarity index 98% rename from tests/providers/amazon/aws/transfers/test_s3_to_sftp.py rename to providers/tests/amazon/aws/transfers/test_s3_to_sftp.py index 58f8a2d6f0f74..545398d9666f8 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_sftp.py +++ b/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py @@ -27,7 +27,8 @@ from airflow.providers.ssh.hooks.ssh import SSHHook from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_sql.py b/providers/tests/amazon/aws/transfers/test_s3_to_sql.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_s3_to_sql.py rename to providers/tests/amazon/aws/transfers/test_s3_to_sql.py diff --git a/tests/providers/amazon/aws/transfers/test_salesforce_to_s3.py b/providers/tests/amazon/aws/transfers/test_salesforce_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_salesforce_to_s3.py rename to providers/tests/amazon/aws/transfers/test_salesforce_to_s3.py diff --git a/tests/providers/amazon/aws/transfers/test_sftp_to_s3.py b/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py similarity index 98% rename from tests/providers/amazon/aws/transfers/test_sftp_to_s3.py rename to providers/tests/amazon/aws/transfers/test_sftp_to_s3.py index be438a85a4950..e4afe5c8c1efb 100644 --- a/tests/providers/amazon/aws/transfers/test_sftp_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py @@ -27,7 +27,8 @@ from airflow.providers.ssh.hooks.ssh import SSHHook from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/amazon/aws/transfers/test_sql_to_s3.py b/providers/tests/amazon/aws/transfers/test_sql_to_s3.py similarity index 100% rename from tests/providers/amazon/aws/transfers/test_sql_to_s3.py rename to providers/tests/amazon/aws/transfers/test_sql_to_s3.py diff --git a/tests/providers/airbyte/hooks/__init__.py b/providers/tests/amazon/aws/triggers/__init__.py similarity index 100% rename from tests/providers/airbyte/hooks/__init__.py rename to providers/tests/amazon/aws/triggers/__init__.py diff --git a/tests/providers/amazon/aws/triggers/test_athena.py b/providers/tests/amazon/aws/triggers/test_athena.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_athena.py rename to providers/tests/amazon/aws/triggers/test_athena.py diff --git a/tests/providers/amazon/aws/triggers/test_base.py b/providers/tests/amazon/aws/triggers/test_base.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_base.py rename to providers/tests/amazon/aws/triggers/test_base.py diff --git a/tests/providers/amazon/aws/triggers/test_batch.py b/providers/tests/amazon/aws/triggers/test_batch.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_batch.py rename to providers/tests/amazon/aws/triggers/test_batch.py diff --git a/tests/providers/amazon/aws/triggers/test_bedrock.py b/providers/tests/amazon/aws/triggers/test_bedrock.py similarity index 99% rename from tests/providers/amazon/aws/triggers/test_bedrock.py rename to providers/tests/amazon/aws/triggers/test_bedrock.py index 90112d8c1dc1d..a619d39b1307c 100644 --- a/tests/providers/amazon/aws/triggers/test_bedrock.py +++ b/providers/tests/amazon/aws/triggers/test_bedrock.py @@ -29,7 +29,8 @@ BedrockProvisionModelThroughputCompletedTrigger, ) from airflow.triggers.base import TriggerEvent -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type + +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type BASE_TRIGGER_CLASSPATH = "airflow.providers.amazon.aws.triggers.bedrock." diff --git a/tests/providers/amazon/aws/triggers/test_comprehend.py b/providers/tests/amazon/aws/triggers/test_comprehend.py similarity index 98% rename from tests/providers/amazon/aws/triggers/test_comprehend.py rename to providers/tests/amazon/aws/triggers/test_comprehend.py index f70d9fc0d696f..c7a53e2d70fce 100644 --- a/tests/providers/amazon/aws/triggers/test_comprehend.py +++ b/providers/tests/amazon/aws/triggers/test_comprehend.py @@ -27,7 +27,8 @@ ComprehendPiiEntitiesDetectionJobCompletedTrigger, ) from airflow.triggers.base import TriggerEvent -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type + +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type BASE_TRIGGER_CLASSPATH = "airflow.providers.amazon.aws.triggers.comprehend." diff --git a/tests/providers/amazon/aws/triggers/test_ec2.py b/providers/tests/amazon/aws/triggers/test_ec2.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_ec2.py rename to providers/tests/amazon/aws/triggers/test_ec2.py diff --git a/tests/providers/amazon/aws/triggers/test_ecs.py b/providers/tests/amazon/aws/triggers/test_ecs.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_ecs.py rename to providers/tests/amazon/aws/triggers/test_ecs.py diff --git a/tests/providers/amazon/aws/triggers/test_eks.py b/providers/tests/amazon/aws/triggers/test_eks.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_eks.py rename to providers/tests/amazon/aws/triggers/test_eks.py diff --git a/tests/providers/amazon/aws/triggers/test_emr.py b/providers/tests/amazon/aws/triggers/test_emr.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_emr.py rename to providers/tests/amazon/aws/triggers/test_emr.py diff --git a/tests/providers/amazon/aws/triggers/test_glue.py b/providers/tests/amazon/aws/triggers/test_glue.py similarity index 99% rename from tests/providers/amazon/aws/triggers/test_glue.py rename to providers/tests/amazon/aws/triggers/test_glue.py index 2ae3e57084ea1..e39e38d8b7609 100644 --- a/tests/providers/amazon/aws/triggers/test_glue.py +++ b/providers/tests/amazon/aws/triggers/test_glue.py @@ -32,7 +32,8 @@ GlueJobCompleteTrigger, ) from airflow.triggers.base import TriggerEvent -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type + +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type BASE_TRIGGER_CLASSPATH = "airflow.providers.amazon.aws.triggers.glue." diff --git a/tests/providers/amazon/aws/triggers/test_glue_crawler.py b/providers/tests/amazon/aws/triggers/test_glue_crawler.py similarity index 97% rename from tests/providers/amazon/aws/triggers/test_glue_crawler.py rename to providers/tests/amazon/aws/triggers/test_glue_crawler.py index 8975aa1aff505..fadc14fa0f271 100644 --- a/tests/providers/amazon/aws/triggers/test_glue_crawler.py +++ b/providers/tests/amazon/aws/triggers/test_glue_crawler.py @@ -24,7 +24,8 @@ from airflow.providers.amazon.aws.hooks.glue_crawler import GlueCrawlerHook from airflow.providers.amazon.aws.triggers.glue_crawler import GlueCrawlerCompleteTrigger from airflow.triggers.base import TriggerEvent -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type + +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type class TestGlueCrawlerCompleteTrigger: diff --git a/tests/providers/amazon/aws/triggers/test_glue_databrew.py b/providers/tests/amazon/aws/triggers/test_glue_databrew.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_glue_databrew.py rename to providers/tests/amazon/aws/triggers/test_glue_databrew.py diff --git a/tests/providers/amazon/aws/triggers/test_kinesis_analytics.py b/providers/tests/amazon/aws/triggers/test_kinesis_analytics.py similarity index 98% rename from tests/providers/amazon/aws/triggers/test_kinesis_analytics.py rename to providers/tests/amazon/aws/triggers/test_kinesis_analytics.py index 3692905f22999..6a0f321545060 100644 --- a/tests/providers/amazon/aws/triggers/test_kinesis_analytics.py +++ b/providers/tests/amazon/aws/triggers/test_kinesis_analytics.py @@ -26,7 +26,8 @@ KinesisAnalyticsV2ApplicationOperationCompleteTrigger, ) from airflow.triggers.base import TriggerEvent -from tests.providers.amazon.aws.utils.test_waiter import assert_expected_waiter_type + +from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type BASE_TRIGGER_CLASSPATH = "airflow.providers.amazon.aws.triggers.kinesis_analytics." diff --git a/tests/providers/amazon/aws/triggers/test_lambda_function.py b/providers/tests/amazon/aws/triggers/test_lambda_function.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_lambda_function.py rename to providers/tests/amazon/aws/triggers/test_lambda_function.py diff --git a/tests/providers/amazon/aws/triggers/test_neptune.py b/providers/tests/amazon/aws/triggers/test_neptune.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_neptune.py rename to providers/tests/amazon/aws/triggers/test_neptune.py diff --git a/tests/providers/amazon/aws/triggers/test_opensearch_serverless.py b/providers/tests/amazon/aws/triggers/test_opensearch_serverless.py similarity index 98% rename from tests/providers/amazon/aws/triggers/test_opensearch_serverless.py rename to providers/tests/amazon/aws/triggers/test_opensearch_serverless.py index c992d6a50da69..429b406072c58 100644 --- a/tests/providers/amazon/aws/triggers/test_opensearch_serverless.py +++ b/providers/tests/amazon/aws/triggers/test_opensearch_serverless.py @@ -27,7 +27,8 @@ ) from airflow.triggers.base import TriggerEvent from airflow.utils.helpers import prune_dict -from tests.providers.amazon.aws.triggers.test_base import TestAwsBaseWaiterTrigger + +from providers.tests.amazon.aws.triggers.test_base import TestAwsBaseWaiterTrigger BASE_TRIGGER_CLASSPATH = "airflow.providers.amazon.aws.triggers.opensearch_serverless." diff --git a/tests/providers/amazon/aws/triggers/test_rds.py b/providers/tests/amazon/aws/triggers/test_rds.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_rds.py rename to providers/tests/amazon/aws/triggers/test_rds.py diff --git a/tests/providers/amazon/aws/triggers/test_redshift_cluster.py b/providers/tests/amazon/aws/triggers/test_redshift_cluster.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_redshift_cluster.py rename to providers/tests/amazon/aws/triggers/test_redshift_cluster.py diff --git a/tests/providers/amazon/aws/triggers/test_redshift_data.py b/providers/tests/amazon/aws/triggers/test_redshift_data.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_redshift_data.py rename to providers/tests/amazon/aws/triggers/test_redshift_data.py diff --git a/tests/providers/amazon/aws/triggers/test_s3.py b/providers/tests/amazon/aws/triggers/test_s3.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_s3.py rename to providers/tests/amazon/aws/triggers/test_s3.py diff --git a/tests/providers/amazon/aws/triggers/test_sagemaker.py b/providers/tests/amazon/aws/triggers/test_sagemaker.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_sagemaker.py rename to providers/tests/amazon/aws/triggers/test_sagemaker.py diff --git a/tests/providers/amazon/aws/triggers/test_serialization.py b/providers/tests/amazon/aws/triggers/test_serialization.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_serialization.py rename to providers/tests/amazon/aws/triggers/test_serialization.py diff --git a/tests/providers/amazon/aws/triggers/test_sqs.py b/providers/tests/amazon/aws/triggers/test_sqs.py similarity index 100% rename from tests/providers/amazon/aws/triggers/test_sqs.py rename to providers/tests/amazon/aws/triggers/test_sqs.py diff --git a/tests/providers/airbyte/operators/__init__.py b/providers/tests/amazon/aws/utils/__init__.py similarity index 100% rename from tests/providers/airbyte/operators/__init__.py rename to providers/tests/amazon/aws/utils/__init__.py diff --git a/tests/providers/amazon/aws/utils/eks_test_constants.py b/providers/tests/amazon/aws/utils/eks_test_constants.py similarity index 100% rename from tests/providers/amazon/aws/utils/eks_test_constants.py rename to providers/tests/amazon/aws/utils/eks_test_constants.py diff --git a/tests/providers/amazon/aws/utils/eks_test_utils.py b/providers/tests/amazon/aws/utils/eks_test_utils.py similarity index 99% rename from tests/providers/amazon/aws/utils/eks_test_utils.py rename to providers/tests/amazon/aws/utils/eks_test_utils.py index fc75880f21f41..8bc9bd9e26227 100644 --- a/tests/providers/amazon/aws/utils/eks_test_utils.py +++ b/providers/tests/amazon/aws/utils/eks_test_utils.py @@ -21,7 +21,7 @@ from copy import deepcopy from typing import TYPE_CHECKING, Pattern, Type, Union -from tests.providers.amazon.aws.utils.eks_test_constants import ( +from providers.tests.amazon.aws.utils.eks_test_constants import ( STATUS, ClusterAttributes, ClusterInputs, diff --git a/tests/providers/amazon/aws/utils/test_connection_wrapper.py b/providers/tests/amazon/aws/utils/test_connection_wrapper.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_connection_wrapper.py rename to providers/tests/amazon/aws/utils/test_connection_wrapper.py diff --git a/tests/providers/amazon/aws/utils/test_eks_get_token.py b/providers/tests/amazon/aws/utils/test_eks_get_token.py similarity index 97% rename from tests/providers/amazon/aws/utils/test_eks_get_token.py rename to providers/tests/amazon/aws/utils/test_eks_get_token.py index 672ccfc9b3fd1..1d6300e45851e 100644 --- a/tests/providers/amazon/aws/utils/test_eks_get_token.py +++ b/providers/tests/amazon/aws/utils/test_eks_get_token.py @@ -63,13 +63,13 @@ class TestGetEksToken: ], ], ) - def test_run(self, mock_eks_hook, args, expected_aws_conn_id, expected_region_name, airflow_root_path): + def test_run(self, mock_eks_hook, args, expected_aws_conn_id, expected_region_name, providers_src_folder): ( mock_eks_hook.return_value.fetch_access_token_for_cluster.return_value ) = "k8s-aws-v1.aHR0cDovL2V4YW1wbGUuY29t" with mock.patch("sys.argv", args), contextlib.redirect_stdout(StringIO()) as temp_stdout: - os.chdir(airflow_root_path) + os.chdir(providers_src_folder) # We are not using run_module because of https://github.com/pytest-dev/pytest/issues/9007 runpy.run_path("airflow/providers/amazon/aws/utils/eks_get_token.py", run_name="__main__") output = temp_stdout.getvalue() diff --git a/tests/providers/amazon/aws/utils/test_emailer.py b/providers/tests/amazon/aws/utils/test_emailer.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_emailer.py rename to providers/tests/amazon/aws/utils/test_emailer.py diff --git a/tests/providers/amazon/aws/utils/test_identifiers.py b/providers/tests/amazon/aws/utils/test_identifiers.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_identifiers.py rename to providers/tests/amazon/aws/utils/test_identifiers.py diff --git a/tests/providers/amazon/aws/utils/test_mixins.py b/providers/tests/amazon/aws/utils/test_mixins.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_mixins.py rename to providers/tests/amazon/aws/utils/test_mixins.py diff --git a/tests/providers/amazon/aws/utils/test_openlineage.py b/providers/tests/amazon/aws/utils/test_openlineage.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_openlineage.py rename to providers/tests/amazon/aws/utils/test_openlineage.py diff --git a/tests/providers/amazon/aws/utils/test_redshift.py b/providers/tests/amazon/aws/utils/test_redshift.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_redshift.py rename to providers/tests/amazon/aws/utils/test_redshift.py diff --git a/tests/providers/amazon/aws/utils/test_sqs.py b/providers/tests/amazon/aws/utils/test_sqs.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_sqs.py rename to providers/tests/amazon/aws/utils/test_sqs.py diff --git a/tests/providers/amazon/aws/utils/test_suppress.py b/providers/tests/amazon/aws/utils/test_suppress.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_suppress.py rename to providers/tests/amazon/aws/utils/test_suppress.py diff --git a/tests/providers/amazon/aws/utils/test_tags.py b/providers/tests/amazon/aws/utils/test_tags.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_tags.py rename to providers/tests/amazon/aws/utils/test_tags.py diff --git a/tests/providers/amazon/aws/utils/test_task_log_fetcher.py b/providers/tests/amazon/aws/utils/test_task_log_fetcher.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_task_log_fetcher.py rename to providers/tests/amazon/aws/utils/test_task_log_fetcher.py diff --git a/tests/providers/amazon/aws/utils/test_template_fields.py b/providers/tests/amazon/aws/utils/test_template_fields.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_template_fields.py rename to providers/tests/amazon/aws/utils/test_template_fields.py diff --git a/tests/providers/amazon/aws/utils/test_utils.py b/providers/tests/amazon/aws/utils/test_utils.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_utils.py rename to providers/tests/amazon/aws/utils/test_utils.py diff --git a/tests/providers/amazon/aws/utils/test_waiter.py b/providers/tests/amazon/aws/utils/test_waiter.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_waiter.py rename to providers/tests/amazon/aws/utils/test_waiter.py diff --git a/tests/providers/amazon/aws/utils/test_waiter_with_logging.py b/providers/tests/amazon/aws/utils/test_waiter_with_logging.py similarity index 100% rename from tests/providers/amazon/aws/utils/test_waiter_with_logging.py rename to providers/tests/amazon/aws/utils/test_waiter_with_logging.py diff --git a/tests/providers/airbyte/sensors/__init__.py b/providers/tests/amazon/aws/waiters/__init__.py similarity index 100% rename from tests/providers/airbyte/sensors/__init__.py rename to providers/tests/amazon/aws/waiters/__init__.py diff --git a/tests/providers/amazon/aws/waiters/test.json b/providers/tests/amazon/aws/waiters/test.json similarity index 100% rename from tests/providers/amazon/aws/waiters/test.json rename to providers/tests/amazon/aws/waiters/test.json diff --git a/tests/providers/amazon/aws/waiters/test_batch.py b/providers/tests/amazon/aws/waiters/test_batch.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_batch.py rename to providers/tests/amazon/aws/waiters/test_batch.py diff --git a/tests/providers/amazon/aws/waiters/test_bedrock.py b/providers/tests/amazon/aws/waiters/test_bedrock.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_bedrock.py rename to providers/tests/amazon/aws/waiters/test_bedrock.py diff --git a/tests/providers/amazon/aws/waiters/test_bedrock_agent.py b/providers/tests/amazon/aws/waiters/test_bedrock_agent.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_bedrock_agent.py rename to providers/tests/amazon/aws/waiters/test_bedrock_agent.py diff --git a/tests/providers/amazon/aws/waiters/test_comprehend.py b/providers/tests/amazon/aws/waiters/test_comprehend.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_comprehend.py rename to providers/tests/amazon/aws/waiters/test_comprehend.py diff --git a/tests/providers/amazon/aws/waiters/test_custom_waiters.py b/providers/tests/amazon/aws/waiters/test_custom_waiters.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_custom_waiters.py rename to providers/tests/amazon/aws/waiters/test_custom_waiters.py diff --git a/tests/providers/amazon/aws/waiters/test_dynamo.py b/providers/tests/amazon/aws/waiters/test_dynamo.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_dynamo.py rename to providers/tests/amazon/aws/waiters/test_dynamo.py diff --git a/tests/providers/amazon/aws/waiters/test_ecs.py b/providers/tests/amazon/aws/waiters/test_ecs.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_ecs.py rename to providers/tests/amazon/aws/waiters/test_ecs.py diff --git a/tests/providers/amazon/aws/waiters/test_eks.py b/providers/tests/amazon/aws/waiters/test_eks.py similarity index 97% rename from tests/providers/amazon/aws/waiters/test_eks.py rename to providers/tests/amazon/aws/waiters/test_eks.py index 9013c8b7c6fca..6a3aa8cf0273a 100644 --- a/tests/providers/amazon/aws/waiters/test_eks.py +++ b/providers/tests/amazon/aws/waiters/test_eks.py @@ -22,7 +22,8 @@ from moto import mock_aws from airflow.providers.amazon.aws.hooks.eks import EksHook -from tests.providers.amazon.aws.waiters.test_custom_waiters import assert_all_match + +from providers.tests.amazon.aws.waiters.test_custom_waiters import assert_all_match class TestCustomEKSServiceWaiters: diff --git a/tests/providers/amazon/aws/waiters/test_emr.py b/providers/tests/amazon/aws/waiters/test_emr.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_emr.py rename to providers/tests/amazon/aws/waiters/test_emr.py diff --git a/tests/providers/amazon/aws/waiters/test_glue.py b/providers/tests/amazon/aws/waiters/test_glue.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_glue.py rename to providers/tests/amazon/aws/waiters/test_glue.py diff --git a/tests/providers/amazon/aws/waiters/test_glue_databrew.py b/providers/tests/amazon/aws/waiters/test_glue_databrew.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_glue_databrew.py rename to providers/tests/amazon/aws/waiters/test_glue_databrew.py diff --git a/tests/providers/amazon/aws/waiters/test_kinesis_analytics.py b/providers/tests/amazon/aws/waiters/test_kinesis_analytics.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_kinesis_analytics.py rename to providers/tests/amazon/aws/waiters/test_kinesis_analytics.py diff --git a/tests/providers/amazon/aws/waiters/test_neptune.py b/providers/tests/amazon/aws/waiters/test_neptune.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_neptune.py rename to providers/tests/amazon/aws/waiters/test_neptune.py diff --git a/tests/providers/amazon/aws/waiters/test_opensearch_serverless.py b/providers/tests/amazon/aws/waiters/test_opensearch_serverless.py similarity index 100% rename from tests/providers/amazon/aws/waiters/test_opensearch_serverless.py rename to providers/tests/amazon/aws/waiters/test_opensearch_serverless.py diff --git a/tests/providers/amazon/conftest.py b/providers/tests/amazon/conftest.py similarity index 100% rename from tests/providers/amazon/conftest.py rename to providers/tests/amazon/conftest.py diff --git a/tests/providers/airbyte/triggers/__init__.py b/providers/tests/apache/__init__.py similarity index 100% rename from tests/providers/airbyte/triggers/__init__.py rename to providers/tests/apache/__init__.py diff --git a/tests/providers/alibaba/__init__.py b/providers/tests/apache/beam/__init__.py similarity index 100% rename from tests/providers/alibaba/__init__.py rename to providers/tests/apache/beam/__init__.py diff --git a/tests/providers/alibaba/cloud/__init__.py b/providers/tests/apache/beam/hooks/__init__.py similarity index 100% rename from tests/providers/alibaba/cloud/__init__.py rename to providers/tests/apache/beam/hooks/__init__.py diff --git a/tests/providers/apache/beam/hooks/test_beam.py b/providers/tests/apache/beam/hooks/test_beam.py similarity index 100% rename from tests/providers/apache/beam/hooks/test_beam.py rename to providers/tests/apache/beam/hooks/test_beam.py diff --git a/tests/providers/alibaba/cloud/hooks/__init__.py b/providers/tests/apache/beam/operators/__init__.py similarity index 100% rename from tests/providers/alibaba/cloud/hooks/__init__.py rename to providers/tests/apache/beam/operators/__init__.py diff --git a/tests/providers/apache/beam/operators/test_beam.py b/providers/tests/apache/beam/operators/test_beam.py similarity index 100% rename from tests/providers/apache/beam/operators/test_beam.py rename to providers/tests/apache/beam/operators/test_beam.py diff --git a/tests/providers/alibaba/cloud/log/__init__.py b/providers/tests/apache/beam/triggers/__init__.py similarity index 100% rename from tests/providers/alibaba/cloud/log/__init__.py rename to providers/tests/apache/beam/triggers/__init__.py diff --git a/tests/providers/apache/beam/triggers/test_beam.py b/providers/tests/apache/beam/triggers/test_beam.py similarity index 100% rename from tests/providers/apache/beam/triggers/test_beam.py rename to providers/tests/apache/beam/triggers/test_beam.py diff --git a/tests/providers/alibaba/cloud/operators/__init__.py b/providers/tests/apache/cassandra/__init__.py similarity index 100% rename from tests/providers/alibaba/cloud/operators/__init__.py rename to providers/tests/apache/cassandra/__init__.py diff --git a/tests/providers/alibaba/cloud/sensors/__init__.py b/providers/tests/apache/cassandra/sensors/__init__.py similarity index 100% rename from tests/providers/alibaba/cloud/sensors/__init__.py rename to providers/tests/apache/cassandra/sensors/__init__.py diff --git a/tests/providers/apache/cassandra/sensors/test_record.py b/providers/tests/apache/cassandra/sensors/test_record.py similarity index 100% rename from tests/providers/apache/cassandra/sensors/test_record.py rename to providers/tests/apache/cassandra/sensors/test_record.py diff --git a/tests/providers/apache/cassandra/sensors/test_table.py b/providers/tests/apache/cassandra/sensors/test_table.py similarity index 100% rename from tests/providers/apache/cassandra/sensors/test_table.py rename to providers/tests/apache/cassandra/sensors/test_table.py diff --git a/airflow/providers/vertica/operators/__init__.py b/providers/tests/apache/drill/__init__.py similarity index 100% rename from airflow/providers/vertica/operators/__init__.py rename to providers/tests/apache/drill/__init__.py diff --git a/airflow/providers/zendesk/hooks/__init__.py b/providers/tests/apache/drill/hooks/__init__.py similarity index 100% rename from airflow/providers/zendesk/hooks/__init__.py rename to providers/tests/apache/drill/hooks/__init__.py diff --git a/tests/providers/apache/drill/hooks/test_drill.py b/providers/tests/apache/drill/hooks/test_drill.py similarity index 94% rename from tests/providers/apache/drill/hooks/test_drill.py rename to providers/tests/apache/drill/hooks/test_drill.py index bfedffd3d7983..a02b74545531a 100644 --- a/tests/providers/apache/drill/hooks/test_drill.py +++ b/providers/tests/apache/drill/hooks/test_drill.py @@ -26,9 +26,10 @@ @pytest.mark.parametrize("host, expect_error", [("host_with?", True), ("good_host", False)]) def test_get_host(host, expect_error): - with patch( - "airflow.providers.apache.drill.hooks.drill.DrillHook.get_connection" - ) as mock_get_connection, patch("sqlalchemy.engine.base.Engine.raw_connection") as raw_connection: + with ( + patch("airflow.providers.apache.drill.hooks.drill.DrillHook.get_connection") as mock_get_connection, + patch("sqlalchemy.engine.base.Engine.raw_connection") as raw_connection, + ): raw_connection.return_value = MagicMock() mock_get_connection.return_value = MagicMock( host=host, port=80, login="drill_user", password="secret" diff --git a/tests/providers/apache/drill/__init__.py b/providers/tests/apache/druid/__init__.py similarity index 100% rename from tests/providers/apache/drill/__init__.py rename to providers/tests/apache/druid/__init__.py diff --git a/tests/providers/apache/drill/hooks/__init__.py b/providers/tests/apache/druid/hooks/__init__.py similarity index 100% rename from tests/providers/apache/drill/hooks/__init__.py rename to providers/tests/apache/druid/hooks/__init__.py diff --git a/tests/providers/apache/druid/hooks/test_druid.py b/providers/tests/apache/druid/hooks/test_druid.py similarity index 100% rename from tests/providers/apache/druid/hooks/test_druid.py rename to providers/tests/apache/druid/hooks/test_druid.py diff --git a/tests/providers/apache/druid/__init__.py b/providers/tests/apache/druid/operators/__init__.py similarity index 100% rename from tests/providers/apache/druid/__init__.py rename to providers/tests/apache/druid/operators/__init__.py diff --git a/tests/providers/apache/druid/operators/test_druid.py b/providers/tests/apache/druid/operators/test_druid.py similarity index 100% rename from tests/providers/apache/druid/operators/test_druid.py rename to providers/tests/apache/druid/operators/test_druid.py diff --git a/tests/providers/alibaba/cloud/utils/__init__.py b/providers/tests/apache/druid/transfers/__init__.py similarity index 100% rename from tests/providers/alibaba/cloud/utils/__init__.py rename to providers/tests/apache/druid/transfers/__init__.py diff --git a/tests/providers/apache/druid/transfers/test_hive_to_druid.py b/providers/tests/apache/druid/transfers/test_hive_to_druid.py similarity index 100% rename from tests/providers/apache/druid/transfers/test_hive_to_druid.py rename to providers/tests/apache/druid/transfers/test_hive_to_druid.py diff --git a/tests/providers/apache/druid/hooks/__init__.py b/providers/tests/apache/flink/__init__.py similarity index 100% rename from tests/providers/apache/druid/hooks/__init__.py rename to providers/tests/apache/flink/__init__.py diff --git a/tests/providers/apache/druid/operators/__init__.py b/providers/tests/apache/flink/operators/__init__.py similarity index 100% rename from tests/providers/apache/druid/operators/__init__.py rename to providers/tests/apache/flink/operators/__init__.py diff --git a/tests/providers/apache/flink/operators/test_flink_kubernetes.py b/providers/tests/apache/flink/operators/test_flink_kubernetes.py similarity index 100% rename from tests/providers/apache/flink/operators/test_flink_kubernetes.py rename to providers/tests/apache/flink/operators/test_flink_kubernetes.py diff --git a/tests/providers/apache/flink/__init__.py b/providers/tests/apache/flink/sensors/__init__.py similarity index 100% rename from tests/providers/apache/flink/__init__.py rename to providers/tests/apache/flink/sensors/__init__.py diff --git a/tests/providers/apache/flink/sensors/test_flink_kubernetes.py b/providers/tests/apache/flink/sensors/test_flink_kubernetes.py similarity index 100% rename from tests/providers/apache/flink/sensors/test_flink_kubernetes.py rename to providers/tests/apache/flink/sensors/test_flink_kubernetes.py diff --git a/tests/providers/apache/flink/operators/__init__.py b/providers/tests/apache/hdfs/__init__.py similarity index 100% rename from tests/providers/apache/flink/operators/__init__.py rename to providers/tests/apache/hdfs/__init__.py diff --git a/tests/providers/apache/flink/sensors/__init__.py b/providers/tests/apache/hdfs/hooks/__init__.py similarity index 100% rename from tests/providers/apache/flink/sensors/__init__.py rename to providers/tests/apache/hdfs/hooks/__init__.py diff --git a/tests/providers/apache/hdfs/hooks/test_webhdfs.py b/providers/tests/apache/hdfs/hooks/test_webhdfs.py similarity index 100% rename from tests/providers/apache/hdfs/hooks/test_webhdfs.py rename to providers/tests/apache/hdfs/hooks/test_webhdfs.py diff --git a/tests/providers/apache/hdfs/__init__.py b/providers/tests/apache/hdfs/sensors/__init__.py similarity index 100% rename from tests/providers/apache/hdfs/__init__.py rename to providers/tests/apache/hdfs/sensors/__init__.py diff --git a/tests/providers/apache/hdfs/sensors/test_web_hdfs.py b/providers/tests/apache/hdfs/sensors/test_web_hdfs.py similarity index 100% rename from tests/providers/apache/hdfs/sensors/test_web_hdfs.py rename to providers/tests/apache/hdfs/sensors/test_web_hdfs.py diff --git a/tests/providers/apache/hive/__init__.py b/providers/tests/apache/hive/__init__.py similarity index 100% rename from tests/providers/apache/hive/__init__.py rename to providers/tests/apache/hive/__init__.py diff --git a/tests/providers/apache/hdfs/hooks/__init__.py b/providers/tests/apache/hive/hooks/__init__.py similarity index 100% rename from tests/providers/apache/hdfs/hooks/__init__.py rename to providers/tests/apache/hive/hooks/__init__.py diff --git a/tests/providers/apache/hive/hooks/query_results.csv b/providers/tests/apache/hive/hooks/query_results.csv similarity index 100% rename from tests/providers/apache/hive/hooks/query_results.csv rename to providers/tests/apache/hive/hooks/query_results.csv diff --git a/tests/providers/apache/hive/hooks/test_hive.py b/providers/tests/apache/hive/hooks/test_hive.py similarity index 98% rename from tests/providers/apache/hive/hooks/test_hive.py rename to providers/tests/apache/hive/hooks/test_hive.py index 93494876175ad..aee09db28088e 100644 --- a/tests/providers/apache/hive/hooks/test_hive.py +++ b/providers/tests/apache/hive/hooks/test_hive.py @@ -33,14 +33,15 @@ from airflow.secrets.environment_variables import CONN_ENV_PREFIX from airflow.utils import timezone from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING -from tests.providers.apache.hive import ( + +from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces +from providers.tests.apache.hive import ( BaseMockConnectionCursor, InvalidHiveCliHook, MockHiveCliHook, MockHiveServer2Hook, MockSubProcess, ) -from tests.test_utils.asserts import assert_equal_ignore_multiple_spaces DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() @@ -333,10 +334,11 @@ def setup_method(self): self.database = "airflow" self.partition_by = "ds" self.table = "static_babynames_partitioned" - with mock.patch( - "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_metastore_client" - ) as get_metastore_mock, mock.patch( - "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection" + with ( + mock.patch( + "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_metastore_client" + ) as get_metastore_mock, + mock.patch("airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection"), ): get_metastore_mock.return_value = mock.MagicMock() @@ -419,11 +421,14 @@ def test_ha_hosts(self, socket_mock): assert socket_mock.socket.call_count == 2 def test_get_conn(self): - with mock.patch( - "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook._find_valid_host" - ) as find_valid_host, mock.patch( - "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection" - ) as get_connection: + with ( + mock.patch( + "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook._find_valid_host" + ) as find_valid_host, + mock.patch( + "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection" + ) as get_connection, + ): find_valid_host.return_value = mock.MagicMock(return_value="") get_connection.return_value = mock.MagicMock(return_value="") metastore_hook = HiveMetastoreHook() diff --git a/tests/providers/apache/hdfs/sensors/__init__.py b/providers/tests/apache/hive/macros/__init__.py similarity index 100% rename from tests/providers/apache/hdfs/sensors/__init__.py rename to providers/tests/apache/hive/macros/__init__.py diff --git a/tests/providers/apache/hive/macros/test_hive.py b/providers/tests/apache/hive/macros/test_hive.py similarity index 100% rename from tests/providers/apache/hive/macros/test_hive.py rename to providers/tests/apache/hive/macros/test_hive.py diff --git a/tests/providers/apache/hive/hooks/__init__.py b/providers/tests/apache/hive/operators/__init__.py similarity index 100% rename from tests/providers/apache/hive/hooks/__init__.py rename to providers/tests/apache/hive/operators/__init__.py diff --git a/tests/providers/apache/hive/operators/test_hive.py b/providers/tests/apache/hive/operators/test_hive.py similarity index 99% rename from tests/providers/apache/hive/operators/test_hive.py rename to providers/tests/apache/hive/operators/test_hive.py index f02f69c2a482d..cda19f9c2a05c 100644 --- a/tests/providers/apache/hive/operators/test_hive.py +++ b/providers/tests/apache/hive/operators/test_hive.py @@ -26,7 +26,8 @@ from airflow.models import DagRun, TaskInstance from airflow.providers.apache.hive.operators.hive import HiveOperator from airflow.utils import timezone -from tests.providers.apache.hive import DEFAULT_DATE, MockSubProcess, TestHiveEnvironment + +from providers.tests.apache.hive import DEFAULT_DATE, MockSubProcess, TestHiveEnvironment class HiveOperatorConfigTest(TestHiveEnvironment): diff --git a/tests/providers/apache/hive/operators/test_hive_stats.py b/providers/tests/apache/hive/operators/test_hive_stats.py similarity index 99% rename from tests/providers/apache/hive/operators/test_hive_stats.py rename to providers/tests/apache/hive/operators/test_hive_stats.py index e419d2da00b92..cce576515cb68 100644 --- a/tests/providers/apache/hive/operators/test_hive_stats.py +++ b/providers/tests/apache/hive/operators/test_hive_stats.py @@ -26,7 +26,8 @@ from airflow.exceptions import AirflowException from airflow.providers.apache.hive.operators.hive_stats import HiveStatsCollectionOperator from airflow.providers.presto.hooks.presto import PrestoHook -from tests.providers.apache.hive import ( + +from providers.tests.apache.hive import ( DEFAULT_DATE, DEFAULT_DATE_DS, MockConnectionCursor, diff --git a/tests/providers/apache/hive/macros/__init__.py b/providers/tests/apache/hive/sensors/__init__.py similarity index 100% rename from tests/providers/apache/hive/macros/__init__.py rename to providers/tests/apache/hive/sensors/__init__.py diff --git a/tests/providers/apache/hive/sensors/test_hive_partition.py b/providers/tests/apache/hive/sensors/test_hive_partition.py similarity index 96% rename from tests/providers/apache/hive/sensors/test_hive_partition.py rename to providers/tests/apache/hive/sensors/test_hive_partition.py index 45e10783d81ea..4df701bfe697b 100644 --- a/tests/providers/apache/hive/sensors/test_hive_partition.py +++ b/providers/tests/apache/hive/sensors/test_hive_partition.py @@ -23,7 +23,8 @@ import pytest from airflow.providers.apache.hive.sensors.hive_partition import HivePartitionSensor -from tests.providers.apache.hive import DEFAULT_DATE, MockHiveMetastoreHook, TestHiveEnvironment + +from providers.tests.apache.hive import DEFAULT_DATE, MockHiveMetastoreHook, TestHiveEnvironment @pytest.mark.skipif( diff --git a/tests/providers/apache/hive/sensors/test_metastore_partition.py b/providers/tests/apache/hive/sensors/test_metastore_partition.py similarity index 96% rename from tests/providers/apache/hive/sensors/test_metastore_partition.py rename to providers/tests/apache/hive/sensors/test_metastore_partition.py index 04a251339dc30..3acb19678a5d8 100644 --- a/tests/providers/apache/hive/sensors/test_metastore_partition.py +++ b/providers/tests/apache/hive/sensors/test_metastore_partition.py @@ -23,7 +23,8 @@ import pytest from airflow.providers.apache.hive.sensors.metastore_partition import MetastorePartitionSensor -from tests.providers.apache.hive import DEFAULT_DATE, DEFAULT_DATE_DS, MockDBConnection, TestHiveEnvironment + +from providers.tests.apache.hive import DEFAULT_DATE, DEFAULT_DATE_DS, MockDBConnection, TestHiveEnvironment @pytest.mark.skipif( diff --git a/tests/providers/apache/hive/sensors/test_named_hive_partition.py b/providers/tests/apache/hive/sensors/test_named_hive_partition.py similarity index 99% rename from tests/providers/apache/hive/sensors/test_named_hive_partition.py rename to providers/tests/apache/hive/sensors/test_named_hive_partition.py index 01827692273a6..ff565c733d8bf 100644 --- a/tests/providers/apache/hive/sensors/test_named_hive_partition.py +++ b/providers/tests/apache/hive/sensors/test_named_hive_partition.py @@ -27,7 +27,8 @@ from airflow.models.dag import DAG from airflow.providers.apache.hive.sensors.named_hive_partition import NamedHivePartitionSensor from airflow.utils.timezone import datetime -from tests.providers.apache.hive import MockHiveMetastoreHook, TestHiveEnvironment + +from providers.tests.apache.hive import MockHiveMetastoreHook, TestHiveEnvironment DEFAULT_DATE = datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() diff --git a/tests/providers/amazon/__init__.py b/providers/tests/apache/hive/transfers/__init__.py similarity index 100% rename from tests/providers/amazon/__init__.py rename to providers/tests/apache/hive/transfers/__init__.py diff --git a/tests/providers/apache/hive/transfers/test_hive_to_mysql.py b/providers/tests/apache/hive/transfers/test_hive_to_mysql.py similarity index 99% rename from tests/providers/apache/hive/transfers/test_hive_to_mysql.py rename to providers/tests/apache/hive/transfers/test_hive_to_mysql.py index a7e2cbe0e0d6d..f7a4e9c013013 100644 --- a/tests/providers/apache/hive/transfers/test_hive_to_mysql.py +++ b/providers/tests/apache/hive/transfers/test_hive_to_mysql.py @@ -26,7 +26,8 @@ from airflow.providers.apache.hive.transfers.hive_to_mysql import HiveToMySqlOperator from airflow.utils import timezone from airflow.utils.operator_helpers import context_to_airflow_vars -from tests.providers.apache.hive import MockHiveServer2Hook, MockMySqlHook, TestHiveEnvironment + +from providers.tests.apache.hive import MockHiveServer2Hook, MockMySqlHook, TestHiveEnvironment DEFAULT_DATE = timezone.datetime(2015, 1, 1) diff --git a/tests/providers/apache/hive/transfers/test_hive_to_samba.py b/providers/tests/apache/hive/transfers/test_hive_to_samba.py similarity index 99% rename from tests/providers/apache/hive/transfers/test_hive_to_samba.py rename to providers/tests/apache/hive/transfers/test_hive_to_samba.py index 7eb50e44b337e..7225e732db812 100644 --- a/tests/providers/apache/hive/transfers/test_hive_to_samba.py +++ b/providers/tests/apache/hive/transfers/test_hive_to_samba.py @@ -25,7 +25,8 @@ from airflow.providers.apache.hive.transfers.hive_to_samba import HiveToSambaOperator from airflow.providers.samba.hooks.samba import SambaHook from airflow.utils.operator_helpers import context_to_airflow_vars -from tests.providers.apache.hive import ( + +from providers.tests.apache.hive import ( DEFAULT_DATE, MockConnectionCursor, MockHiveServer2Hook, diff --git a/tests/providers/apache/hive/transfers/test_mssql_to_hive.py b/providers/tests/apache/hive/transfers/test_mssql_to_hive.py similarity index 100% rename from tests/providers/apache/hive/transfers/test_mssql_to_hive.py rename to providers/tests/apache/hive/transfers/test_mssql_to_hive.py diff --git a/tests/providers/apache/hive/transfers/test_mysql_to_hive.py b/providers/tests/apache/hive/transfers/test_mysql_to_hive.py similarity index 100% rename from tests/providers/apache/hive/transfers/test_mysql_to_hive.py rename to providers/tests/apache/hive/transfers/test_mysql_to_hive.py diff --git a/tests/providers/apache/hive/transfers/test_s3_to_hive.py b/providers/tests/apache/hive/transfers/test_s3_to_hive.py similarity index 100% rename from tests/providers/apache/hive/transfers/test_s3_to_hive.py rename to providers/tests/apache/hive/transfers/test_s3_to_hive.py diff --git a/tests/providers/apache/hive/transfers/test_vertica_to_hive.py b/providers/tests/apache/hive/transfers/test_vertica_to_hive.py similarity index 100% rename from tests/providers/apache/hive/transfers/test_vertica_to_hive.py rename to providers/tests/apache/hive/transfers/test_vertica_to_hive.py diff --git a/tests/providers/amazon/aws/__init__.py b/providers/tests/apache/iceberg/__init__.py similarity index 100% rename from tests/providers/amazon/aws/__init__.py rename to providers/tests/apache/iceberg/__init__.py diff --git a/tests/providers/amazon/aws/assets/__init__.py b/providers/tests/apache/iceberg/hooks/__init__.py similarity index 100% rename from tests/providers/amazon/aws/assets/__init__.py rename to providers/tests/apache/iceberg/hooks/__init__.py diff --git a/tests/providers/apache/iceberg/hooks/test_iceberg.py b/providers/tests/apache/iceberg/hooks/test_iceberg.py similarity index 100% rename from tests/providers/apache/iceberg/hooks/test_iceberg.py rename to providers/tests/apache/iceberg/hooks/test_iceberg.py diff --git a/tests/providers/amazon/aws/auth_manager/__init__.py b/providers/tests/apache/impala/__init__.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/__init__.py rename to providers/tests/apache/impala/__init__.py diff --git a/tests/providers/amazon/aws/auth_manager/avp/__init__.py b/providers/tests/apache/impala/hooks/__init__.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/avp/__init__.py rename to providers/tests/apache/impala/hooks/__init__.py diff --git a/tests/providers/apache/impala/hooks/test_impala.py b/providers/tests/apache/impala/hooks/test_impala.py similarity index 100% rename from tests/providers/apache/impala/hooks/test_impala.py rename to providers/tests/apache/impala/hooks/test_impala.py diff --git a/tests/providers/amazon/aws/auth_manager/cli/__init__.py b/providers/tests/apache/kafka/__init__.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/cli/__init__.py rename to providers/tests/apache/kafka/__init__.py diff --git a/tests/providers/amazon/aws/auth_manager/security_manager/__init__.py b/providers/tests/apache/kafka/hooks/__init__.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/security_manager/__init__.py rename to providers/tests/apache/kafka/hooks/__init__.py diff --git a/tests/providers/apache/kafka/hooks/test_base.py b/providers/tests/apache/kafka/hooks/test_base.py similarity index 100% rename from tests/providers/apache/kafka/hooks/test_base.py rename to providers/tests/apache/kafka/hooks/test_base.py diff --git a/tests/providers/apache/kafka/hooks/test_client.py b/providers/tests/apache/kafka/hooks/test_client.py similarity index 100% rename from tests/providers/apache/kafka/hooks/test_client.py rename to providers/tests/apache/kafka/hooks/test_client.py diff --git a/tests/providers/apache/kafka/hooks/test_consume.py b/providers/tests/apache/kafka/hooks/test_consume.py similarity index 100% rename from tests/providers/apache/kafka/hooks/test_consume.py rename to providers/tests/apache/kafka/hooks/test_consume.py diff --git a/tests/providers/apache/kafka/hooks/test_produce.py b/providers/tests/apache/kafka/hooks/test_produce.py similarity index 100% rename from tests/providers/apache/kafka/hooks/test_produce.py rename to providers/tests/apache/kafka/hooks/test_produce.py diff --git a/tests/providers/amazon/aws/auth_manager/views/__init__.py b/providers/tests/apache/kafka/operators/__init__.py similarity index 100% rename from tests/providers/amazon/aws/auth_manager/views/__init__.py rename to providers/tests/apache/kafka/operators/__init__.py diff --git a/tests/providers/apache/kafka/operators/test_consume.py b/providers/tests/apache/kafka/operators/test_consume.py similarity index 98% rename from tests/providers/apache/kafka/operators/test_consume.py rename to providers/tests/apache/kafka/operators/test_consume.py index 699e0abe0643c..7e02a273894b3 100644 --- a/tests/providers/apache/kafka/operators/test_consume.py +++ b/providers/tests/apache/kafka/operators/test_consume.py @@ -64,7 +64,7 @@ def test_operator(self): operator = ConsumeFromTopicOperator( kafka_config_id="kafka_d", topics=["test"], - apply_function="tests.providers.apache.kafka.operators.test_consume._no_op", + apply_function="providers.tests.apache.kafka.operators.test_consume._no_op", task_id="test", poll_timeout=0.0001, ) diff --git a/tests/providers/apache/kafka/operators/test_produce.py b/providers/tests/apache/kafka/operators/test_produce.py similarity index 97% rename from tests/providers/apache/kafka/operators/test_produce.py rename to providers/tests/apache/kafka/operators/test_produce.py index 49559984f1d0a..6c1b853ecef90 100644 --- a/tests/providers/apache/kafka/operators/test_produce.py +++ b/providers/tests/apache/kafka/operators/test_produce.py @@ -69,7 +69,7 @@ def test_operator_string(self): operator = ProduceToTopicOperator( kafka_config_id="kafka_d", topic="test_1", - producer_function="tests.providers.apache.kafka.operators.test_produce._simple_producer", + producer_function="providers.tests.apache.kafka.operators.test_produce._simple_producer", producer_function_args=(b"test", b"test"), task_id="test", synchronous=False, diff --git a/tests/providers/amazon/aws/config_templates/__init__.py b/providers/tests/apache/kafka/sensors/__init__.py similarity index 100% rename from tests/providers/amazon/aws/config_templates/__init__.py rename to providers/tests/apache/kafka/sensors/__init__.py diff --git a/tests/providers/apache/kafka/sensors/test_kafka.py b/providers/tests/apache/kafka/sensors/test_kafka.py similarity index 100% rename from tests/providers/apache/kafka/sensors/test_kafka.py rename to providers/tests/apache/kafka/sensors/test_kafka.py diff --git a/tests/providers/amazon/aws/executors/__init__.py b/providers/tests/apache/kafka/triggers/__init__.py similarity index 100% rename from tests/providers/amazon/aws/executors/__init__.py rename to providers/tests/apache/kafka/triggers/__init__.py diff --git a/tests/providers/apache/kafka/triggers/test_await_message.py b/providers/tests/apache/kafka/triggers/test_await_message.py similarity index 96% rename from tests/providers/apache/kafka/triggers/test_await_message.py rename to providers/tests/apache/kafka/triggers/test_await_message.py index db17d5e4e758c..0f2b3f6f6b074 100644 --- a/tests/providers/apache/kafka/triggers/test_await_message.py +++ b/providers/tests/apache/kafka/triggers/test_await_message.py @@ -100,7 +100,7 @@ async def test_trigger_run_good(self, mocker): trigger = AwaitMessageTrigger( kafka_config_id="kafka_d", - apply_function="tests.providers.apache.kafka.triggers.test_await_message.apply_function_true", + apply_function="providers.tests.apache.kafka.triggers.test_await_message.apply_function_true", topics=["noop"], poll_timeout=0.0001, poll_interval=5, @@ -117,7 +117,7 @@ async def test_trigger_run_bad(self, mocker): trigger = AwaitMessageTrigger( kafka_config_id="kafka_d", - apply_function="tests.providers.apache.kafka.triggers.test_await_message.apply_function_false", + apply_function="providers.tests.apache.kafka.triggers.test_await_message.apply_function_false", topics=["noop"], poll_timeout=0.0001, poll_interval=5, diff --git a/tests/providers/apache/hive/operators/__init__.py b/providers/tests/apache/kylin/__init__.py similarity index 100% rename from tests/providers/apache/hive/operators/__init__.py rename to providers/tests/apache/kylin/__init__.py diff --git a/tests/providers/apache/hive/sensors/__init__.py b/providers/tests/apache/kylin/hooks/__init__.py similarity index 100% rename from tests/providers/apache/hive/sensors/__init__.py rename to providers/tests/apache/kylin/hooks/__init__.py diff --git a/tests/providers/apache/kylin/hooks/test_kylin.py b/providers/tests/apache/kylin/hooks/test_kylin.py similarity index 100% rename from tests/providers/apache/kylin/hooks/test_kylin.py rename to providers/tests/apache/kylin/hooks/test_kylin.py diff --git a/tests/providers/apache/kylin/__init__.py b/providers/tests/apache/kylin/operators/__init__.py similarity index 100% rename from tests/providers/apache/kylin/__init__.py rename to providers/tests/apache/kylin/operators/__init__.py diff --git a/tests/providers/apache/kylin/operators/test_kylin_cube.py b/providers/tests/apache/kylin/operators/test_kylin_cube.py similarity index 100% rename from tests/providers/apache/kylin/operators/test_kylin_cube.py rename to providers/tests/apache/kylin/operators/test_kylin_cube.py diff --git a/tests/providers/amazon/aws/executors/batch/__init__.py b/providers/tests/apache/livy/__init__.py similarity index 100% rename from tests/providers/amazon/aws/executors/batch/__init__.py rename to providers/tests/apache/livy/__init__.py diff --git a/tests/providers/amazon/aws/executors/ecs/__init__.py b/providers/tests/apache/livy/hooks/__init__.py similarity index 100% rename from tests/providers/amazon/aws/executors/ecs/__init__.py rename to providers/tests/apache/livy/hooks/__init__.py diff --git a/tests/providers/apache/livy/hooks/test_livy.py b/providers/tests/apache/livy/hooks/test_livy.py similarity index 99% rename from tests/providers/apache/livy/hooks/test_livy.py rename to providers/tests/apache/livy/hooks/test_livy.py index d3c110e68b4b0..0fca347b06a99 100644 --- a/tests/providers/apache/livy/hooks/test_livy.py +++ b/providers/tests/apache/livy/hooks/test_livy.py @@ -30,7 +30,8 @@ from airflow.models import Connection from airflow.providers.apache.livy.hooks.livy import BatchState, LivyAsyncHook, LivyHook from airflow.utils import db -from tests.test_utils.db import clear_db_connections + +from dev.tests_common.test_utils.db import clear_db_connections pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/tests/providers/amazon/aws/executors/utils/__init__.py b/providers/tests/apache/livy/operators/__init__.py similarity index 100% rename from tests/providers/amazon/aws/executors/utils/__init__.py rename to providers/tests/apache/livy/operators/__init__.py diff --git a/tests/providers/apache/livy/operators/test_livy.py b/providers/tests/apache/livy/operators/test_livy.py similarity index 100% rename from tests/providers/apache/livy/operators/test_livy.py rename to providers/tests/apache/livy/operators/test_livy.py diff --git a/tests/providers/amazon/aws/fs/__init__.py b/providers/tests/apache/livy/sensors/__init__.py similarity index 100% rename from tests/providers/amazon/aws/fs/__init__.py rename to providers/tests/apache/livy/sensors/__init__.py diff --git a/tests/providers/apache/livy/sensors/test_livy.py b/providers/tests/apache/livy/sensors/test_livy.py similarity index 100% rename from tests/providers/apache/livy/sensors/test_livy.py rename to providers/tests/apache/livy/sensors/test_livy.py diff --git a/tests/providers/amazon/aws/hooks/__init__.py b/providers/tests/apache/livy/triggers/__init__.py similarity index 100% rename from tests/providers/amazon/aws/hooks/__init__.py rename to providers/tests/apache/livy/triggers/__init__.py diff --git a/tests/providers/apache/livy/triggers/test_livy.py b/providers/tests/apache/livy/triggers/test_livy.py similarity index 100% rename from tests/providers/apache/livy/triggers/test_livy.py rename to providers/tests/apache/livy/triggers/test_livy.py diff --git a/tests/providers/apache/kylin/hooks/__init__.py b/providers/tests/apache/pig/__init__.py similarity index 100% rename from tests/providers/apache/kylin/hooks/__init__.py rename to providers/tests/apache/pig/__init__.py diff --git a/tests/providers/apache/kylin/operators/__init__.py b/providers/tests/apache/pig/hooks/__init__.py similarity index 100% rename from tests/providers/apache/kylin/operators/__init__.py rename to providers/tests/apache/pig/hooks/__init__.py diff --git a/tests/providers/apache/pig/hooks/test_pig.py b/providers/tests/apache/pig/hooks/test_pig.py similarity index 100% rename from tests/providers/apache/pig/hooks/test_pig.py rename to providers/tests/apache/pig/hooks/test_pig.py diff --git a/tests/providers/amazon/aws/infrastructure/__init__.py b/providers/tests/apache/pig/operators/__init__.py similarity index 100% rename from tests/providers/amazon/aws/infrastructure/__init__.py rename to providers/tests/apache/pig/operators/__init__.py diff --git a/tests/providers/apache/pig/operators/test_pig.py b/providers/tests/apache/pig/operators/test_pig.py similarity index 100% rename from tests/providers/apache/pig/operators/test_pig.py rename to providers/tests/apache/pig/operators/test_pig.py diff --git a/tests/providers/apache/pig/__init__.py b/providers/tests/apache/pinot/__init__.py similarity index 100% rename from tests/providers/apache/pig/__init__.py rename to providers/tests/apache/pinot/__init__.py diff --git a/tests/providers/apache/pig/hooks/__init__.py b/providers/tests/apache/pinot/hooks/__init__.py similarity index 100% rename from tests/providers/apache/pig/hooks/__init__.py rename to providers/tests/apache/pinot/hooks/__init__.py diff --git a/tests/providers/apache/pinot/hooks/test_pinot.py b/providers/tests/apache/pinot/hooks/test_pinot.py similarity index 100% rename from tests/providers/apache/pinot/hooks/test_pinot.py rename to providers/tests/apache/pinot/hooks/test_pinot.py diff --git a/tests/providers/apache/pinot/__init__.py b/providers/tests/apache/spark/__init__.py similarity index 100% rename from tests/providers/apache/pinot/__init__.py rename to providers/tests/apache/spark/__init__.py diff --git a/tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py b/providers/tests/apache/spark/decorators/__init__.py similarity index 100% rename from tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py rename to providers/tests/apache/spark/decorators/__init__.py diff --git a/tests/providers/apache/spark/decorators/test_pyspark.py b/providers/tests/apache/spark/decorators/test_pyspark.py similarity index 100% rename from tests/providers/apache/spark/decorators/test_pyspark.py rename to providers/tests/apache/spark/decorators/test_pyspark.py diff --git a/tests/providers/apache/pinot/hooks/__init__.py b/providers/tests/apache/spark/hooks/__init__.py similarity index 100% rename from tests/providers/apache/pinot/hooks/__init__.py rename to providers/tests/apache/spark/hooks/__init__.py diff --git a/tests/providers/apache/spark/hooks/test_spark_connect.py b/providers/tests/apache/spark/hooks/test_spark_connect.py similarity index 100% rename from tests/providers/apache/spark/hooks/test_spark_connect.py rename to providers/tests/apache/spark/hooks/test_spark_connect.py diff --git a/tests/providers/apache/spark/hooks/test_spark_jdbc.py b/providers/tests/apache/spark/hooks/test_spark_jdbc.py similarity index 100% rename from tests/providers/apache/spark/hooks/test_spark_jdbc.py rename to providers/tests/apache/spark/hooks/test_spark_jdbc.py diff --git a/tests/providers/apache/spark/hooks/test_spark_jdbc_script.py b/providers/tests/apache/spark/hooks/test_spark_jdbc_script.py similarity index 100% rename from tests/providers/apache/spark/hooks/test_spark_jdbc_script.py rename to providers/tests/apache/spark/hooks/test_spark_jdbc_script.py diff --git a/tests/providers/apache/spark/hooks/test_spark_sql.py b/providers/tests/apache/spark/hooks/test_spark_sql.py similarity index 99% rename from tests/providers/apache/spark/hooks/test_spark_sql.py rename to providers/tests/apache/spark/hooks/test_spark_sql.py index 9a33bfc2923d9..10f63cce99326 100644 --- a/tests/providers/apache/spark/hooks/test_spark_sql.py +++ b/providers/tests/apache/spark/hooks/test_spark_sql.py @@ -27,7 +27,8 @@ from airflow.models import Connection from airflow.providers.apache.spark.hooks.spark_sql import SparkSqlHook from airflow.utils import db -from tests.test_utils.db import clear_db_connections + +from dev.tests_common.test_utils.db import clear_db_connections pytestmark = pytest.mark.db_test diff --git a/tests/providers/apache/spark/hooks/test_spark_submit.py b/providers/tests/apache/spark/hooks/test_spark_submit.py similarity index 100% rename from tests/providers/apache/spark/hooks/test_spark_submit.py rename to providers/tests/apache/spark/hooks/test_spark_submit.py diff --git a/tests/providers/apache/spark/__init__.py b/providers/tests/apache/spark/operators/__init__.py similarity index 100% rename from tests/providers/apache/spark/__init__.py rename to providers/tests/apache/spark/operators/__init__.py diff --git a/tests/providers/apache/spark/operators/test_spark_jdbc.py b/providers/tests/apache/spark/operators/test_spark_jdbc.py similarity index 100% rename from tests/providers/apache/spark/operators/test_spark_jdbc.py rename to providers/tests/apache/spark/operators/test_spark_jdbc.py diff --git a/tests/providers/apache/spark/operators/test_spark_sql.py b/providers/tests/apache/spark/operators/test_spark_sql.py similarity index 100% rename from tests/providers/apache/spark/operators/test_spark_sql.py rename to providers/tests/apache/spark/operators/test_spark_sql.py diff --git a/tests/providers/apache/spark/operators/test_spark_submit.py b/providers/tests/apache/spark/operators/test_spark_submit.py similarity index 100% rename from tests/providers/apache/spark/operators/test_spark_submit.py rename to providers/tests/apache/spark/operators/test_spark_submit.py diff --git a/tests/providers/amazon/aws/links/__init__.py b/providers/tests/apprise/__init__.py similarity index 100% rename from tests/providers/amazon/aws/links/__init__.py rename to providers/tests/apprise/__init__.py diff --git a/tests/providers/apache/spark/hooks/__init__.py b/providers/tests/apprise/hooks/__init__.py similarity index 100% rename from tests/providers/apache/spark/hooks/__init__.py rename to providers/tests/apprise/hooks/__init__.py diff --git a/tests/providers/apprise/hooks/test_apprise.py b/providers/tests/apprise/hooks/test_apprise.py similarity index 100% rename from tests/providers/apprise/hooks/test_apprise.py rename to providers/tests/apprise/hooks/test_apprise.py diff --git a/tests/providers/apache/spark/operators/__init__.py b/providers/tests/apprise/notifications/__init__.py similarity index 100% rename from tests/providers/apache/spark/operators/__init__.py rename to providers/tests/apprise/notifications/__init__.py diff --git a/tests/providers/apprise/notifications/test_apprise.py b/providers/tests/apprise/notifications/test_apprise.py similarity index 100% rename from tests/providers/apprise/notifications/test_apprise.py rename to providers/tests/apprise/notifications/test_apprise.py diff --git a/tests/providers/amazon/aws/log/__init__.py b/providers/tests/arangodb/__init__.py similarity index 100% rename from tests/providers/amazon/aws/log/__init__.py rename to providers/tests/arangodb/__init__.py diff --git a/tests/providers/amazon/aws/notifications/__init__.py b/providers/tests/arangodb/hooks/__init__.py similarity index 100% rename from tests/providers/amazon/aws/notifications/__init__.py rename to providers/tests/arangodb/hooks/__init__.py diff --git a/tests/providers/arangodb/hooks/test_arangodb.py b/providers/tests/arangodb/hooks/test_arangodb.py similarity index 100% rename from tests/providers/arangodb/hooks/test_arangodb.py rename to providers/tests/arangodb/hooks/test_arangodb.py diff --git a/tests/providers/amazon/aws/operators/__init__.py b/providers/tests/arangodb/operators/__init__.py similarity index 100% rename from tests/providers/amazon/aws/operators/__init__.py rename to providers/tests/arangodb/operators/__init__.py diff --git a/tests/providers/arangodb/operators/test_arangodb.py b/providers/tests/arangodb/operators/test_arangodb.py similarity index 100% rename from tests/providers/arangodb/operators/test_arangodb.py rename to providers/tests/arangodb/operators/test_arangodb.py diff --git a/tests/providers/amazon/aws/secrets/__init__.py b/providers/tests/arangodb/sensors/__init__.py similarity index 100% rename from tests/providers/amazon/aws/secrets/__init__.py rename to providers/tests/arangodb/sensors/__init__.py diff --git a/tests/providers/arangodb/sensors/test_arangodb.py b/providers/tests/arangodb/sensors/test_arangodb.py similarity index 100% rename from tests/providers/arangodb/sensors/test_arangodb.py rename to providers/tests/arangodb/sensors/test_arangodb.py diff --git a/tests/providers/apprise/hooks/__init__.py b/providers/tests/asana/__init__.py similarity index 100% rename from tests/providers/apprise/hooks/__init__.py rename to providers/tests/asana/__init__.py diff --git a/tests/providers/apprise/notifications/__init__.py b/providers/tests/asana/hooks/__init__.py similarity index 100% rename from tests/providers/apprise/notifications/__init__.py rename to providers/tests/asana/hooks/__init__.py diff --git a/tests/providers/asana/hooks/test_asana.py b/providers/tests/asana/hooks/test_asana.py similarity index 100% rename from tests/providers/asana/hooks/test_asana.py rename to providers/tests/asana/hooks/test_asana.py diff --git a/tests/providers/asana/__init__.py b/providers/tests/asana/operators/__init__.py similarity index 100% rename from tests/providers/asana/__init__.py rename to providers/tests/asana/operators/__init__.py diff --git a/tests/providers/asana/operators/test_asana_tasks.py b/providers/tests/asana/operators/test_asana_tasks.py similarity index 100% rename from tests/providers/asana/operators/test_asana_tasks.py rename to providers/tests/asana/operators/test_asana_tasks.py diff --git a/tests/providers/amazon/aws/sensors/__init__.py b/providers/tests/atlassian/__init__.py similarity index 100% rename from tests/providers/amazon/aws/sensors/__init__.py rename to providers/tests/atlassian/__init__.py diff --git a/tests/providers/amazon/aws/system/__init__.py b/providers/tests/atlassian/jira/__init__.py similarity index 100% rename from tests/providers/amazon/aws/system/__init__.py rename to providers/tests/atlassian/jira/__init__.py diff --git a/tests/providers/amazon/aws/system/utils/__init__.py b/providers/tests/atlassian/jira/hooks/__init__.py similarity index 100% rename from tests/providers/amazon/aws/system/utils/__init__.py rename to providers/tests/atlassian/jira/hooks/__init__.py diff --git a/tests/providers/atlassian/jira/hooks/test_jira.py b/providers/tests/atlassian/jira/hooks/test_jira.py similarity index 98% rename from tests/providers/atlassian/jira/hooks/test_jira.py rename to providers/tests/atlassian/jira/hooks/test_jira.py index 339274e317c52..e2cf9389471e7 100644 --- a/tests/providers/atlassian/jira/hooks/test_jira.py +++ b/providers/tests/atlassian/jira/hooks/test_jira.py @@ -24,7 +24,8 @@ from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.atlassian.jira.hooks.jira import JiraHook -from tests.test_utils.compat import connection_as_json + +from dev.tests_common.test_utils.compat import connection_as_json @pytest.fixture diff --git a/tests/providers/amazon/aws/transfers/__init__.py b/providers/tests/atlassian/jira/notifications/__init__.py similarity index 100% rename from tests/providers/amazon/aws/transfers/__init__.py rename to providers/tests/atlassian/jira/notifications/__init__.py diff --git a/tests/providers/atlassian/jira/notifications/test_jira.py b/providers/tests/atlassian/jira/notifications/test_jira.py similarity index 100% rename from tests/providers/atlassian/jira/notifications/test_jira.py rename to providers/tests/atlassian/jira/notifications/test_jira.py diff --git a/tests/providers/amazon/aws/triggers/__init__.py b/providers/tests/atlassian/jira/operators/__init__.py similarity index 100% rename from tests/providers/amazon/aws/triggers/__init__.py rename to providers/tests/atlassian/jira/operators/__init__.py diff --git a/tests/providers/atlassian/jira/operators/test_jira.py b/providers/tests/atlassian/jira/operators/test_jira.py similarity index 98% rename from tests/providers/atlassian/jira/operators/test_jira.py rename to providers/tests/atlassian/jira/operators/test_jira.py index 9282089bae567..614c1a4939a90 100644 --- a/tests/providers/atlassian/jira/operators/test_jira.py +++ b/providers/tests/atlassian/jira/operators/test_jira.py @@ -24,7 +24,8 @@ from airflow.models import Connection from airflow.providers.atlassian.jira.operators.jira import JiraOperator from airflow.utils import timezone -from tests.test_utils.compat import connection_as_json + +from dev.tests_common.test_utils.compat import connection_as_json DEFAULT_DATE = timezone.datetime(2017, 1, 1) MINIMAL_TEST_TICKET = { diff --git a/tests/providers/amazon/aws/utils/__init__.py b/providers/tests/atlassian/jira/sensors/__init__.py similarity index 100% rename from tests/providers/amazon/aws/utils/__init__.py rename to providers/tests/atlassian/jira/sensors/__init__.py diff --git a/tests/providers/atlassian/jira/sensors/test_jira.py b/providers/tests/atlassian/jira/sensors/test_jira.py similarity index 97% rename from tests/providers/atlassian/jira/sensors/test_jira.py rename to providers/tests/atlassian/jira/sensors/test_jira.py index 5e2ce896b6c67..9cb2572969ad4 100644 --- a/tests/providers/atlassian/jira/sensors/test_jira.py +++ b/providers/tests/atlassian/jira/sensors/test_jira.py @@ -24,7 +24,8 @@ from airflow.models import Connection from airflow.providers.atlassian.jira.sensors.jira import JiraTicketSensor from airflow.utils import timezone -from tests.test_utils.compat import connection_as_json + +from dev.tests_common.test_utils.compat import connection_as_json DEFAULT_DATE = timezone.datetime(2017, 1, 1) MINIMAL_TEST_TICKET = { diff --git a/tests/providers/asana/hooks/__init__.py b/providers/tests/celery/__init__.py similarity index 100% rename from tests/providers/asana/hooks/__init__.py rename to providers/tests/celery/__init__.py diff --git a/tests/providers/amazon/aws/waiters/__init__.py b/providers/tests/celery/cli/__init__.py similarity index 100% rename from tests/providers/amazon/aws/waiters/__init__.py rename to providers/tests/celery/cli/__init__.py diff --git a/tests/providers/celery/cli/test_celery_command.py b/providers/tests/celery/cli/test_celery_command.py similarity index 99% rename from tests/providers/celery/cli/test_celery_command.py rename to providers/tests/celery/cli/test_celery_command.py index d7dcdc4741918..03f5d60dabe40 100644 --- a/tests/providers/celery/cli/test_celery_command.py +++ b/providers/tests/celery/cli/test_celery_command.py @@ -30,8 +30,9 @@ from airflow.configuration import conf from airflow.executors import executor_loader from airflow.providers.celery.cli import celery_command -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/asana/operators/__init__.py b/providers/tests/celery/executors/__init__.py similarity index 100% rename from tests/providers/asana/operators/__init__.py rename to providers/tests/celery/executors/__init__.py diff --git a/tests/providers/celery/executors/test_celery_executor.py b/providers/tests/celery/executors/test_celery_executor.py similarity index 96% rename from tests/providers/celery/executors/test_celery_executor.py rename to providers/tests/celery/executors/test_celery_executor.py index 63ca4a47d255e..261a11f837e42 100644 --- a/tests/providers/celery/executors/test_celery_executor.py +++ b/providers/tests/celery/executors/test_celery_executor.py @@ -41,9 +41,10 @@ from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.utils import timezone from airflow.utils.state import State -from tests.test_utils import db -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test @@ -165,11 +166,15 @@ def test_command_validation(self, command, raise_exception): ValueError, match=r'The command must start with \["airflow", "tasks", "run"\]\.' ) - with mock.patch( - "airflow.providers.celery.executors.celery_executor_utils._execute_in_subprocess" - ) as mock_subproc, mock.patch( - "airflow.providers.celery.executors.celery_executor_utils._execute_in_fork" - ) as mock_fork, mock.patch("celery.app.task.Task.request") as mock_task: + with ( + mock.patch( + "airflow.providers.celery.executors.celery_executor_utils._execute_in_subprocess" + ) as mock_subproc, + mock.patch( + "airflow.providers.celery.executors.celery_executor_utils._execute_in_fork" + ) as mock_fork, + mock.patch("celery.app.task.Task.request") as mock_task, + ): mock_task.id = "abcdef-124215-abcdef" with expected_context: celery_executor_utils.execute_command(command) diff --git a/tests/providers/celery/executors/test_celery_kubernetes_executor.py b/providers/tests/celery/executors/test_celery_kubernetes_executor.py similarity index 100% rename from tests/providers/celery/executors/test_celery_kubernetes_executor.py rename to providers/tests/celery/executors/test_celery_kubernetes_executor.py diff --git a/tests/providers/apache/__init__.py b/providers/tests/celery/log_handlers/__init__.py similarity index 100% rename from tests/providers/apache/__init__.py rename to providers/tests/celery/log_handlers/__init__.py diff --git a/tests/providers/celery/log_handlers/test_log_handlers.py b/providers/tests/celery/log_handlers/test_log_handlers.py similarity index 96% rename from tests/providers/celery/log_handlers/test_log_handlers.py rename to providers/tests/celery/log_handlers/test_log_handlers.py index 5e93b076e41ea..9eb9e33e2ae0d 100644 --- a/tests/providers/celery/log_handlers/test_log_handlers.py +++ b/providers/tests/celery/log_handlers/test_log_handlers.py @@ -35,8 +35,9 @@ from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: pass diff --git a/tests/providers/celery/__init__.py b/providers/tests/celery/sensors/__init__.py similarity index 100% rename from tests/providers/celery/__init__.py rename to providers/tests/celery/sensors/__init__.py diff --git a/tests/providers/celery/sensors/test_celery_queue.py b/providers/tests/celery/sensors/test_celery_queue.py similarity index 100% rename from tests/providers/celery/sensors/test_celery_queue.py rename to providers/tests/celery/sensors/test_celery_queue.py diff --git a/tests/providers/celery/executors/__init__.py b/providers/tests/cloudant/__init__.py similarity index 100% rename from tests/providers/celery/executors/__init__.py rename to providers/tests/cloudant/__init__.py diff --git a/tests/providers/celery/sensors/__init__.py b/providers/tests/cloudant/hooks/__init__.py similarity index 100% rename from tests/providers/celery/sensors/__init__.py rename to providers/tests/cloudant/hooks/__init__.py diff --git a/tests/providers/cloudant/hooks/test_cloudant.py b/providers/tests/cloudant/hooks/test_cloudant.py similarity index 100% rename from tests/providers/cloudant/hooks/test_cloudant.py rename to providers/tests/cloudant/hooks/test_cloudant.py diff --git a/tests/providers/apache/beam/__init__.py b/providers/tests/cncf/__init__.py similarity index 100% rename from tests/providers/apache/beam/__init__.py rename to providers/tests/cncf/__init__.py diff --git a/tests/providers/cloudant/__init__.py b/providers/tests/cncf/kubernetes/__init__.py similarity index 100% rename from tests/providers/cloudant/__init__.py rename to providers/tests/cncf/kubernetes/__init__.py diff --git a/tests/providers/apache/beam/hooks/__init__.py b/providers/tests/cncf/kubernetes/cli/__init__.py similarity index 100% rename from tests/providers/apache/beam/hooks/__init__.py rename to providers/tests/cncf/kubernetes/cli/__init__.py diff --git a/tests/providers/cncf/kubernetes/cli/test_kubernetes_command.py b/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py similarity index 99% rename from tests/providers/cncf/kubernetes/cli/test_kubernetes_command.py rename to providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py index 790801a3bbcf9..62019a8d2fe90 100644 --- a/tests/providers/cncf/kubernetes/cli/test_kubernetes_command.py +++ b/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py @@ -28,7 +28,8 @@ from airflow.cli import cli_parser from airflow.executors import executor_loader from airflow.providers.cncf.kubernetes.cli import kubernetes_command -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/cncf/kubernetes/conftest.py b/providers/tests/cncf/kubernetes/conftest.py similarity index 100% rename from tests/providers/cncf/kubernetes/conftest.py rename to providers/tests/cncf/kubernetes/conftest.py diff --git a/tests/providers/apache/beam/operators/__init__.py b/providers/tests/cncf/kubernetes/data_files/__init__.py similarity index 100% rename from tests/providers/apache/beam/operators/__init__.py rename to providers/tests/cncf/kubernetes/data_files/__init__.py diff --git a/tests/providers/apache/beam/triggers/__init__.py b/providers/tests/cncf/kubernetes/data_files/executor/__init__.py similarity index 100% rename from tests/providers/apache/beam/triggers/__init__.py rename to providers/tests/cncf/kubernetes/data_files/executor/__init__.py diff --git a/tests/providers/cncf/kubernetes/data_files/executor/basic_template.yaml b/providers/tests/cncf/kubernetes/data_files/executor/basic_template.yaml similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/executor/basic_template.yaml rename to providers/tests/cncf/kubernetes/data_files/executor/basic_template.yaml diff --git a/tests/providers/cncf/kubernetes/data_files/kube_config b/providers/tests/cncf/kubernetes/data_files/kube_config similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/kube_config rename to providers/tests/cncf/kubernetes/data_files/kube_config diff --git a/tests/providers/apache/cassandra/__init__.py b/providers/tests/cncf/kubernetes/data_files/pods/__init__.py similarity index 100% rename from tests/providers/apache/cassandra/__init__.py rename to providers/tests/cncf/kubernetes/data_files/pods/__init__.py diff --git a/tests/providers/cncf/kubernetes/data_files/pods/generator_base.yaml b/providers/tests/cncf/kubernetes/data_files/pods/generator_base.yaml similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/pods/generator_base.yaml rename to providers/tests/cncf/kubernetes/data_files/pods/generator_base.yaml diff --git a/tests/providers/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml b/providers/tests/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml rename to providers/tests/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml diff --git a/tests/providers/cncf/kubernetes/data_files/pods/template.yaml b/providers/tests/cncf/kubernetes/data_files/pods/template.yaml similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/pods/template.yaml rename to providers/tests/cncf/kubernetes/data_files/pods/template.yaml diff --git a/tests/providers/apache/cassandra/sensors/__init__.py b/providers/tests/cncf/kubernetes/data_files/spark/__init__.py similarity index 100% rename from tests/providers/apache/cassandra/sensors/__init__.py rename to providers/tests/cncf/kubernetes/data_files/spark/__init__.py diff --git a/tests/providers/cncf/kubernetes/data_files/spark/application_template.yaml b/providers/tests/cncf/kubernetes/data_files/spark/application_template.yaml similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/spark/application_template.yaml rename to providers/tests/cncf/kubernetes/data_files/spark/application_template.yaml diff --git a/tests/providers/cncf/kubernetes/data_files/spark/application_test.json b/providers/tests/cncf/kubernetes/data_files/spark/application_test.json similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/spark/application_test.json rename to providers/tests/cncf/kubernetes/data_files/spark/application_test.json diff --git a/tests/providers/cncf/kubernetes/data_files/spark/application_test.yaml b/providers/tests/cncf/kubernetes/data_files/spark/application_test.yaml similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/spark/application_test.yaml rename to providers/tests/cncf/kubernetes/data_files/spark/application_test.yaml diff --git a/tests/providers/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json b/providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json rename to providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json diff --git a/tests/providers/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml b/providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml rename to providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml diff --git a/tests/providers/apache/druid/transfers/__init__.py b/providers/tests/cncf/kubernetes/decorators/__init__.py similarity index 100% rename from tests/providers/apache/druid/transfers/__init__.py rename to providers/tests/cncf/kubernetes/decorators/__init__.py diff --git a/tests/providers/cncf/kubernetes/decorators/test_kubernetes.py b/providers/tests/cncf/kubernetes/decorators/test_kubernetes.py similarity index 100% rename from tests/providers/cncf/kubernetes/decorators/test_kubernetes.py rename to providers/tests/cncf/kubernetes/decorators/test_kubernetes.py diff --git a/tests/providers/apache/hive/transfers/__init__.py b/providers/tests/cncf/kubernetes/executors/__init__.py similarity index 100% rename from tests/providers/apache/hive/transfers/__init__.py rename to providers/tests/cncf/kubernetes/executors/__init__.py diff --git a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py similarity index 99% rename from tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py rename to providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py index 12435426dd899..5240bf0faecb2 100644 --- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -54,8 +54,9 @@ from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator from airflow.utils import timezone from airflow.utils.state import State, TaskInstanceState -from tests.test_utils.compat import BashOperator -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import BashOperator +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.skip_if_database_isolation_mode @@ -1608,12 +1609,15 @@ def setup_method(self): self.events = [] def _run(self): - with mock.patch( - "airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils.watch" - ) as mock_watch, mock.patch.object( - KubernetesJobWatcher, - "_pod_events", - ) as mock_pod_events: + with ( + mock.patch( + "airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils.watch" + ) as mock_watch, + mock.patch.object( + KubernetesJobWatcher, + "_pod_events", + ) as mock_pod_events, + ): mock_watch.Watch.return_value.stream.return_value = self.events mock_pod_events.return_value = self.events latest_resource_version = self.watcher._run( diff --git a/tests/providers/cncf/kubernetes/executors/test_local_kubernetes_executor.py b/providers/tests/cncf/kubernetes/executors/test_local_kubernetes_executor.py similarity index 100% rename from tests/providers/cncf/kubernetes/executors/test_local_kubernetes_executor.py rename to providers/tests/cncf/kubernetes/executors/test_local_kubernetes_executor.py diff --git a/tests/providers/apache/iceberg/__init__.py b/providers/tests/cncf/kubernetes/hooks/__init__.py similarity index 100% rename from tests/providers/apache/iceberg/__init__.py rename to providers/tests/cncf/kubernetes/hooks/__init__.py diff --git a/tests/providers/cncf/kubernetes/hooks/test_kubernetes.py b/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py similarity index 99% rename from tests/providers/cncf/kubernetes/hooks/test_kubernetes.py rename to providers/tests/cncf/kubernetes/hooks/test_kubernetes.py index 065768def24ea..a6847e9c2fe56 100644 --- a/tests/providers/cncf/kubernetes/hooks/test_kubernetes.py +++ b/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py @@ -36,8 +36,9 @@ from airflow.providers.cncf.kubernetes.hooks.kubernetes import AsyncKubernetesHook, KubernetesHook from airflow.utils import db from airflow.utils.db import merge_conn -from tests.test_utils.db import clear_db_connections -from tests.test_utils.providers import get_provider_min_airflow_version + +from dev.tests_common.test_utils.db import clear_db_connections +from dev.tests_common.test_utils.providers import get_provider_min_airflow_version pytestmark = pytest.mark.db_test @@ -677,8 +678,9 @@ class TestKubernetesHookIncorrectConfiguration: ) def test_should_raise_exception_on_invalid_configuration(self, conn_uri): kubernetes_hook = KubernetesHook() - with mock.patch.dict("os.environ", AIRFLOW_CONN_KUBERNETES_DEFAULT=conn_uri), pytest.raises( - AirflowException, match="Invalid connection configuration" + with ( + mock.patch.dict("os.environ", AIRFLOW_CONN_KUBERNETES_DEFAULT=conn_uri), + pytest.raises(AirflowException, match="Invalid connection configuration"), ): kubernetes_hook.get_conn() diff --git a/tests/providers/apache/iceberg/hooks/__init__.py b/providers/tests/cncf/kubernetes/log_handlers/__init__.py similarity index 100% rename from tests/providers/apache/iceberg/hooks/__init__.py rename to providers/tests/cncf/kubernetes/log_handlers/__init__.py diff --git a/tests/providers/cncf/kubernetes/log_handlers/test_log_handlers.py b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py similarity index 98% rename from tests/providers/cncf/kubernetes/log_handlers/test_log_handlers.py rename to providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py index 3f9bc963f0e60..b02f16ac92786 100644 --- a/tests/providers/cncf/kubernetes/log_handlers/test_log_handlers.py +++ b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py @@ -41,8 +41,9 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/cloudant/hooks/__init__.py b/providers/tests/cncf/kubernetes/models/__init__.py similarity index 100% rename from tests/providers/cloudant/hooks/__init__.py rename to providers/tests/cncf/kubernetes/models/__init__.py diff --git a/tests/providers/cncf/kubernetes/models/test_secret.py b/providers/tests/cncf/kubernetes/models/test_secret.py similarity index 100% rename from tests/providers/cncf/kubernetes/models/test_secret.py rename to providers/tests/cncf/kubernetes/models/test_secret.py diff --git a/tests/providers/apache/impala/__init__.py b/providers/tests/cncf/kubernetes/operators/__init__.py similarity index 100% rename from tests/providers/apache/impala/__init__.py rename to providers/tests/cncf/kubernetes/operators/__init__.py diff --git a/tests/providers/cncf/kubernetes/operators/test_custom_object_launcher.py b/providers/tests/cncf/kubernetes/operators/test_custom_object_launcher.py similarity index 100% rename from tests/providers/cncf/kubernetes/operators/test_custom_object_launcher.py rename to providers/tests/cncf/kubernetes/operators/test_custom_object_launcher.py diff --git a/tests/providers/cncf/kubernetes/operators/test_job.py b/providers/tests/cncf/kubernetes/operators/test_job.py similarity index 100% rename from tests/providers/cncf/kubernetes/operators/test_job.py rename to providers/tests/cncf/kubernetes/operators/test_job.py diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/providers/tests/cncf/kubernetes/operators/test_pod.py similarity index 99% rename from tests/providers/cncf/kubernetes/operators/test_pod.py rename to providers/tests/cncf/kubernetes/operators/test_pod.py index a4ccb4b44b4bf..ac50641370fed 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/providers/tests/cncf/kubernetes/operators/test_pod.py @@ -50,7 +50,8 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.types import DagRunType -from tests.test_utils import db + +from dev.tests_common.test_utils import db pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] @@ -1513,7 +1514,8 @@ def test_get_logs_but_not_for_base_container( @patch(KUB_OP_PATH.format("find_pod")) def test_execute_sync_callbacks(self, find_pod_mock): from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode - from tests.providers.cncf.kubernetes.test_callbacks import ( + + from providers.tests.cncf.kubernetes.test_callbacks import ( MockKubernetesPodOperatorCallback, MockWrapper, ) @@ -1578,7 +1580,8 @@ def test_execute_sync_callbacks(self, find_pod_mock): @patch(HOOK_CLASS, new=MagicMock) def test_execute_async_callbacks(self): from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode - from tests.providers.cncf.kubernetes.test_callbacks import ( + + from providers.tests.cncf.kubernetes.test_callbacks import ( MockKubernetesPodOperatorCallback, MockWrapper, ) diff --git a/tests/providers/cncf/kubernetes/operators/test_resource.py b/providers/tests/cncf/kubernetes/operators/test_resource.py similarity index 100% rename from tests/providers/cncf/kubernetes/operators/test_resource.py rename to providers/tests/cncf/kubernetes/operators/test_resource.py diff --git a/tests/providers/cncf/kubernetes/operators/test_spark_kubernetes.py b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py similarity index 99% rename from tests/providers/cncf/kubernetes/operators/test_spark_kubernetes.py rename to providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py index 9c8c40de6558d..18bce53688d4f 100644 --- a/tests/providers/cncf/kubernetes/operators/test_spark_kubernetes.py +++ b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py @@ -33,7 +33,8 @@ from airflow.providers.cncf.kubernetes.operators.spark_kubernetes import SparkKubernetesOperator from airflow.utils import db, timezone from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS @patch("airflow.providers.cncf.kubernetes.operators.spark_kubernetes.KubernetesHook") diff --git a/tests/providers/apache/impala/hooks/__init__.py b/providers/tests/cncf/kubernetes/resource_convert/__init__.py similarity index 100% rename from tests/providers/apache/impala/hooks/__init__.py rename to providers/tests/cncf/kubernetes/resource_convert/__init__.py diff --git a/tests/providers/cncf/kubernetes/resource_convert/test_configmap.py b/providers/tests/cncf/kubernetes/resource_convert/test_configmap.py similarity index 100% rename from tests/providers/cncf/kubernetes/resource_convert/test_configmap.py rename to providers/tests/cncf/kubernetes/resource_convert/test_configmap.py diff --git a/tests/providers/cncf/kubernetes/resource_convert/test_env_variable.py b/providers/tests/cncf/kubernetes/resource_convert/test_env_variable.py similarity index 100% rename from tests/providers/cncf/kubernetes/resource_convert/test_env_variable.py rename to providers/tests/cncf/kubernetes/resource_convert/test_env_variable.py diff --git a/tests/providers/cncf/kubernetes/resource_convert/test_secret.py b/providers/tests/cncf/kubernetes/resource_convert/test_secret.py similarity index 100% rename from tests/providers/cncf/kubernetes/resource_convert/test_secret.py rename to providers/tests/cncf/kubernetes/resource_convert/test_secret.py diff --git a/tests/providers/apache/kafka/__init__.py b/providers/tests/cncf/kubernetes/sensors/__init__.py similarity index 100% rename from tests/providers/apache/kafka/__init__.py rename to providers/tests/cncf/kubernetes/sensors/__init__.py diff --git a/tests/providers/cncf/kubernetes/sensors/test_spark_kubernetes.py b/providers/tests/cncf/kubernetes/sensors/test_spark_kubernetes.py similarity index 100% rename from tests/providers/cncf/kubernetes/sensors/test_spark_kubernetes.py rename to providers/tests/cncf/kubernetes/sensors/test_spark_kubernetes.py diff --git a/tests/providers/cncf/kubernetes/test_callbacks.py b/providers/tests/cncf/kubernetes/test_callbacks.py similarity index 100% rename from tests/providers/cncf/kubernetes/test_callbacks.py rename to providers/tests/cncf/kubernetes/test_callbacks.py diff --git a/tests/providers/cncf/kubernetes/test_client.py b/providers/tests/cncf/kubernetes/test_client.py similarity index 98% rename from tests/providers/cncf/kubernetes/test_client.py rename to providers/tests/cncf/kubernetes/test_client.py index 1384068fd2860..269717ffaa0f0 100644 --- a/tests/providers/cncf/kubernetes/test_client.py +++ b/providers/tests/cncf/kubernetes/test_client.py @@ -28,7 +28,8 @@ _enable_tcp_keepalive, get_kube_client, ) -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars class TestClient: diff --git a/tests/providers/cncf/kubernetes/test_kubernetes_helper_functions.py b/providers/tests/cncf/kubernetes/test_kubernetes_helper_functions.py similarity index 100% rename from tests/providers/cncf/kubernetes/test_kubernetes_helper_functions.py rename to providers/tests/cncf/kubernetes/test_kubernetes_helper_functions.py diff --git a/tests/providers/cncf/kubernetes/test_pod_generator.py b/providers/tests/cncf/kubernetes/test_pod_generator.py similarity index 100% rename from tests/providers/cncf/kubernetes/test_pod_generator.py rename to providers/tests/cncf/kubernetes/test_pod_generator.py diff --git a/tests/providers/cncf/kubernetes/test_template_rendering.py b/providers/tests/cncf/kubernetes/test_template_rendering.py similarity index 96% rename from tests/providers/cncf/kubernetes/test_template_rendering.py rename to providers/tests/cncf/kubernetes/test_template_rendering.py index 4c087d6040e68..180b7a1e2d2ef 100644 --- a/tests/providers/cncf/kubernetes/test_template_rendering.py +++ b/providers/tests/cncf/kubernetes/test_template_rendering.py @@ -22,16 +22,18 @@ import pytest from sqlalchemy.orm import make_transient -from airflow.configuration import TEST_DAGS_FOLDER from airflow.models.renderedtifields import RenderedTaskInstanceFields, RenderedTaskInstanceFields as RTIF from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec, render_k8s_pod_yaml +from airflow.utils import timezone from airflow.utils.session import create_session from airflow.version import version -from tests.models import DEFAULT_DATE -from tests.test_utils.compat import BashOperator + +from dev.tests_common.test_utils.compat import BashOperator pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] +DEFAULT_DATE = timezone.datetime(2021, 9, 9) + @mock.patch.dict(os.environ, {"AIRFLOW_IS_K8S_EXECUTOR_POD": "True"}) @mock.patch("airflow.settings.pod_mutation_hook") @@ -126,7 +128,7 @@ def test_get_k8s_pod_yaml(render_k8s_pod_yaml, redact, dag_maker, session): with dag_maker("test_get_k8s_pod_yaml") as dag: task = BashOperator(task_id="test", bash_command="echo hi") dr = dag_maker.create_dagrun() - dag.fileloc = TEST_DAGS_FOLDER + "/test_get_k8s_pod_yaml.py" + dag.fileloc = "/test_get_k8s_pod_yaml.py" ti = dr.task_instances[0] ti.task = task diff --git a/tests/providers/apache/kafka/hooks/__init__.py b/providers/tests/cncf/kubernetes/triggers/__init__.py similarity index 100% rename from tests/providers/apache/kafka/hooks/__init__.py rename to providers/tests/cncf/kubernetes/triggers/__init__.py diff --git a/tests/providers/cncf/kubernetes/triggers/test_job.py b/providers/tests/cncf/kubernetes/triggers/test_job.py similarity index 100% rename from tests/providers/cncf/kubernetes/triggers/test_job.py rename to providers/tests/cncf/kubernetes/triggers/test_job.py diff --git a/tests/providers/cncf/kubernetes/triggers/test_pod.py b/providers/tests/cncf/kubernetes/triggers/test_pod.py similarity index 100% rename from tests/providers/cncf/kubernetes/triggers/test_pod.py rename to providers/tests/cncf/kubernetes/triggers/test_pod.py diff --git a/tests/providers/apache/kafka/operators/__init__.py b/providers/tests/cncf/kubernetes/utils/__init__.py similarity index 100% rename from tests/providers/apache/kafka/operators/__init__.py rename to providers/tests/cncf/kubernetes/utils/__init__.py diff --git a/tests/providers/cncf/kubernetes/utils/test_k8s_resource_iterator.py b/providers/tests/cncf/kubernetes/utils/test_k8s_resource_iterator.py similarity index 100% rename from tests/providers/cncf/kubernetes/utils/test_k8s_resource_iterator.py rename to providers/tests/cncf/kubernetes/utils/test_k8s_resource_iterator.py diff --git a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py b/providers/tests/cncf/kubernetes/utils/test_pod_manager.py similarity index 99% rename from tests/providers/cncf/kubernetes/utils/test_pod_manager.py rename to providers/tests/cncf/kubernetes/utils/test_pod_manager.py index 73dac5255d620..b577ea969ea3d 100644 --- a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py +++ b/providers/tests/cncf/kubernetes/utils/test_pod_manager.py @@ -40,7 +40,8 @@ container_is_terminated, ) from airflow.utils.timezone import utc -from tests.providers.cncf.kubernetes.test_callbacks import MockKubernetesPodOperatorCallback, MockWrapper + +from providers.tests.cncf.kubernetes.test_callbacks import MockKubernetesPodOperatorCallback, MockWrapper if TYPE_CHECKING: from pendulum import DateTime diff --git a/tests/providers/apache/kafka/sensors/__init__.py b/providers/tests/cohere/__init__.py similarity index 100% rename from tests/providers/apache/kafka/sensors/__init__.py rename to providers/tests/cohere/__init__.py diff --git a/tests/providers/apache/kafka/triggers/__init__.py b/providers/tests/cohere/hooks/__init__.py similarity index 100% rename from tests/providers/apache/kafka/triggers/__init__.py rename to providers/tests/cohere/hooks/__init__.py diff --git a/tests/providers/cohere/hooks/test_cohere.py b/providers/tests/cohere/hooks/test_cohere.py similarity index 84% rename from tests/providers/cohere/hooks/test_cohere.py rename to providers/tests/cohere/hooks/test_cohere.py index 8f566ec0c6405..28aef3ebaf795 100644 --- a/tests/providers/cohere/hooks/test_cohere.py +++ b/providers/tests/cohere/hooks/test_cohere.py @@ -34,11 +34,14 @@ def test__get_api_key(self): api_url = "http://some_host.com" timeout = 150 max_retries = 5 - with patch.object( - CohereHook, - "get_connection", - return_value=Connection(conn_type="cohere", password=api_key, host=api_url), - ), patch("cohere.Client") as client: + with ( + patch.object( + CohereHook, + "get_connection", + return_value=Connection(conn_type="cohere", password=api_key, host=api_url), + ), + patch("cohere.Client") as client, + ): hook = CohereHook(timeout=timeout, max_retries=max_retries) _ = hook.get_conn client.assert_called_once_with( diff --git a/tests/providers/apache/livy/__init__.py b/providers/tests/cohere/operators/__init__.py similarity index 100% rename from tests/providers/apache/livy/__init__.py rename to providers/tests/cohere/operators/__init__.py diff --git a/tests/providers/cohere/operators/test_embedding.py b/providers/tests/cohere/operators/test_embedding.py similarity index 100% rename from tests/providers/cohere/operators/test_embedding.py rename to providers/tests/cohere/operators/test_embedding.py diff --git a/tests/providers/apache/livy/hooks/__init__.py b/providers/tests/common/__init__.py similarity index 100% rename from tests/providers/apache/livy/hooks/__init__.py rename to providers/tests/common/__init__.py diff --git a/tests/providers/apache/livy/operators/__init__.py b/providers/tests/common/compat/__init__.py similarity index 100% rename from tests/providers/apache/livy/operators/__init__.py rename to providers/tests/common/compat/__init__.py diff --git a/tests/providers/apache/livy/sensors/__init__.py b/providers/tests/common/compat/lineage/__init__.py similarity index 100% rename from tests/providers/apache/livy/sensors/__init__.py rename to providers/tests/common/compat/lineage/__init__.py diff --git a/tests/providers/common/compat/lineage/test_hook.py b/providers/tests/common/compat/lineage/test_hook.py similarity index 100% rename from tests/providers/common/compat/lineage/test_hook.py rename to providers/tests/common/compat/lineage/test_hook.py diff --git a/tests/providers/apache/livy/triggers/__init__.py b/providers/tests/common/compat/openlineage/__init__.py similarity index 100% rename from tests/providers/apache/livy/triggers/__init__.py rename to providers/tests/common/compat/openlineage/__init__.py diff --git a/tests/providers/common/compat/openlineage/test_facet.py b/providers/tests/common/compat/openlineage/test_facet.py similarity index 100% rename from tests/providers/common/compat/openlineage/test_facet.py rename to providers/tests/common/compat/openlineage/test_facet.py diff --git a/tests/providers/apache/pig/operators/__init__.py b/providers/tests/common/compat/openlineage/utils/__init__.py similarity index 100% rename from tests/providers/apache/pig/operators/__init__.py rename to providers/tests/common/compat/openlineage/utils/__init__.py diff --git a/tests/providers/common/compat/openlineage/utils/test_utils.py b/providers/tests/common/compat/openlineage/utils/test_utils.py similarity index 100% rename from tests/providers/common/compat/openlineage/utils/test_utils.py rename to providers/tests/common/compat/openlineage/utils/test_utils.py diff --git a/tests/providers/apache/spark/decorators/__init__.py b/providers/tests/common/compat/security/__init__.py similarity index 100% rename from tests/providers/apache/spark/decorators/__init__.py rename to providers/tests/common/compat/security/__init__.py diff --git a/tests/providers/common/compat/security/test_permissions.py b/providers/tests/common/compat/security/test_permissions.py similarity index 100% rename from tests/providers/common/compat/security/test_permissions.py rename to providers/tests/common/compat/security/test_permissions.py diff --git a/tests/providers/apprise/__init__.py b/providers/tests/common/io/__init__.py similarity index 100% rename from tests/providers/apprise/__init__.py rename to providers/tests/common/io/__init__.py diff --git a/tests/providers/arangodb/__init__.py b/providers/tests/common/io/assets/__init__.py similarity index 100% rename from tests/providers/arangodb/__init__.py rename to providers/tests/common/io/assets/__init__.py diff --git a/tests/providers/common/io/assets/test_file.py b/providers/tests/common/io/assets/test_file.py similarity index 100% rename from tests/providers/common/io/assets/test_file.py rename to providers/tests/common/io/assets/test_file.py diff --git a/tests/providers/arangodb/hooks/__init__.py b/providers/tests/common/io/operators/__init__.py similarity index 100% rename from tests/providers/arangodb/hooks/__init__.py rename to providers/tests/common/io/operators/__init__.py diff --git a/tests/providers/common/io/operators/test_file_transfer.py b/providers/tests/common/io/operators/test_file_transfer.py similarity index 97% rename from tests/providers/common/io/operators/test_file_transfer.py rename to providers/tests/common/io/operators/test_file_transfer.py index 698c33582b82c..55a196fa8918c 100644 --- a/tests/providers/common/io/operators/test_file_transfer.py +++ b/providers/tests/common/io/operators/test_file_transfer.py @@ -20,7 +20,8 @@ from unittest import mock from airflow.providers.common.compat.openlineage.facet import Dataset -from tests.test_utils.compat import ignore_provider_compatibility_error + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.8.0", __file__): from airflow.providers.common.io.operators.file_transfer import FileTransferOperator diff --git a/tests/providers/arangodb/operators/__init__.py b/providers/tests/common/io/xcom/__init__.py similarity index 100% rename from tests/providers/arangodb/operators/__init__.py rename to providers/tests/common/io/xcom/__init__.py diff --git a/tests/providers/common/io/xcom/test_backend.py b/providers/tests/common/io/xcom/test_backend.py similarity index 96% rename from tests/providers/common/io/xcom/test_backend.py rename to providers/tests/common/io/xcom/test_backend.py index b0516ad487d44..ae45a2f863b78 100644 --- a/tests/providers/common/io/xcom/test_backend.py +++ b/providers/tests/common/io/xcom/test_backend.py @@ -19,8 +19,8 @@ import pytest -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS, ignore_provider_compatibility_error -from tests.www.test_utils import is_db_isolation_mode +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS, ignore_provider_compatibility_error +from dev.tests_common.test_utils.db import is_db_isolation_mode pytestmark = [ pytest.mark.db_test, @@ -37,8 +37,9 @@ from airflow.utils import timezone from airflow.utils.xcom import XCOM_RETURN_KEY -from tests.test_utils import db -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.config import conf_vars @pytest.fixture(autouse=True) diff --git a/tests/providers/arangodb/sensors/__init__.py b/providers/tests/common/sql/__init__.py similarity index 100% rename from tests/providers/arangodb/sensors/__init__.py rename to providers/tests/common/sql/__init__.py diff --git a/tests/providers/atlassian/__init__.py b/providers/tests/common/sql/hooks/__init__.py similarity index 100% rename from tests/providers/atlassian/__init__.py rename to providers/tests/common/sql/hooks/__init__.py diff --git a/tests/providers/common/sql/hooks/test_dbapi.py b/providers/tests/common/sql/hooks/test_dbapi.py similarity index 99% rename from tests/providers/common/sql/hooks/test_dbapi.py rename to providers/tests/common/sql/hooks/test_dbapi.py index 6a744e9718da1..5017a5f347c82 100644 --- a/tests/providers/common/sql/hooks/test_dbapi.py +++ b/providers/tests/common/sql/hooks/test_dbapi.py @@ -29,7 +29,8 @@ from airflow.hooks.base import BaseHook from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler, fetch_one_handler -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), diff --git a/tests/providers/common/sql/hooks/test_sql.py b/providers/tests/common/sql/hooks/test_sql.py similarity index 98% rename from tests/providers/common/sql/hooks/test_sql.py rename to providers/tests/common/sql/hooks/test_sql.py index 9bc246192b77f..87673f6de62e0 100644 --- a/tests/providers/common/sql/hooks/test_sql.py +++ b/providers/tests/common/sql/hooks/test_sql.py @@ -30,8 +30,9 @@ from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler from airflow.utils.session import provide_session -from tests.providers.common.sql.test_utils import mock_hook -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS +from providers.tests.common.sql.test_utils import mock_hook pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), diff --git a/tests/providers/common/sql/hooks/test_sqlparse.py b/providers/tests/common/sql/hooks/test_sqlparse.py similarity index 96% rename from tests/providers/common/sql/hooks/test_sqlparse.py rename to providers/tests/common/sql/hooks/test_sqlparse.py index 1ce7cb5d2183f..b3622bd8eff21 100644 --- a/tests/providers/common/sql/hooks/test_sqlparse.py +++ b/providers/tests/common/sql/hooks/test_sqlparse.py @@ -19,7 +19,8 @@ import pytest from airflow.providers.common.sql.hooks.sql import DbApiHook -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), diff --git a/tests/providers/atlassian/jira/__init__.py b/providers/tests/common/sql/operators/__init__.py similarity index 100% rename from tests/providers/atlassian/jira/__init__.py rename to providers/tests/common/sql/operators/__init__.py diff --git a/tests/providers/common/sql/operators/test_sql.py b/providers/tests/common/sql/operators/test_sql.py similarity index 99% rename from tests/providers/common/sql/operators/test_sql.py rename to providers/tests/common/sql/operators/test_sql.py index 5144b05b07e2c..c2e4bc6f2e86f 100644 --- a/tests/providers/common/sql/operators/test_sql.py +++ b/providers/tests/common/sql/operators/test_sql.py @@ -43,7 +43,8 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/common/sql/operators/test_sql_execute.py b/providers/tests/common/sql/operators/test_sql_execute.py similarity index 99% rename from tests/providers/common/sql/operators/test_sql_execute.py rename to providers/tests/common/sql/operators/test_sql_execute.py index 1527f3190d5ba..bb42cbaba2026 100644 --- a/tests/providers/common/sql/operators/test_sql_execute.py +++ b/providers/tests/common/sql/operators/test_sql_execute.py @@ -33,7 +33,8 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.openlineage.extractors.base import OperatorLineage -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), diff --git a/tests/providers/atlassian/jira/hooks/__init__.py b/providers/tests/common/sql/sensors/__init__.py similarity index 100% rename from tests/providers/atlassian/jira/hooks/__init__.py rename to providers/tests/common/sql/sensors/__init__.py diff --git a/tests/providers/common/sql/sensors/test_sql.py b/providers/tests/common/sql/sensors/test_sql.py similarity index 99% rename from tests/providers/common/sql/sensors/test_sql.py rename to providers/tests/common/sql/sensors/test_sql.py index f4437a265a08c..33e8fe6399738 100644 --- a/tests/providers/common/sql/sensors/test_sql.py +++ b/providers/tests/common/sql/sensors/test_sql.py @@ -26,7 +26,8 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.providers.common.sql.sensors.sql import SqlSensor from airflow.utils.timezone import datetime -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.8.0+ only"), diff --git a/tests/providers/common/sql/test_utils.py b/providers/tests/common/sql/test_utils.py similarity index 96% rename from tests/providers/common/sql/test_utils.py rename to providers/tests/common/sql/test_utils.py index 7c76f3a7fa507..19b7bcc339411 100644 --- a/tests/providers/common/sql/test_utils.py +++ b/providers/tests/common/sql/test_utils.py @@ -23,7 +23,8 @@ import pytest from airflow.models import Connection -from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), diff --git a/tests/providers/conftest.py b/providers/tests/conftest.py similarity index 81% rename from tests/providers/conftest.py rename to providers/tests/conftest.py index 7dd0079ae6c7a..6ca7990e6054f 100644 --- a/tests/providers/conftest.py +++ b/providers/tests/conftest.py @@ -17,11 +17,20 @@ from __future__ import annotations +import pathlib from unittest import mock import pytest -from airflow.models import Connection +pytest_plugins = "dev.tests_common.pytest_plugin" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + dep_path = [pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml")] + config.inicfg["airflow_deprecations_ignore"] = ( + config.inicfg.get("airflow_deprecations_ignore", []) + dep_path # type: ignore[assignment,operator] + ) @pytest.fixture @@ -34,6 +43,8 @@ def hook_conn(request): If param is exception than add side effect. Otherwise, it raises an error """ + from airflow.models import Connection + try: conn = request.param except AttributeError: diff --git a/tests/providers/cncf/kubernetes/__init__.py b/providers/tests/databricks/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/__init__.py rename to providers/tests/databricks/__init__.py diff --git a/tests/providers/cncf/kubernetes/models/__init__.py b/providers/tests/databricks/hooks/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/models/__init__.py rename to providers/tests/databricks/hooks/__init__.py diff --git a/tests/providers/databricks/hooks/test_databricks.py b/providers/tests/databricks/hooks/test_databricks.py similarity index 100% rename from tests/providers/databricks/hooks/test_databricks.py rename to providers/tests/databricks/hooks/test_databricks.py diff --git a/tests/providers/databricks/hooks/test_databricks_base.py b/providers/tests/databricks/hooks/test_databricks_base.py similarity index 100% rename from tests/providers/databricks/hooks/test_databricks_base.py rename to providers/tests/databricks/hooks/test_databricks_base.py diff --git a/tests/providers/databricks/hooks/test_databricks_sql.py b/providers/tests/databricks/hooks/test_databricks_sql.py similarity index 90% rename from tests/providers/databricks/hooks/test_databricks_sql.py rename to providers/tests/databricks/hooks/test_databricks_sql.py index 0fdfba4776d07..824be94421d72 100644 --- a/tests/providers/databricks/hooks/test_databricks_sql.py +++ b/providers/tests/databricks/hooks/test_databricks_sql.py @@ -308,6 +308,7 @@ def test_query( mock_get_conn, mock_get_requests, ): +<<<<<<< HEAD:tests/providers/databricks/hooks/test_databricks_sql.py connections = [] cursors = [] for index in range(len(cursor_descriptions)): @@ -321,6 +322,40 @@ def test_query( cursors.append(cur) connections.append(conn) mock_get_conn.side_effect = connections +======= + with ( + patch("airflow.providers.databricks.hooks.databricks_sql.DatabricksSqlHook.get_conn") as mock_conn, + patch("airflow.providers.databricks.hooks.databricks_base.requests") as mock_requests, + ): + mock_requests.codes.ok = 200 + mock_requests.get.return_value.json.return_value = { + "endpoints": [ + { + "id": "1264e5078741679a", + "name": "Test", + "odbc_params": { + "hostname": "xx.cloud.databricks.com", + "path": "/sql/1.0/endpoints/1264e5078741679a", + }, + } + ] + } + status_code_mock = mock.PropertyMock(return_value=200) + type(mock_requests.get.return_value).status_code = status_code_mock + connections = [] + cursors = [] + for index in range(len(cursor_descriptions)): + conn = mock.MagicMock() + cur = mock.MagicMock( + rowcount=len(cursor_results[index]), + description=get_cursor_descriptions(cursor_descriptions[index]), + ) + cur.fetchall.return_value = cursor_results[index] + conn.cursor.return_value = cur + cursors.append(cur) + connections.append(conn) + mock_conn.side_effect = connections +>>>>>>> 857ca4c06c (Split providers out of the main "airflow/" tree into a UV workspace project (#42505)):providers/tests/databricks/hooks/test_databricks_sql.py if not return_tuple: with pytest.warns( diff --git a/tests/providers/databricks/__init__.py b/providers/tests/databricks/operators/__init__.py similarity index 100% rename from tests/providers/databricks/__init__.py rename to providers/tests/databricks/operators/__init__.py diff --git a/tests/providers/databricks/operators/test_databricks.py b/providers/tests/databricks/operators/test_databricks.py similarity index 100% rename from tests/providers/databricks/operators/test_databricks.py rename to providers/tests/databricks/operators/test_databricks.py diff --git a/tests/providers/databricks/operators/test_databricks_copy.py b/providers/tests/databricks/operators/test_databricks_copy.py similarity index 100% rename from tests/providers/databricks/operators/test_databricks_copy.py rename to providers/tests/databricks/operators/test_databricks_copy.py diff --git a/tests/providers/databricks/operators/test_databricks_repos.py b/providers/tests/databricks/operators/test_databricks_repos.py similarity index 100% rename from tests/providers/databricks/operators/test_databricks_repos.py rename to providers/tests/databricks/operators/test_databricks_repos.py diff --git a/tests/providers/databricks/operators/test_databricks_sql.py b/providers/tests/databricks/operators/test_databricks_sql.py similarity index 100% rename from tests/providers/databricks/operators/test_databricks_sql.py rename to providers/tests/databricks/operators/test_databricks_sql.py diff --git a/tests/providers/databricks/operators/test_databricks_workflow.py b/providers/tests/databricks/operators/test_databricks_workflow.py similarity index 100% rename from tests/providers/databricks/operators/test_databricks_workflow.py rename to providers/tests/databricks/operators/test_databricks_workflow.py diff --git a/tests/providers/atlassian/jira/notifications/__init__.py b/providers/tests/databricks/plugins/__init__.py similarity index 100% rename from tests/providers/atlassian/jira/notifications/__init__.py rename to providers/tests/databricks/plugins/__init__.py diff --git a/tests/providers/databricks/plugins/test_databricks_workflow.py b/providers/tests/databricks/plugins/test_databricks_workflow.py similarity index 97% rename from tests/providers/databricks/plugins/test_databricks_workflow.py rename to providers/tests/databricks/plugins/test_databricks_workflow.py index c140ac4450b8f..695466c62d266 100644 --- a/tests/providers/databricks/plugins/test_databricks_workflow.py +++ b/providers/tests/databricks/plugins/test_databricks_workflow.py @@ -41,6 +41,8 @@ ) from airflow.www.app import create_app +from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES + DAG_ID = "test_dag" TASK_ID = "test_task" RUN_ID = "test_run_1" @@ -190,6 +192,9 @@ def test_workflow_job_run_link(app): assert "https://mockhost/#job/1/run/1" in result +@pytest.mark.skipif( + RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES, reason="Web plugin test doesn't work when not against sources" +) @pytest.mark.db_test def test_workflow_job_repair_single_failed_link(app): with app.app_context(): diff --git a/tests/providers/atlassian/jira/operators/__init__.py b/providers/tests/databricks/sensors/__init__.py similarity index 100% rename from tests/providers/atlassian/jira/operators/__init__.py rename to providers/tests/databricks/sensors/__init__.py diff --git a/tests/providers/databricks/sensors/test_databricks_partition.py b/providers/tests/databricks/sensors/test_databricks_partition.py similarity index 100% rename from tests/providers/databricks/sensors/test_databricks_partition.py rename to providers/tests/databricks/sensors/test_databricks_partition.py diff --git a/tests/providers/databricks/sensors/test_databricks_sql.py b/providers/tests/databricks/sensors/test_databricks_sql.py similarity index 100% rename from tests/providers/databricks/sensors/test_databricks_sql.py rename to providers/tests/databricks/sensors/test_databricks_sql.py diff --git a/tests/providers/databricks/test_exceptions.py b/providers/tests/databricks/test_exceptions.py similarity index 100% rename from tests/providers/databricks/test_exceptions.py rename to providers/tests/databricks/test_exceptions.py diff --git a/tests/providers/databricks/hooks/__init__.py b/providers/tests/databricks/triggers/__init__.py similarity index 100% rename from tests/providers/databricks/hooks/__init__.py rename to providers/tests/databricks/triggers/__init__.py diff --git a/tests/providers/databricks/triggers/test_databricks.py b/providers/tests/databricks/triggers/test_databricks.py similarity index 100% rename from tests/providers/databricks/triggers/test_databricks.py rename to providers/tests/databricks/triggers/test_databricks.py diff --git a/tests/providers/atlassian/jira/sensors/__init__.py b/providers/tests/databricks/utils/__init__.py similarity index 100% rename from tests/providers/atlassian/jira/sensors/__init__.py rename to providers/tests/databricks/utils/__init__.py diff --git a/tests/providers/databricks/utils/test_databricks.py b/providers/tests/databricks/utils/test_databricks.py similarity index 100% rename from tests/providers/databricks/utils/test_databricks.py rename to providers/tests/databricks/utils/test_databricks.py diff --git a/tests/providers/databricks/operators/__init__.py b/providers/tests/datadog/__init__.py similarity index 100% rename from tests/providers/databricks/operators/__init__.py rename to providers/tests/datadog/__init__.py diff --git a/tests/providers/databricks/triggers/__init__.py b/providers/tests/datadog/hooks/__init__.py similarity index 100% rename from tests/providers/databricks/triggers/__init__.py rename to providers/tests/datadog/hooks/__init__.py diff --git a/tests/providers/datadog/hooks/test_datadog.py b/providers/tests/datadog/hooks/test_datadog.py similarity index 100% rename from tests/providers/datadog/hooks/test_datadog.py rename to providers/tests/datadog/hooks/test_datadog.py diff --git a/tests/providers/datadog/__init__.py b/providers/tests/datadog/sensors/__init__.py similarity index 100% rename from tests/providers/datadog/__init__.py rename to providers/tests/datadog/sensors/__init__.py diff --git a/tests/providers/datadog/sensors/test_datadog.py b/providers/tests/datadog/sensors/test_datadog.py similarity index 100% rename from tests/providers/datadog/sensors/test_datadog.py rename to providers/tests/datadog/sensors/test_datadog.py diff --git a/tests/providers/celery/cli/__init__.py b/providers/tests/dbt/__init__.py similarity index 100% rename from tests/providers/celery/cli/__init__.py rename to providers/tests/dbt/__init__.py diff --git a/tests/providers/celery/log_handlers/__init__.py b/providers/tests/dbt/cloud/__init__.py similarity index 100% rename from tests/providers/celery/log_handlers/__init__.py rename to providers/tests/dbt/cloud/__init__.py diff --git a/tests/providers/cncf/__init__.py b/providers/tests/dbt/cloud/hooks/__init__.py similarity index 100% rename from tests/providers/cncf/__init__.py rename to providers/tests/dbt/cloud/hooks/__init__.py diff --git a/tests/providers/dbt/cloud/hooks/test_dbt.py b/providers/tests/dbt/cloud/hooks/test_dbt.py similarity index 99% rename from tests/providers/dbt/cloud/hooks/test_dbt.py rename to providers/tests/dbt/cloud/hooks/test_dbt.py index 0d84189bc8049..590f1b677f10c 100644 --- a/tests/providers/dbt/cloud/hooks/test_dbt.py +++ b/providers/tests/dbt/cloud/hooks/test_dbt.py @@ -676,8 +676,9 @@ def fake_sleep(seconds): # Shift frozen time every time we call a ``time.sleep`` during this test case. time_machine.shift(timedelta(seconds=seconds)) - with patch.object(DbtCloudHook, "get_job_run_status") as mock_job_run_status, patch( - "airflow.providers.dbt.cloud.hooks.dbt.time.sleep", side_effect=fake_sleep + with ( + patch.object(DbtCloudHook, "get_job_run_status") as mock_job_run_status, + patch("airflow.providers.dbt.cloud.hooks.dbt.time.sleep", side_effect=fake_sleep), ): mock_job_run_status.return_value = job_run_status diff --git a/tests/providers/cncf/kubernetes/cli/__init__.py b/providers/tests/dbt/cloud/operators/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/cli/__init__.py rename to providers/tests/dbt/cloud/operators/__init__.py diff --git a/tests/providers/dbt/cloud/operators/test_dbt.py b/providers/tests/dbt/cloud/operators/test_dbt.py similarity index 99% rename from tests/providers/dbt/cloud/operators/test_dbt.py rename to providers/tests/dbt/cloud/operators/test_dbt.py index 658fe84a49d67..eb50bd5a22a2e 100644 --- a/tests/providers/dbt/cloud/operators/test_dbt.py +++ b/providers/tests/dbt/cloud/operators/test_dbt.py @@ -235,8 +235,9 @@ def fake_sleep(seconds): overall_delta = timedelta(seconds=seconds) + timedelta(microseconds=42) time_machine.shift(overall_delta) - with patch.object(DbtCloudHook, "get_job_run") as mock_get_job_run, patch( - "airflow.providers.dbt.cloud.hooks.dbt.time.sleep", side_effect=fake_sleep + with ( + patch.object(DbtCloudHook, "get_job_run") as mock_get_job_run, + patch("airflow.providers.dbt.cloud.hooks.dbt.time.sleep", side_effect=fake_sleep), ): mock_get_job_run.return_value.json.return_value = { "data": {"status": job_run_status, "id": RUN_ID} diff --git a/tests/providers/cncf/kubernetes/data_files/__init__.py b/providers/tests/dbt/cloud/sensors/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/__init__.py rename to providers/tests/dbt/cloud/sensors/__init__.py diff --git a/tests/providers/dbt/cloud/sensors/test_dbt.py b/providers/tests/dbt/cloud/sensors/test_dbt.py similarity index 100% rename from tests/providers/dbt/cloud/sensors/test_dbt.py rename to providers/tests/dbt/cloud/sensors/test_dbt.py diff --git a/tests/providers/cncf/kubernetes/data_files/executor/__init__.py b/providers/tests/dbt/cloud/test_data/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/executor/__init__.py rename to providers/tests/dbt/cloud/test_data/__init__.py diff --git a/tests/providers/dbt/cloud/test_data/catalog.json b/providers/tests/dbt/cloud/test_data/catalog.json similarity index 100% rename from tests/providers/dbt/cloud/test_data/catalog.json rename to providers/tests/dbt/cloud/test_data/catalog.json diff --git a/tests/providers/dbt/cloud/test_data/job_run.json b/providers/tests/dbt/cloud/test_data/job_run.json similarity index 100% rename from tests/providers/dbt/cloud/test_data/job_run.json rename to providers/tests/dbt/cloud/test_data/job_run.json diff --git a/tests/providers/dbt/cloud/test_data/manifest.json b/providers/tests/dbt/cloud/test_data/manifest.json similarity index 100% rename from tests/providers/dbt/cloud/test_data/manifest.json rename to providers/tests/dbt/cloud/test_data/manifest.json diff --git a/tests/providers/dbt/cloud/test_data/run_results.json b/providers/tests/dbt/cloud/test_data/run_results.json similarity index 100% rename from tests/providers/dbt/cloud/test_data/run_results.json rename to providers/tests/dbt/cloud/test_data/run_results.json diff --git a/tests/providers/cncf/kubernetes/data_files/pods/__init__.py b/providers/tests/dbt/cloud/triggers/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/pods/__init__.py rename to providers/tests/dbt/cloud/triggers/__init__.py diff --git a/tests/providers/dbt/cloud/triggers/test_dbt.py b/providers/tests/dbt/cloud/triggers/test_dbt.py similarity index 100% rename from tests/providers/dbt/cloud/triggers/test_dbt.py rename to providers/tests/dbt/cloud/triggers/test_dbt.py diff --git a/tests/providers/cncf/kubernetes/data_files/spark/__init__.py b/providers/tests/dbt/cloud/utils/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/data_files/spark/__init__.py rename to providers/tests/dbt/cloud/utils/__init__.py diff --git a/tests/providers/dbt/cloud/utils/test_openlineage.py b/providers/tests/dbt/cloud/utils/test_openlineage.py similarity index 100% rename from tests/providers/dbt/cloud/utils/test_openlineage.py rename to providers/tests/dbt/cloud/utils/test_openlineage.py diff --git a/providers/tests/deprecations_ignore.yml b/providers/tests/deprecations_ignore.yml new file mode 100644 index 0000000000000..b5aff3c84ea5b --- /dev/null +++ b/providers/tests/deprecations_ignore.yml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- + +- providers/tests/amazon/aws/deferrable/hooks/test_base_aws.py::TestAwsBaseAsyncHook::test_get_client_async +- providers/tests/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_cluster_status +- providers/tests/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_get_cluster_status +- providers/tests/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_get_cluster_status_exception +- providers/tests/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_pause_cluster +- providers/tests/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_resume_cluster +- providers/tests/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_resume_cluster_exception +- providers/tests/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_exception +- providers/tests/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_resuming_status +- providers/tests/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_success +- providers/tests/google/common/auth_backend/test_google_openid.py::TestGoogleOpenID::test_success +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_api_resource_configs +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_api_resource_configs_duplication_warning +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_queries +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_query_cancel_completed +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_query_cancel_timeout +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_query_jobs_to_cancel +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_get_dataset_tables_list +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_invalid_schema_update_and_write_disposition +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_invalid_schema_update_options +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_invalid_source_format +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_extract +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_load_with_non_csv_as_src_fmt +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_schema_update_options +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_schema_update_options_incorrect +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect_default +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect_legacy_with_query_params +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect_legacy_with_query_params_fails +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_with_arg +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_without_sql_fails +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_table_delete +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryHookRunWithConfiguration::test_run_with_configuration_location +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_create_external_table_with_kms +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_run_copy_with_kms +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_run_load_with_kms +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_run_query_with_kms +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_create_external_table_description +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_create_external_table_labels +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_run_load_description +- providers/tests/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_run_load_labels +- providers/tests/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_load_default +- providers/tests/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_load_with_arg +- providers/tests/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_query_default +- providers/tests/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_query_with_arg +- providers/tests/google/cloud/hooks/test_bigquery.py::TestDatasetsOperations::test_patch_dataset +- providers/tests/google/cloud/hooks/test_bigquery.py::TestTableOperations::test_patch_table +- providers/tests/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_load_default +- providers/tests/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_load_with_arg +- providers/tests/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_query_with_arg +- providers/tests/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_with_auto_detect +- providers/tests/google/cloud/hooks/test_gcs.py::TestGCSHook::test_list__error_match_glob_and_invalid_delimiter +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_error_operation +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_life_science_client_creation +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_run_pipeline_immediately_complete +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_waiting_operation +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_delegate_to_runtime_error +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_error_operation +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_life_science_client_creation +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_location_path +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_run_pipeline_immediately_complete +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_waiting_operation +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithoutProjectId::test_life_science_client_creation +- providers/tests/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithoutProjectId::test_run_pipeline +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_cancel_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_create_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_delete_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_get_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_list_pipeline_jobs +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_cancel_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_create_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_delete_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_get_pipeline_job +- providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_list_pipeline_jobs +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcHadoopOperator::test_execute +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcHiveOperator::test_builder +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcHiveOperator::test_execute +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcPigOperator::test_builder +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcPigOperator::test_execute +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcPySparkOperator::test_execute +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcSparkOperator::test_execute +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcSparkSqlOperator::test_builder +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcSparkSqlOperator::test_execute +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataProcSparkSqlOperator::test_execute_override_project_id +- providers/tests/google/cloud/operators/test_dataproc.py::TestDataprocClusterScaleOperator::test_execute +- providers/tests/google/cloud/operators/test_dataproc.py::test_create_cluster_operator_extra_links +- providers/tests/google/cloud/operators/test_dataproc.py::test_scale_cluster_operator_extra_links +- providers/tests/google/cloud/operators/test_dataproc.py::test_submit_spark_job_operator_extra_links +- providers/tests/google/cloud/operators/test_gcs.py::TestGoogleCloudStorageListOperator::test_execute__delimiter +- providers/tests/google/cloud/operators/test_kubernetes_engine.py::TestGoogleCloudPlatformContainerOperator::test_create_execute_error_body +- providers/tests/google/cloud/operators/test_life_sciences.py::TestLifeSciencesRunPipelineOperator::test_executes +- providers/tests/google/cloud/operators/test_life_sciences.py::TestLifeSciencesRunPipelineOperator::test_executes_without_project_id +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_copy_files_into_a_folder +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_last_modified_time +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_more_than_1_wildcard +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_no_prefix +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_no_suffix +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_prefix_and_suffix +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_empty_destination_object +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_destination_object +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_destination_object_retained_prefix +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_replace_flag_false +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_replace_flag_false_with_destination_object +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_without_destination_object +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_executes_with_a_delimiter +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_executes_with_delimiter_and_destination_object +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_executes_with_different_delimiter_and_destination_object +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_wc_with_last_modified_time_with_all_true_cond +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_wc_with_last_modified_time_with_one_true_cond +- providers/tests/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_wc_with_no_last_modified_time diff --git a/tests/providers/datadog/hooks/__init__.py b/providers/tests/dingding/__init__.py similarity index 100% rename from tests/providers/datadog/hooks/__init__.py rename to providers/tests/dingding/__init__.py diff --git a/tests/providers/datadog/sensors/__init__.py b/providers/tests/dingding/hooks/__init__.py similarity index 100% rename from tests/providers/datadog/sensors/__init__.py rename to providers/tests/dingding/hooks/__init__.py diff --git a/tests/providers/dingding/hooks/test_dingding.py b/providers/tests/dingding/hooks/test_dingding.py similarity index 100% rename from tests/providers/dingding/hooks/test_dingding.py rename to providers/tests/dingding/hooks/test_dingding.py diff --git a/tests/providers/dingding/__init__.py b/providers/tests/dingding/operators/__init__.py similarity index 100% rename from tests/providers/dingding/__init__.py rename to providers/tests/dingding/operators/__init__.py diff --git a/tests/providers/dingding/operators/test_dingding.py b/providers/tests/dingding/operators/test_dingding.py similarity index 100% rename from tests/providers/dingding/operators/test_dingding.py rename to providers/tests/dingding/operators/test_dingding.py diff --git a/tests/providers/dingding/hooks/__init__.py b/providers/tests/discord/__init__.py similarity index 100% rename from tests/providers/dingding/hooks/__init__.py rename to providers/tests/discord/__init__.py diff --git a/tests/providers/dingding/operators/__init__.py b/providers/tests/discord/hooks/__init__.py similarity index 100% rename from tests/providers/dingding/operators/__init__.py rename to providers/tests/discord/hooks/__init__.py diff --git a/tests/providers/discord/hooks/test_discord_webhook.py b/providers/tests/discord/hooks/test_discord_webhook.py similarity index 100% rename from tests/providers/discord/hooks/test_discord_webhook.py rename to providers/tests/discord/hooks/test_discord_webhook.py diff --git a/tests/providers/cncf/kubernetes/decorators/__init__.py b/providers/tests/discord/notifications/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/decorators/__init__.py rename to providers/tests/discord/notifications/__init__.py diff --git a/tests/providers/discord/notifications/test_discord.py b/providers/tests/discord/notifications/test_discord.py similarity index 100% rename from tests/providers/discord/notifications/test_discord.py rename to providers/tests/discord/notifications/test_discord.py diff --git a/tests/providers/discord/__init__.py b/providers/tests/discord/operators/__init__.py similarity index 100% rename from tests/providers/discord/__init__.py rename to providers/tests/discord/operators/__init__.py diff --git a/tests/providers/discord/operators/test_discord_webhook.py b/providers/tests/discord/operators/test_discord_webhook.py similarity index 100% rename from tests/providers/discord/operators/test_discord_webhook.py rename to providers/tests/discord/operators/test_discord_webhook.py diff --git a/tests/providers/discord/hooks/__init__.py b/providers/tests/docker/__init__.py similarity index 100% rename from tests/providers/discord/hooks/__init__.py rename to providers/tests/docker/__init__.py diff --git a/tests/providers/docker/conftest.py b/providers/tests/docker/conftest.py similarity index 100% rename from tests/providers/docker/conftest.py rename to providers/tests/docker/conftest.py diff --git a/tests/providers/cncf/kubernetes/executors/__init__.py b/providers/tests/docker/decorators/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/executors/__init__.py rename to providers/tests/docker/decorators/__init__.py diff --git a/tests/providers/docker/decorators/test_docker.py b/providers/tests/docker/decorators/test_docker.py similarity index 100% rename from tests/providers/docker/decorators/test_docker.py rename to providers/tests/docker/decorators/test_docker.py diff --git a/tests/providers/discord/operators/__init__.py b/providers/tests/docker/hooks/__init__.py similarity index 100% rename from tests/providers/discord/operators/__init__.py rename to providers/tests/docker/hooks/__init__.py diff --git a/tests/providers/docker/hooks/test_docker.py b/providers/tests/docker/hooks/test_docker.py similarity index 100% rename from tests/providers/docker/hooks/test_docker.py rename to providers/tests/docker/hooks/test_docker.py diff --git a/tests/providers/docker/__init__.py b/providers/tests/docker/operators/__init__.py similarity index 100% rename from tests/providers/docker/__init__.py rename to providers/tests/docker/operators/__init__.py diff --git a/tests/providers/docker/operators/test_docker.py b/providers/tests/docker/operators/test_docker.py similarity index 99% rename from tests/providers/docker/operators/test_docker.py rename to providers/tests/docker/operators/test_docker.py index 8a461f7c084a7..9dbc84657f4f8 100644 --- a/tests/providers/docker/operators/test_docker.py +++ b/providers/tests/docker/operators/test_docker.py @@ -860,7 +860,8 @@ def test_partial_deprecated_skip_exit_code_ambiguous( with set_current_task_instance_session(session=session): warning_match = r"`skip_exit_code` is deprecated and will be removed" for ti in tis: - with pytest.warns(AirflowProviderDeprecationWarning, match=warning_match), pytest.raises( - ValueError, match="Conflicting `skip_on_exit_code` provided" + with ( + pytest.warns(AirflowProviderDeprecationWarning, match=warning_match), + pytest.raises(ValueError, match="Conflicting `skip_on_exit_code` provided"), ): ti.render_templates() diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/providers/tests/docker/operators/test_docker_swarm.py similarity index 100% rename from tests/providers/docker/operators/test_docker_swarm.py rename to providers/tests/docker/operators/test_docker_swarm.py diff --git a/tests/providers/docker/test_exceptions.py b/providers/tests/docker/test_exceptions.py similarity index 100% rename from tests/providers/docker/test_exceptions.py rename to providers/tests/docker/test_exceptions.py diff --git a/tests/providers/docker/hooks/__init__.py b/providers/tests/edge/__init__.py similarity index 100% rename from tests/providers/docker/hooks/__init__.py rename to providers/tests/edge/__init__.py diff --git a/tests/providers/docker/operators/__init__.py b/providers/tests/edge/api_endpoints/__init__.py similarity index 100% rename from tests/providers/docker/operators/__init__.py rename to providers/tests/edge/api_endpoints/__init__.py diff --git a/tests/providers/edge/api_endpoints/test_health_endpoint.py b/providers/tests/edge/api_endpoints/test_health_endpoint.py similarity index 100% rename from tests/providers/edge/api_endpoints/test_health_endpoint.py rename to providers/tests/edge/api_endpoints/test_health_endpoint.py diff --git a/tests/providers/edge/api_endpoints/test_rpc_api_endpoint.py b/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py similarity index 98% rename from tests/providers/edge/api_endpoints/test_rpc_api_endpoint.py rename to providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py index becf2f9397e31..9a9b4a7dd885f 100644 --- a/tests/providers/edge/api_endpoints/test_rpc_api_endpoint.py +++ b/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py @@ -40,8 +40,9 @@ from airflow.utils.jwt_signer import JWTSigner from airflow.utils.state import State from airflow.www import app -from tests.test_utils.decorators import dont_initialize_flask_app_submodules -from tests.test_utils.mock_plugins import mock_plugin_manager + +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager # Note: Sounds a bit strange to disable internal API tests in isolation mode but... # As long as the test is modelled to run its own internal API endpoints, it is conflicting diff --git a/tests/providers/edge/__init__.py b/providers/tests/edge/cli/__init__.py similarity index 100% rename from tests/providers/edge/__init__.py rename to providers/tests/edge/cli/__init__.py diff --git a/tests/providers/edge/cli/test_edge_command.py b/providers/tests/edge/cli/test_edge_command.py similarity index 99% rename from tests/providers/edge/cli/test_edge_command.py rename to providers/tests/edge/cli/test_edge_command.py index 398c221db02f9..af3e2c00e2998 100644 --- a/tests/providers/edge/cli/test_edge_command.py +++ b/providers/tests/edge/cli/test_edge_command.py @@ -33,7 +33,8 @@ from airflow.providers.edge.models.edge_job import EdgeJob from airflow.providers.edge.models.edge_worker import EdgeWorker, EdgeWorkerState from airflow.utils.state import TaskInstanceState -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytest.importorskip("pydantic", minversion="2.0.0") diff --git a/tests/providers/edge/api_endpoints/__init__.py b/providers/tests/edge/models/__init__.py similarity index 100% rename from tests/providers/edge/api_endpoints/__init__.py rename to providers/tests/edge/models/__init__.py diff --git a/tests/providers/edge/models/test_edge_job.py b/providers/tests/edge/models/test_edge_job.py similarity index 100% rename from tests/providers/edge/models/test_edge_job.py rename to providers/tests/edge/models/test_edge_job.py diff --git a/tests/providers/edge/models/test_edge_logs.py b/providers/tests/edge/models/test_edge_logs.py similarity index 100% rename from tests/providers/edge/models/test_edge_logs.py rename to providers/tests/edge/models/test_edge_logs.py diff --git a/tests/providers/edge/models/test_edge_worker.py b/providers/tests/edge/models/test_edge_worker.py similarity index 100% rename from tests/providers/edge/models/test_edge_worker.py rename to providers/tests/edge/models/test_edge_worker.py diff --git a/tests/providers/edge/cli/__init__.py b/providers/tests/edge/plugins/__init__.py similarity index 100% rename from tests/providers/edge/cli/__init__.py rename to providers/tests/edge/plugins/__init__.py diff --git a/tests/providers/edge/plugins/test_edge_executor_plugin.py b/providers/tests/edge/plugins/test_edge_executor_plugin.py similarity index 97% rename from tests/providers/edge/plugins/test_edge_executor_plugin.py rename to providers/tests/edge/plugins/test_edge_executor_plugin.py index e3422b17da3c8..d0c5a40770b31 100644 --- a/tests/providers/edge/plugins/test_edge_executor_plugin.py +++ b/providers/tests/edge/plugins/test_edge_executor_plugin.py @@ -22,7 +22,8 @@ from airflow.plugins_manager import AirflowPlugin from airflow.providers.edge.plugins import edge_executor_plugin -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars def test_plugin_inactive(): diff --git a/tests/providers/edge/models/__init__.py b/providers/tests/elasticsearch/__init__.py similarity index 100% rename from tests/providers/edge/models/__init__.py rename to providers/tests/elasticsearch/__init__.py diff --git a/tests/providers/edge/plugins/__init__.py b/providers/tests/elasticsearch/hooks/__init__.py similarity index 100% rename from tests/providers/edge/plugins/__init__.py rename to providers/tests/elasticsearch/hooks/__init__.py diff --git a/tests/providers/elasticsearch/hooks/test_elasticsearch.py b/providers/tests/elasticsearch/hooks/test_elasticsearch.py similarity index 100% rename from tests/providers/elasticsearch/hooks/test_elasticsearch.py rename to providers/tests/elasticsearch/hooks/test_elasticsearch.py diff --git a/tests/providers/cncf/kubernetes/hooks/__init__.py b/providers/tests/elasticsearch/log/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/hooks/__init__.py rename to providers/tests/elasticsearch/log/__init__.py diff --git a/tests/providers/elasticsearch/log/elasticmock/__init__.py b/providers/tests/elasticsearch/log/elasticmock/__init__.py similarity index 98% rename from tests/providers/elasticsearch/log/elasticmock/__init__.py rename to providers/tests/elasticsearch/log/elasticmock/__init__.py index 00d92471ba9f4..44e242d114574 100644 --- a/tests/providers/elasticsearch/log/elasticmock/__init__.py +++ b/providers/tests/elasticsearch/log/elasticmock/__init__.py @@ -43,7 +43,7 @@ from unittest.mock import patch from urllib.parse import unquote, urlparse -from tests.providers.elasticsearch.log.elasticmock.fake_elasticsearch import FakeElasticsearch +from providers.tests.elasticsearch.log.elasticmock.fake_elasticsearch import FakeElasticsearch ELASTIC_INSTANCES: dict[str, FakeElasticsearch] = {} diff --git a/tests/providers/elasticsearch/log/elasticmock/fake_elasticsearch.py b/providers/tests/elasticsearch/log/elasticmock/fake_elasticsearch.py similarity index 99% rename from tests/providers/elasticsearch/log/elasticmock/fake_elasticsearch.py rename to providers/tests/elasticsearch/log/elasticmock/fake_elasticsearch.py index 26e47cbf08f7a..1d975ee718c3f 100644 --- a/tests/providers/elasticsearch/log/elasticmock/fake_elasticsearch.py +++ b/providers/tests/elasticsearch/log/elasticmock/fake_elasticsearch.py @@ -22,7 +22,7 @@ from elasticsearch import Elasticsearch from elasticsearch.exceptions import NotFoundError -from tests.providers.elasticsearch.log.elasticmock.utilities import ( +from providers.tests.elasticsearch.log.elasticmock.utilities import ( MissingIndexException, get_random_id, query_params, diff --git a/tests/providers/elasticsearch/log/elasticmock/utilities/__init__.py b/providers/tests/elasticsearch/log/elasticmock/utilities/__init__.py similarity index 100% rename from tests/providers/elasticsearch/log/elasticmock/utilities/__init__.py rename to providers/tests/elasticsearch/log/elasticmock/utilities/__init__.py diff --git a/tests/providers/elasticsearch/log/test_es_json_formatter.py b/providers/tests/elasticsearch/log/test_es_json_formatter.py similarity index 100% rename from tests/providers/elasticsearch/log/test_es_json_formatter.py rename to providers/tests/elasticsearch/log/test_es_json_formatter.py diff --git a/tests/providers/elasticsearch/log/test_es_response.py b/providers/tests/elasticsearch/log/test_es_response.py similarity index 100% rename from tests/providers/elasticsearch/log/test_es_response.py rename to providers/tests/elasticsearch/log/test_es_response.py diff --git a/tests/providers/elasticsearch/log/test_es_task_handler.py b/providers/tests/elasticsearch/log/test_es_task_handler.py similarity index 99% rename from tests/providers/elasticsearch/log/test_es_task_handler.py rename to providers/tests/elasticsearch/log/test_es_task_handler.py index 9321f49d77878..abde5daf8bf18 100644 --- a/tests/providers/elasticsearch/log/test_es_task_handler.py +++ b/providers/tests/elasticsearch/log/test_es_task_handler.py @@ -43,10 +43,11 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import datetime -from tests.providers.elasticsearch.log.elasticmock import elasticmock -from tests.providers.elasticsearch.log.elasticmock.utilities import SearchFailedException -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from providers.tests.elasticsearch.log.elasticmock import elasticmock +from providers.tests.elasticsearch.log.elasticmock.utilities import SearchFailedException pytestmark = pytest.mark.db_test diff --git a/tests/providers/elasticsearch/__init__.py b/providers/tests/email/__init__.py similarity index 100% rename from tests/providers/elasticsearch/__init__.py rename to providers/tests/email/__init__.py diff --git a/tests/providers/elasticsearch/hooks/__init__.py b/providers/tests/email/operators/__init__.py similarity index 100% rename from tests/providers/elasticsearch/hooks/__init__.py rename to providers/tests/email/operators/__init__.py diff --git a/tests/providers/email/__init__.py b/providers/tests/exasol/__init__.py similarity index 100% rename from tests/providers/email/__init__.py rename to providers/tests/exasol/__init__.py diff --git a/tests/providers/email/operators/__init__.py b/providers/tests/exasol/hooks/__init__.py similarity index 100% rename from tests/providers/email/operators/__init__.py rename to providers/tests/exasol/hooks/__init__.py diff --git a/tests/providers/exasol/hooks/test_exasol.py b/providers/tests/exasol/hooks/test_exasol.py similarity index 100% rename from tests/providers/exasol/hooks/test_exasol.py rename to providers/tests/exasol/hooks/test_exasol.py diff --git a/tests/providers/exasol/hooks/test_sql.py b/providers/tests/exasol/hooks/test_sql.py similarity index 100% rename from tests/providers/exasol/hooks/test_sql.py rename to providers/tests/exasol/hooks/test_sql.py diff --git a/tests/providers/exasol/__init__.py b/providers/tests/exasol/operators/__init__.py similarity index 100% rename from tests/providers/exasol/__init__.py rename to providers/tests/exasol/operators/__init__.py diff --git a/tests/providers/exasol/operators/test_exasol.py b/providers/tests/exasol/operators/test_exasol.py similarity index 100% rename from tests/providers/exasol/operators/test_exasol.py rename to providers/tests/exasol/operators/test_exasol.py diff --git a/tests/providers/exasol/operators/test_exasol_sql.py b/providers/tests/exasol/operators/test_exasol_sql.py similarity index 100% rename from tests/providers/exasol/operators/test_exasol_sql.py rename to providers/tests/exasol/operators/test_exasol_sql.py diff --git a/tests/providers/exasol/hooks/__init__.py b/providers/tests/fab/__init__.py similarity index 100% rename from tests/providers/exasol/hooks/__init__.py rename to providers/tests/fab/__init__.py diff --git a/tests/providers/exasol/operators/__init__.py b/providers/tests/fab/auth_manager/__init__.py similarity index 100% rename from tests/providers/exasol/operators/__init__.py rename to providers/tests/fab/auth_manager/__init__.py diff --git a/tests/providers/fab/__init__.py b/providers/tests/fab/auth_manager/api/__init__.py similarity index 100% rename from tests/providers/fab/__init__.py rename to providers/tests/fab/auth_manager/api/__init__.py diff --git a/tests/providers/cncf/kubernetes/log_handlers/__init__.py b/providers/tests/fab/auth_manager/api/auth/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/log_handlers/__init__.py rename to providers/tests/fab/auth_manager/api/auth/__init__.py diff --git a/tests/providers/cncf/kubernetes/operators/__init__.py b/providers/tests/fab/auth_manager/api/auth/backend/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/operators/__init__.py rename to providers/tests/fab/auth_manager/api/auth/backend/__init__.py diff --git a/tests/providers/fab/auth_manager/api/auth/backend/test_basic_auth.py b/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py similarity index 98% rename from tests/providers/fab/auth_manager/api/auth/backend/test_basic_auth.py rename to providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py index 4a5104829a065..2c783e2046f88 100644 --- a/tests/providers/fab/auth_manager/api/auth/backend/test_basic_auth.py +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py @@ -24,7 +24,8 @@ from airflow.providers.fab.auth_manager.api.auth.backend.basic_auth import requires_authentication from airflow.www import app as application -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py b/providers/tests/fab/auth_manager/api/auth/backend/test_kerberos_auth.py similarity index 92% rename from tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py rename to providers/tests/fab/auth_manager/api/auth/backend/test_kerberos_auth.py index c763042e1c955..7d13a89e8db80 100644 --- a/tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_kerberos_auth.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth import init_app diff --git a/tests/providers/cncf/kubernetes/resource_convert/__init__.py b/providers/tests/fab/auth_manager/api_endpoints/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/resource_convert/__init__.py rename to providers/tests/fab/auth_manager/api_endpoints/__init__.py diff --git a/tests/providers/fab/auth_manager/api_endpoints/api_connexion_utils.py b/providers/tests/fab/auth_manager/api_endpoints/api_connexion_utils.py similarity index 97% rename from tests/providers/fab/auth_manager/api_endpoints/api_connexion_utils.py rename to providers/tests/fab/auth_manager/api_endpoints/api_connexion_utils.py index 61d923d5ff125..e4cbe93c9d62a 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/api_connexion_utils.py +++ b/providers/tests/fab/auth_manager/api_endpoints/api_connexion_utils.py @@ -18,7 +18,7 @@ from contextlib import contextmanager -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES diff --git a/tests/providers/fab/auth_manager/api_endpoints/remote_user_api_auth_backend.py b/providers/tests/fab/auth_manager/api_endpoints/remote_user_api_auth_backend.py similarity index 100% rename from tests/providers/fab/auth_manager/api_endpoints/remote_user_api_auth_backend.py rename to providers/tests/fab/auth_manager/api_endpoints/remote_user_api_auth_backend.py diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_asset_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py similarity index 97% rename from tests/providers/fab/auth_manager/api_endpoints/test_asset_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py index 8e749d1dd18f2..5c22e5ba7ff92 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_asset_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py @@ -22,7 +22,13 @@ import time_machine from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from airflow.security import permissions +from airflow.utils import timezone + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_assets, clear_db_runs +from dev.tests_common.test_utils.www import _check_last_log +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user try: from airflow.models.asset import AssetDagRunQueue, AssetModel @@ -31,11 +37,6 @@ raise else: pass -from airflow.security import permissions -from airflow.utils import timezone -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.db import clear_db_assets, clear_db_runs -from tests.test_utils.www import _check_last_log pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_auth.py b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py similarity index 95% rename from tests/providers/fab/auth_manager/api_endpoints/test_auth.py rename to providers/tests/fab/auth_manager/api_endpoints/test_auth.py index d3012e2f1b43e..630ce7050bed5 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_auth.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py @@ -21,11 +21,11 @@ import pytest from flask_login import current_user -from tests.test_utils.api_connexion_utils import assert_401 -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_pools -from tests.test_utils.www import client_with_login +from dev.tests_common.test_utils.api_connexion_utils import assert_401 +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_pools +from dev.tests_common.test_utils.www import client_with_login pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_backfill_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py similarity index 97% rename from tests/providers/fab/auth_manager/api_endpoints/test_backfill_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py index 56f135d457e9c..9d9a79af51135 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_backfill_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py @@ -25,7 +25,21 @@ import pytest from airflow.models import DagBag, DagModel -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from airflow.models.dag import DAG +from airflow.models.serialized_dag import SerializedDagModel +from airflow.operators.empty import EmptyOperator +from airflow.security import permissions +from airflow.utils import timezone +from airflow.utils.session import provide_session + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import ( + clear_db_backfills, + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, +) +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user try: from airflow.models.backfill import Backfill @@ -34,14 +48,6 @@ raise else: pass -from airflow.models.dag import DAG -from airflow.models.serialized_dag import SerializedDagModel -from airflow.operators.empty import EmptyOperator -from airflow.security import permissions -from airflow.utils import timezone -from airflow.utils.session import provide_session -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.db import clear_db_backfills, clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_cors.py b/providers/tests/fab/auth_manager/api_endpoints/test_cors.py similarity index 96% rename from tests/providers/fab/auth_manager/api_endpoints/test_cors.py rename to providers/tests/fab/auth_manager/api_endpoints/test_cors.py index b44eab8820ec6..8dbc4f964e61a 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_cors.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_cors.py @@ -20,9 +20,9 @@ import pytest -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_pools +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_pools pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_dag_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py similarity index 96% rename from tests/providers/fab/auth_manager/api_endpoints/test_dag_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py index b78ac58e442e0..e93425591f708 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_dag_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py @@ -28,10 +28,11 @@ from airflow.operators.empty import EmptyOperator from airflow.security import permissions from airflow.utils.session import provide_session -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from dev.tests_common.test_utils.www import _check_last_log +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py similarity index 97% rename from tests/providers/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py index a58ea08ff31cf..fa09df1be74e0 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py @@ -27,7 +27,14 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import DagRunState -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( + create_user, + delete_roles, + delete_user, +) try: from airflow.utils.types import DagRunTriggeredByType, DagRunType @@ -36,12 +43,6 @@ raise else: pass -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( - create_user, - delete_roles, - delete_user, -) -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py similarity index 95% rename from tests/providers/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py index f0d9b0da298c6..27e41136555ea 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py @@ -24,9 +24,10 @@ from airflow.models import DagBag from airflow.security import permissions -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py similarity index 92% rename from tests/providers/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py index adfde1cc5b3eb..01a2c68091ce1 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py @@ -22,9 +22,10 @@ from airflow.models.dagwarning import DagWarning from airflow.security import permissions from airflow.utils.session import create_session -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_dag_warnings, clear_db_dags + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_event_log_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py similarity index 96% rename from tests/providers/fab/auth_manager/api_endpoints/test_event_log_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py index acf3ca62684a1..4794893241f6f 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_event_log_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py @@ -21,9 +21,10 @@ from airflow.models import Log from airflow.security import permissions from airflow.utils import timezone -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_logs + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_logs +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_import_error_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py similarity index 96% rename from tests/providers/fab/auth_manager/api_endpoints/test_import_error_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py index a2fa1d028a3f2..110e8e6302553 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_import_error_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py @@ -21,10 +21,11 @@ from airflow.models.dag import DagModel from airflow.security import permissions from airflow.utils import timezone -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError -from tests.test_utils.db import clear_db_dags, clear_db_import_errors -from tests.test_utils.permissions import _resource_name + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_import_errors +from dev.tests_common.test_utils.permissions import _resource_name +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py similarity index 99% rename from tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py index 413a49a9d86a1..b72c6fe6612f3 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py @@ -19,22 +19,21 @@ import pytest from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( +from airflow.security import permissions + +from dev.tests_common.test_utils.api_connexion_utils import assert_401 +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_role, create_user, delete_role, delete_user, ) -from tests.test_utils.api_connexion_utils import assert_401 -from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import Role from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES - -from airflow.security import permissions - pytestmark = pytest.mark.db_test diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py similarity index 97% rename from tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py rename to providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py index 4a2f0068e5e4a..f5ec73a3e9893 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py @@ -24,7 +24,8 @@ role_schema, ) from airflow.security import permissions -from tests.test_utils.api_connexion_utils import create_role, delete_role + +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py similarity index 98% rename from tests/providers/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py index 69b3c221eae93..7d379b6f8c007 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py @@ -28,13 +28,14 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, delete_user, ) -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py similarity index 99% rename from tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py index 7f2c885bab52c..c7a20cb59c320 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py @@ -25,18 +25,19 @@ from airflow.security import permissions from airflow.utils import timezone from airflow.utils.session import create_session -from tests.test_utils.compat import ignore_provider_compatibility_error -with ignore_provider_compatibility_error("2.9.0+", __file__): - from airflow.providers.fab.auth_manager.models import User - -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( +from dev.tests_common.test_utils.api_connexion_utils import assert_401 +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.config import conf_vars +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_role, delete_user, ) -from tests.test_utils.api_connexion_utils import assert_401 -from tests.test_utils.config import conf_vars + +with ignore_provider_compatibility_error("2.9.0+", __file__): + from airflow.providers.fab.auth_manager.models import User + pytestmark = pytest.mark.db_test diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_user_schema.py b/providers/tests/fab/auth_manager/api_endpoints/test_user_schema.py similarity index 97% rename from tests/providers/fab/auth_manager/api_endpoints/test_user_schema.py rename to providers/tests/fab/auth_manager/api_endpoints/test_user_schema.py index f3399de6a9775..b37f27abe0397 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_user_schema.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_user_schema.py @@ -18,14 +18,15 @@ import pytest -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role -from tests.test_utils.compat import ignore_provider_compatibility_error +from airflow.utils import timezone + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.api_connexion.schemas.user_schema import user_collection_item_schema, user_schema from airflow.providers.fab.auth_manager.models import User -from airflow.utils import timezone TEST_EMAIL = "test@example.org" diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_variable_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py similarity index 93% rename from tests/providers/fab/auth_manager/api_endpoints/test_variable_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py index a8e71e1a82466..802eb4824087e 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_variable_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py @@ -20,9 +20,10 @@ from airflow.models import Variable from airflow.security import permissions -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_variables + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_variables +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_xcom_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py similarity index 97% rename from tests/providers/fab/auth_manager/api_endpoints/test_xcom_endpoint.py rename to providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py index 01336f9957c6d..06e8ee4847762 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_xcom_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py @@ -29,9 +29,10 @@ from airflow.utils.dates import parse_execution_date from airflow.utils.session import create_session from airflow.utils.types import DagRunType -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/cncf/kubernetes/sensors/__init__.py b/providers/tests/fab/auth_manager/cli_commands/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/sensors/__init__.py rename to providers/tests/fab/auth_manager/cli_commands/__init__.py diff --git a/tests/providers/fab/auth_manager/cli_commands/test_db_command.py b/providers/tests/fab/auth_manager/cli_commands/test_db_command.py similarity index 100% rename from tests/providers/fab/auth_manager/cli_commands/test_db_command.py rename to providers/tests/fab/auth_manager/cli_commands/test_db_command.py diff --git a/tests/providers/fab/auth_manager/cli_commands/test_definition.py b/providers/tests/fab/auth_manager/cli_commands/test_definition.py similarity index 94% rename from tests/providers/fab/auth_manager/cli_commands/test_definition.py rename to providers/tests/fab/auth_manager/cli_commands/test_definition.py index 2db5d352ecc19..de906bef1ba30 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_definition.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_definition.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands.definition import ( diff --git a/tests/providers/fab/auth_manager/cli_commands/test_role_command.py b/providers/tests/fab/auth_manager/cli_commands/test_role_command.py similarity index 98% rename from tests/providers/fab/auth_manager/cli_commands/test_role_command.py rename to providers/tests/fab/auth_manager/cli_commands/test_role_command.py index 5f12c01860d1f..fd176702c7219 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_role_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_role_command.py @@ -25,8 +25,9 @@ import pytest from airflow.cli import cli_parser -from tests.test_utils.compat import ignore_provider_compatibility_error -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.config import conf_vars with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import role_command diff --git a/tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py b/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py similarity index 96% rename from tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py rename to providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py index 9e1817bd5617c..e0e4a70f4ac88 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py @@ -22,7 +22,8 @@ import pytest from airflow.cli import cli_parser -from tests.test_utils.compat import ignore_provider_compatibility_error + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import sync_perm_command diff --git a/tests/providers/fab/auth_manager/cli_commands/test_user_command.py b/providers/tests/fab/auth_manager/cli_commands/test_user_command.py similarity index 99% rename from tests/providers/fab/auth_manager/cli_commands/test_user_command.py rename to providers/tests/fab/auth_manager/cli_commands/test_user_command.py index b8ce2f48d6c03..5f2f66b0866a2 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_user_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_user_command.py @@ -26,7 +26,8 @@ import pytest from airflow.cli import cli_parser -from tests.test_utils.compat import ignore_provider_compatibility_error + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import user_command diff --git a/tests/providers/fab/auth_manager/cli_commands/test_utils.py b/providers/tests/fab/auth_manager/cli_commands/test_utils.py similarity index 93% rename from tests/providers/fab/auth_manager/cli_commands/test_utils.py rename to providers/tests/fab/auth_manager/cli_commands/test_utils.py index fd8b1dfd50c89..f52defae6bb8f 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_utils.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_utils.py @@ -18,7 +18,7 @@ import pytest -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder diff --git a/tests/providers/fab/auth_manager/conftest.py b/providers/tests/fab/auth_manager/conftest.py similarity index 91% rename from tests/providers/fab/auth_manager/conftest.py rename to providers/tests/fab/auth_manager/conftest.py index a8fbe5fbdaaae..9102f5d0f65d4 100644 --- a/tests/providers/fab/auth_manager/conftest.py +++ b/providers/tests/fab/auth_manager/conftest.py @@ -19,8 +19,9 @@ import pytest from airflow.www import app -from tests.test_utils.config import conf_vars -from tests.test_utils.decorators import dont_initialize_flask_app_submodules + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules @pytest.fixture(scope="session") @@ -42,7 +43,7 @@ def factory(): ( "api", "auth_backends", - ): "tests.providers.fab.auth_manager.api_endpoints.remote_user_api_auth_backend,airflow.api.auth.backend.session", + ): "providers.tests.fab.auth_manager.api_endpoints.remote_user_api_auth_backend,airflow.api.auth.backend.session", ( "core", "auth_manager", diff --git a/tests/providers/cncf/kubernetes/triggers/__init__.py b/providers/tests/fab/auth_manager/decorators/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/triggers/__init__.py rename to providers/tests/fab/auth_manager/decorators/__init__.py diff --git a/tests/providers/fab/auth_manager/decorators/test_auth.py b/providers/tests/fab/auth_manager/decorators/test_auth.py similarity index 98% rename from tests/providers/fab/auth_manager/decorators/test_auth.py rename to providers/tests/fab/auth_manager/decorators/test_auth.py index 98f77a4f34271..202f0d6227073 100644 --- a/tests/providers/fab/auth_manager/decorators/test_auth.py +++ b/providers/tests/fab/auth_manager/decorators/test_auth.py @@ -21,7 +21,8 @@ import pytest from airflow.security.permissions import ACTION_CAN_READ, RESOURCE_DAG -from tests.test_utils.compat import ignore_provider_compatibility_error + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error permissions = [(ACTION_CAN_READ, RESOURCE_DAG)] diff --git a/tests/providers/fab/auth_manager/__init__.py b/providers/tests/fab/auth_manager/models/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/__init__.py rename to providers/tests/fab/auth_manager/models/__init__.py diff --git a/tests/providers/fab/auth_manager/models/test_anonymous_user.py b/providers/tests/fab/auth_manager/models/test_anonymous_user.py similarity index 93% rename from tests/providers/fab/auth_manager/models/test_anonymous_user.py rename to providers/tests/fab/auth_manager/models/test_anonymous_user.py index 4e365e3c8b705..419d17aa9f3f9 100644 --- a/tests/providers/fab/auth_manager/models/test_anonymous_user.py +++ b/providers/tests/fab/auth_manager/models/test_anonymous_user.py @@ -17,7 +17,7 @@ # under the License. from __future__ import annotations -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser diff --git a/tests/providers/fab/auth_manager/models/test_db.py b/providers/tests/fab/auth_manager/models/test_db.py similarity index 93% rename from tests/providers/fab/auth_manager/models/test_db.py rename to providers/tests/fab/auth_manager/models/test_db.py index 528e1cbf099ff..3af94ceed7b18 100644 --- a/tests/providers/fab/auth_manager/models/test_db.py +++ b/providers/tests/fab/auth_manager/models/test_db.py @@ -16,7 +16,6 @@ # under the License. from __future__ import annotations -import os import re from unittest import mock @@ -25,7 +24,7 @@ from alembic.migration import MigrationContext from sqlalchemy import MetaData -import airflow +import airflow.providers from airflow.settings import engine from airflow.utils.db import ( compare_server_default, @@ -38,20 +37,16 @@ class TestFABDBManager: def setup_method(self): - self.airflow_dir = os.path.dirname(airflow.__file__) + self.providers_dir: str = airflow.providers.__path__[0] def test_version_table_name_set(self, session): assert FABDBManager(session=session).version_table_name == "alembic_version_fab" def test_migration_dir_set(self, session): - assert ( - FABDBManager(session=session).migration_dir == f"{self.airflow_dir}/providers/fab/migrations" - ) + assert FABDBManager(session=session).migration_dir == f"{self.providers_dir}/fab/migrations" def test_alembic_file_set(self, session): - assert ( - FABDBManager(session=session).alembic_file == f"{self.airflow_dir}/providers/fab/alembic.ini" - ) + assert FABDBManager(session=session).alembic_file == f"{self.providers_dir}/fab/alembic.ini" def test_supports_table_dropping_set(self, session): assert FABDBManager(session=session).supports_table_dropping is True diff --git a/tests/providers/cncf/kubernetes/utils/__init__.py b/providers/tests/fab/auth_manager/security_manager/__init__.py similarity index 100% rename from tests/providers/cncf/kubernetes/utils/__init__.py rename to providers/tests/fab/auth_manager/security_manager/__init__.py diff --git a/tests/providers/fab/auth_manager/security_manager/test_constants.py b/providers/tests/fab/auth_manager/security_manager/test_constants.py similarity index 93% rename from tests/providers/fab/auth_manager/security_manager/test_constants.py rename to providers/tests/fab/auth_manager/security_manager/test_constants.py index 5a718eee4b639..a6566fd987109 100644 --- a/tests/providers/fab/auth_manager/security_manager/test_constants.py +++ b/providers/tests/fab/auth_manager/security_manager/test_constants.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.constants import EXISTING_ROLES diff --git a/tests/providers/fab/auth_manager/security_manager/test_override.py b/providers/tests/fab/auth_manager/security_manager/test_override.py similarity index 95% rename from tests/providers/fab/auth_manager/security_manager/test_override.py rename to providers/tests/fab/auth_manager/security_manager/test_override.py index 6d85c0319dc44..2733b688d1ae5 100644 --- a/tests/providers/fab/auth_manager/security_manager/test_override.py +++ b/providers/tests/fab/auth_manager/security_manager/test_override.py @@ -19,7 +19,7 @@ from unittest import mock from unittest.mock import Mock -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride diff --git a/tests/providers/fab/auth_manager/test_fab_auth_manager.py b/providers/tests/fab/auth_manager/test_fab_auth_manager.py similarity index 99% rename from tests/providers/fab/auth_manager/test_fab_auth_manager.py rename to providers/tests/fab/auth_manager/test_fab_auth_manager.py index d727b6090822f..064c8e0dfd4c7 100644 --- a/tests/providers/fab/auth_manager/test_fab_auth_manager.py +++ b/providers/tests/fab/auth_manager/test_fab_auth_manager.py @@ -31,7 +31,7 @@ except ImportError: pass -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager diff --git a/tests/providers/fab/auth_manager/test_models.py b/providers/tests/fab/auth_manager/test_models.py similarity index 96% rename from tests/providers/fab/auth_manager/test_models.py rename to providers/tests/fab/auth_manager/test_models.py index 30677d7095753..6f03be373187b 100644 --- a/tests/providers/fab/auth_manager/test_models.py +++ b/providers/tests/fab/auth_manager/test_models.py @@ -20,7 +20,7 @@ from sqlalchemy import Column, MetaData, String, Table -from tests.test_utils.compat import ignore_provider_compatibility_error +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import ( diff --git a/tests/providers/fab/auth_manager/test_security.py b/providers/tests/fab/auth_manager/test_security.py similarity index 98% rename from tests/providers/fab/auth_manager/test_security.py rename to providers/tests/fab/auth_manager/test_security.py index bebb52c256fc8..8c9e221b71df0 100644 --- a/tests/providers/fab/auth_manager/test_security.py +++ b/providers/tests/fab/auth_manager/test_security.py @@ -36,7 +36,8 @@ from airflow.exceptions import AirflowException from airflow.models import DagModel from airflow.models.dag import DAG -from tests.test_utils.compat import ignore_provider_compatibility_error + +from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager @@ -49,17 +50,18 @@ from airflow.www.auth import get_access_denied_message from airflow.www.extensions.init_auth_manager import get_auth_manager from airflow.www.utils import CustomSQLAInterface -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from dev.tests_common.test_utils.mock_security_manager import MockSecurityManager +from dev.tests_common.test_utils.permissions import _resource_name +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, create_user_scope, delete_role, delete_user, set_user_single_role, ) -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.db import clear_db_dags, clear_db_runs -from tests.test_utils.mock_security_manager import MockSecurityManager -from tests.test_utils.permissions import _resource_name if TYPE_CHECKING: from airflow.security.permissions import RESOURCE_ASSET diff --git a/tests/providers/fab/auth_manager/api/__init__.py b/providers/tests/fab/auth_manager/views/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/api/__init__.py rename to providers/tests/fab/auth_manager/views/__init__.py diff --git a/tests/providers/fab/auth_manager/views/test_permissions.py b/providers/tests/fab/auth_manager/views/test_permissions.py similarity index 93% rename from tests/providers/fab/auth_manager/views/test_permissions.py rename to providers/tests/fab/auth_manager/views/test_permissions.py index f24d9b738343b..2ac26ffe45e87 100644 --- a/tests/providers/fab/auth_manager/views/test_permissions.py +++ b/providers/tests/fab/auth_manager/views/test_permissions.py @@ -21,9 +21,10 @@ from airflow.security import permissions from airflow.www import app as application -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.www import client_with_login + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.www import client_with_login +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_roles_list.py b/providers/tests/fab/auth_manager/views/test_roles_list.py similarity index 91% rename from tests/providers/fab/auth_manager/views/test_roles_list.py rename to providers/tests/fab/auth_manager/views/test_roles_list.py index 8de63ad5ba88a..3c1509d75350a 100644 --- a/tests/providers/fab/auth_manager/views/test_roles_list.py +++ b/providers/tests/fab/auth_manager/views/test_roles_list.py @@ -21,9 +21,10 @@ from airflow.security import permissions from airflow.www import app as application -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.www import client_with_login + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.www import client_with_login +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_user.py b/providers/tests/fab/auth_manager/views/test_user.py similarity index 91% rename from tests/providers/fab/auth_manager/views/test_user.py rename to providers/tests/fab/auth_manager/views/test_user.py index 62b03a99e7c2c..ee068f7f319fe 100644 --- a/tests/providers/fab/auth_manager/views/test_user.py +++ b/providers/tests/fab/auth_manager/views/test_user.py @@ -21,9 +21,10 @@ from airflow.security import permissions from airflow.www import app as application -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.www import client_with_login + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.www import client_with_login +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_user_edit.py b/providers/tests/fab/auth_manager/views/test_user_edit.py similarity index 91% rename from tests/providers/fab/auth_manager/views/test_user_edit.py rename to providers/tests/fab/auth_manager/views/test_user_edit.py index 8099f67948183..7cdc1a493b007 100644 --- a/tests/providers/fab/auth_manager/views/test_user_edit.py +++ b/providers/tests/fab/auth_manager/views/test_user_edit.py @@ -21,9 +21,10 @@ from airflow.security import permissions from airflow.www import app as application -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.www import client_with_login + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.www import client_with_login +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_user_stats.py b/providers/tests/fab/auth_manager/views/test_user_stats.py similarity index 91% rename from tests/providers/fab/auth_manager/views/test_user_stats.py rename to providers/tests/fab/auth_manager/views/test_user_stats.py index ae09cf92252c6..e50bc87535a48 100644 --- a/tests/providers/fab/auth_manager/views/test_user_stats.py +++ b/providers/tests/fab/auth_manager/views/test_user_stats.py @@ -21,9 +21,10 @@ from airflow.security import permissions from airflow.www import app as application -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.www import client_with_login + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.www import client_with_login +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/cohere/__init__.py b/providers/tests/facebook/__init__.py similarity index 100% rename from tests/providers/cohere/__init__.py rename to providers/tests/facebook/__init__.py diff --git a/tests/providers/cohere/hooks/__init__.py b/providers/tests/facebook/ads/__init__.py similarity index 100% rename from tests/providers/cohere/hooks/__init__.py rename to providers/tests/facebook/ads/__init__.py diff --git a/tests/providers/cohere/operators/__init__.py b/providers/tests/facebook/ads/hooks/__init__.py similarity index 100% rename from tests/providers/cohere/operators/__init__.py rename to providers/tests/facebook/ads/hooks/__init__.py diff --git a/tests/providers/facebook/ads/hooks/test_ads.py b/providers/tests/facebook/ads/hooks/test_ads.py similarity index 100% rename from tests/providers/facebook/ads/hooks/test_ads.py rename to providers/tests/facebook/ads/hooks/test_ads.py diff --git a/tests/providers/fab/auth_manager/models/__init__.py b/providers/tests/ftp/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/models/__init__.py rename to providers/tests/ftp/__init__.py diff --git a/tests/providers/fab/auth_manager/views/__init__.py b/providers/tests/ftp/hooks/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/views/__init__.py rename to providers/tests/ftp/hooks/__init__.py diff --git a/tests/providers/ftp/hooks/test_ftp.py b/providers/tests/ftp/hooks/test_ftp.py similarity index 100% rename from tests/providers/ftp/hooks/test_ftp.py rename to providers/tests/ftp/hooks/test_ftp.py diff --git a/tests/providers/common/__init__.py b/providers/tests/ftp/operators/__init__.py similarity index 100% rename from tests/providers/common/__init__.py rename to providers/tests/ftp/operators/__init__.py diff --git a/tests/providers/ftp/operators/test_ftp.py b/providers/tests/ftp/operators/test_ftp.py similarity index 100% rename from tests/providers/ftp/operators/test_ftp.py rename to providers/tests/ftp/operators/test_ftp.py diff --git a/tests/providers/ftp/__init__.py b/providers/tests/ftp/sensors/__init__.py similarity index 100% rename from tests/providers/ftp/__init__.py rename to providers/tests/ftp/sensors/__init__.py diff --git a/tests/providers/ftp/sensors/test_ftp.py b/providers/tests/ftp/sensors/test_ftp.py similarity index 100% rename from tests/providers/ftp/sensors/test_ftp.py rename to providers/tests/ftp/sensors/test_ftp.py diff --git a/tests/providers/common/compat/__init__.py b/providers/tests/github/__init__.py similarity index 100% rename from tests/providers/common/compat/__init__.py rename to providers/tests/github/__init__.py diff --git a/tests/providers/common/compat/lineage/__init__.py b/providers/tests/github/hooks/__init__.py similarity index 100% rename from tests/providers/common/compat/lineage/__init__.py rename to providers/tests/github/hooks/__init__.py diff --git a/tests/providers/github/hooks/test_github.py b/providers/tests/github/hooks/test_github.py similarity index 100% rename from tests/providers/github/hooks/test_github.py rename to providers/tests/github/hooks/test_github.py diff --git a/tests/providers/common/compat/openlineage/__init__.py b/providers/tests/github/operators/__init__.py similarity index 100% rename from tests/providers/common/compat/openlineage/__init__.py rename to providers/tests/github/operators/__init__.py diff --git a/tests/providers/github/operators/test_github.py b/providers/tests/github/operators/test_github.py similarity index 100% rename from tests/providers/github/operators/test_github.py rename to providers/tests/github/operators/test_github.py diff --git a/tests/providers/common/compat/openlineage/utils/__init__.py b/providers/tests/github/sensors/__init__.py similarity index 100% rename from tests/providers/common/compat/openlineage/utils/__init__.py rename to providers/tests/github/sensors/__init__.py diff --git a/tests/providers/github/sensors/test_github.py b/providers/tests/github/sensors/test_github.py similarity index 100% rename from tests/providers/github/sensors/test_github.py rename to providers/tests/github/sensors/test_github.py diff --git a/tests/providers/common/compat/security/__init__.py b/providers/tests/google/__init__.py similarity index 100% rename from tests/providers/common/compat/security/__init__.py rename to providers/tests/google/__init__.py diff --git a/tests/providers/common/io/__init__.py b/providers/tests/google/ads/__init__.py similarity index 100% rename from tests/providers/common/io/__init__.py rename to providers/tests/google/ads/__init__.py diff --git a/tests/providers/common/io/assets/__init__.py b/providers/tests/google/ads/hooks/__init__.py similarity index 100% rename from tests/providers/common/io/assets/__init__.py rename to providers/tests/google/ads/hooks/__init__.py diff --git a/tests/providers/google/ads/hooks/test_ads.py b/providers/tests/google/ads/hooks/test_ads.py similarity index 100% rename from tests/providers/google/ads/hooks/test_ads.py rename to providers/tests/google/ads/hooks/test_ads.py diff --git a/tests/providers/common/io/operators/__init__.py b/providers/tests/google/ads/operators/__init__.py similarity index 100% rename from tests/providers/common/io/operators/__init__.py rename to providers/tests/google/ads/operators/__init__.py diff --git a/tests/providers/google/ads/operators/test_ads.py b/providers/tests/google/ads/operators/test_ads.py similarity index 100% rename from tests/providers/google/ads/operators/test_ads.py rename to providers/tests/google/ads/operators/test_ads.py diff --git a/tests/providers/common/io/xcom/__init__.py b/providers/tests/google/ads/transfers/__init__.py similarity index 100% rename from tests/providers/common/io/xcom/__init__.py rename to providers/tests/google/ads/transfers/__init__.py diff --git a/tests/providers/google/ads/transfers/test_ads_to_gcs.py b/providers/tests/google/ads/transfers/test_ads_to_gcs.py similarity index 97% rename from tests/providers/google/ads/transfers/test_ads_to_gcs.py rename to providers/tests/google/ads/transfers/test_ads_to_gcs.py index a97131203fa13..3d9c428494493 100644 --- a/tests/providers/google/ads/transfers/test_ads_to_gcs.py +++ b/providers/tests/google/ads/transfers/test_ads_to_gcs.py @@ -19,7 +19,8 @@ from unittest import mock from airflow.providers.google.ads.transfers.ads_to_gcs import GoogleAdsToGcsOperator -from tests.providers.google.ads.operators.test_ads import ( + +from providers.tests.google.ads.operators.test_ads import ( BUCKET, CLIENT_IDS, FIELDS_TO_EXTRACT, diff --git a/tests/providers/common/sql/__init__.py b/providers/tests/google/assets/__init__.py similarity index 100% rename from tests/providers/common/sql/__init__.py rename to providers/tests/google/assets/__init__.py diff --git a/tests/providers/google/datasets/test_bigquery.py b/providers/tests/google/assets/test_bigquery.py similarity index 100% rename from tests/providers/google/datasets/test_bigquery.py rename to providers/tests/google/assets/test_bigquery.py diff --git a/tests/providers/common/sql/hooks/__init__.py b/providers/tests/google/cloud/__init__.py similarity index 100% rename from tests/providers/common/sql/hooks/__init__.py rename to providers/tests/google/cloud/__init__.py diff --git a/tests/providers/common/sql/operators/__init__.py b/providers/tests/google/cloud/_internal_client/__init__.py similarity index 100% rename from tests/providers/common/sql/operators/__init__.py rename to providers/tests/google/cloud/_internal_client/__init__.py diff --git a/tests/providers/google/cloud/_internal_client/test_secret_manager_client.py b/providers/tests/google/cloud/_internal_client/test_secret_manager_client.py similarity index 100% rename from tests/providers/google/cloud/_internal_client/test_secret_manager_client.py rename to providers/tests/google/cloud/_internal_client/test_secret_manager_client.py diff --git a/tests/providers/common/sql/sensors/__init__.py b/providers/tests/google/cloud/hooks/__init__.py similarity index 100% rename from tests/providers/common/sql/sensors/__init__.py rename to providers/tests/google/cloud/hooks/__init__.py diff --git a/tests/providers/google/cloud/hooks/test_automl.py b/providers/tests/google/cloud/hooks/test_automl.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_automl.py rename to providers/tests/google/cloud/hooks/test_automl.py index 47f956c6d1032..26db131a9eef6 100644 --- a/tests/providers/google/cloud/hooks/test_automl.py +++ b/providers/tests/google/cloud/hooks/test_automl.py @@ -25,7 +25,8 @@ from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id CREDENTIALS = "test-creds" TASK_ID = "test-automl-hook" diff --git a/tests/providers/google/cloud/hooks/test_bigquery.py b/providers/tests/google/cloud/hooks/test_bigquery.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_bigquery.py rename to providers/tests/google/cloud/hooks/test_bigquery.py diff --git a/tests/providers/google/cloud/hooks/test_bigquery_dts.py b/providers/tests/google/cloud/hooks/test_bigquery_dts.py similarity index 98% rename from tests/providers/google/cloud/hooks/test_bigquery_dts.py rename to providers/tests/google/cloud/hooks/test_bigquery_dts.py index 1f973a9f21d85..163ca25273b18 100644 --- a/tests/providers/google/cloud/hooks/test_bigquery_dts.py +++ b/providers/tests/google/cloud/hooks/test_bigquery_dts.py @@ -30,7 +30,8 @@ AsyncBiqQueryDataTransferServiceHook, BiqQueryDataTransferServiceHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id CREDENTIALS = "test-creds" PROJECT_ID = "id" diff --git a/tests/providers/google/cloud/hooks/test_bigquery_system.py b/providers/tests/google/cloud/hooks/test_bigquery_system.py similarity index 94% rename from tests/providers/google/cloud/hooks/test_bigquery_system.py rename to providers/tests/google/cloud/hooks/test_bigquery_system.py index 3076f4a123e37..676ab35035839 100644 --- a/tests/providers/google/cloud/hooks/test_bigquery_system.py +++ b/providers/tests/google/cloud/hooks/test_bigquery_system.py @@ -22,8 +22,9 @@ import pytest from airflow.providers.google.cloud.hooks import bigquery as hook -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY -from tests.test_utils.gcp_system_helpers import GoogleSystemTest + +from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY @pytest.mark.system("google.cloud") diff --git a/tests/providers/google/cloud/hooks/test_bigtable.py b/providers/tests/google/cloud/hooks/test_bigtable.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_bigtable.py rename to providers/tests/google/cloud/hooks/test_bigtable.py index 4dda2fb009e76..f1929ba00d580 100644 --- a/tests/providers/google/cloud/hooks/test_bigtable.py +++ b/providers/tests/google/cloud/hooks/test_bigtable.py @@ -27,7 +27,8 @@ from airflow.providers.google.cloud.hooks.bigtable import BigtableHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, diff --git a/tests/providers/google/cloud/hooks/test_cloud_batch.py b/providers/tests/google/cloud/hooks/test_cloud_batch.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_cloud_batch.py rename to providers/tests/google/cloud/hooks/test_cloud_batch.py index de83b64fbe799..05cc26b6b479b 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_batch.py +++ b/providers/tests/google/cloud/hooks/test_cloud_batch.py @@ -25,7 +25,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.cloud_batch import CloudBatchAsyncHook, CloudBatchHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id pytestmark = pytest.mark.db_test diff --git a/tests/providers/google/cloud/hooks/test_cloud_build.py b/providers/tests/google/cloud/hooks/test_cloud_build.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_cloud_build.py rename to providers/tests/google/cloud/hooks/test_cloud_build.py index 7672d460d0fb2..e65a5cdb8ba37 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_build.py +++ b/providers/tests/google/cloud/hooks/test_cloud_build.py @@ -32,7 +32,8 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildAsyncHook, CloudBuildHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id PROJECT_ID = "cloud-build-project" LOCATION = "test-location" diff --git a/tests/providers/google/cloud/hooks/test_cloud_composer.py b/providers/tests/google/cloud/hooks/test_cloud_composer.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_cloud_composer.py rename to providers/tests/google/cloud/hooks/test_cloud_composer.py diff --git a/tests/providers/google/cloud/hooks/test_cloud_memorystore.py b/providers/tests/google/cloud/hooks/test_cloud_memorystore.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_cloud_memorystore.py rename to providers/tests/google/cloud/hooks/test_cloud_memorystore.py index 592acb82c2362..d96ebc5b4322a 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_memorystore.py +++ b/providers/tests/google/cloud/hooks/test_cloud_memorystore.py @@ -32,7 +32,8 @@ CloudMemorystoreHook, CloudMemorystoreMemcachedHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, diff --git a/tests/providers/google/cloud/hooks/test_cloud_run.py b/providers/tests/google/cloud/hooks/test_cloud_run.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_cloud_run.py rename to providers/tests/google/cloud/hooks/test_cloud_run.py index 48b76e1bca128..d23e96ff6e039 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_run.py +++ b/providers/tests/google/cloud/hooks/test_cloud_run.py @@ -41,7 +41,8 @@ CloudRunServiceAsyncHook, CloudRunServiceHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id @pytest.mark.db_test diff --git a/tests/providers/google/cloud/hooks/test_cloud_sql.py b/providers/tests/google/cloud/hooks/test_cloud_sql.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_cloud_sql.py rename to providers/tests/google/cloud/hooks/test_cloud_sql.py index a5d6e16664072..3365dc76aec37 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_sql.py +++ b/providers/tests/google/cloud/hooks/test_cloud_sql.py @@ -40,7 +40,8 @@ CloudSQLHook, CloudSqlProxyRunner, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service.py b/providers/tests/google/cloud/hooks/test_cloud_storage_transfer_service.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service.py rename to providers/tests/google/cloud/hooks/test_cloud_storage_transfer_service.py index 672a80d2fae1e..a68bcd3afef2c 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service.py +++ b/providers/tests/google/cloud/hooks/test_cloud_storage_transfer_service.py @@ -46,7 +46,8 @@ GcpTransferOperationStatus, gen_job_name, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, diff --git a/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service_async.py b/providers/tests/google/cloud/hooks/test_cloud_storage_transfer_service_async.py similarity index 98% rename from tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service_async.py rename to providers/tests/google/cloud/hooks/test_cloud_storage_transfer_service_async.py index e05bacbbd2c30..968c1a95efbdf 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service_async.py +++ b/providers/tests/google/cloud/hooks/test_cloud_storage_transfer_service_async.py @@ -26,7 +26,8 @@ from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import ( CloudDataTransferServiceAsyncHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id TEST_PROJECT_ID = "project-id" TRANSFER_HOOK_PATH = "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service" diff --git a/tests/providers/google/cloud/hooks/test_compute.py b/providers/tests/google/cloud/hooks/test_compute.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_compute.py rename to providers/tests/google/cloud/hooks/test_compute.py index f4d5da5414ac0..24a43ac3d87bb 100644 --- a/tests/providers/google/cloud/hooks/test_compute.py +++ b/providers/tests/google/cloud/hooks/test_compute.py @@ -25,7 +25,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook, GceOperationStatus -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, diff --git a/tests/providers/google/cloud/hooks/test_compute_ssh.py b/providers/tests/google/cloud/hooks/test_compute_ssh.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_compute_ssh.py rename to providers/tests/google/cloud/hooks/test_compute_ssh.py diff --git a/tests/providers/google/cloud/hooks/test_datacatalog.py b/providers/tests/google/cloud/hooks/test_datacatalog.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_datacatalog.py rename to providers/tests/google/cloud/hooks/test_datacatalog.py index c97cef7c4ae1b..29ce515e8886c 100644 --- a/tests/providers/google/cloud/hooks/test_datacatalog.py +++ b/providers/tests/google/cloud/hooks/test_datacatalog.py @@ -28,7 +28,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.datacatalog import CloudDataCatalogHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/test_dataflow.py b/providers/tests/google/cloud/hooks/test_dataflow.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_dataflow.py rename to providers/tests/google/cloud/hooks/test_dataflow.py diff --git a/tests/providers/google/cloud/hooks/test_dataform.py b/providers/tests/google/cloud/hooks/test_dataform.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_dataform.py rename to providers/tests/google/cloud/hooks/test_dataform.py index d5c94e40d5f97..bedf91dadf6e4 100644 --- a/tests/providers/google/cloud/hooks/test_dataform.py +++ b/providers/tests/google/cloud/hooks/test_dataform.py @@ -25,7 +25,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.dataform import DataformHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id pytestmark = pytest.mark.db_test diff --git a/tests/providers/google/cloud/hooks/test_datafusion.py b/providers/tests/google/cloud/hooks/test_datafusion.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_datafusion.py rename to providers/tests/google/cloud/hooks/test_datafusion.py index 7358c334fda11..271662f7bca5e 100644 --- a/tests/providers/google/cloud/hooks/test_datafusion.py +++ b/providers/tests/google/cloud/hooks/test_datafusion.py @@ -27,7 +27,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.datafusion import DataFusionAsyncHook, DataFusionHook from airflow.providers.google.cloud.utils.datafusion import DataFusionPipelineType -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id API_VERSION = "v1beta1" GCP_CONN_ID = "google_cloud_default" diff --git a/tests/providers/google/cloud/hooks/test_datapipeline.py b/providers/tests/google/cloud/hooks/test_datapipeline.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_datapipeline.py rename to providers/tests/google/cloud/hooks/test_datapipeline.py diff --git a/tests/providers/google/cloud/hooks/test_dataplex.py b/providers/tests/google/cloud/hooks/test_dataplex.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_dataplex.py rename to providers/tests/google/cloud/hooks/test_dataplex.py index deca942a02fc2..9221a0e094653 100644 --- a/tests/providers/google/cloud/hooks/test_dataplex.py +++ b/providers/tests/google/cloud/hooks/test_dataplex.py @@ -22,7 +22,8 @@ from google.api_core.gapic_v1.method import DEFAULT from airflow.providers.google.cloud.operators.dataplex import DataplexHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id BASE_STRING = "airflow.providers.google.common.hooks.base_google.{}" DATAPLEX_STRING = "airflow.providers.google.cloud.hooks.dataplex.{}" diff --git a/tests/providers/google/cloud/hooks/test_dataprep.py b/providers/tests/google/cloud/hooks/test_dataprep.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_dataprep.py rename to providers/tests/google/cloud/hooks/test_dataprep.py diff --git a/tests/providers/google/cloud/hooks/test_dataproc.py b/providers/tests/google/cloud/hooks/test_dataproc.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_dataproc.py rename to providers/tests/google/cloud/hooks/test_dataproc.py diff --git a/tests/providers/google/cloud/hooks/test_dataproc_metastore.py b/providers/tests/google/cloud/hooks/test_dataproc_metastore.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_dataproc_metastore.py rename to providers/tests/google/cloud/hooks/test_dataproc_metastore.py index 28f506ce301b8..58a04d6bc8351 100644 --- a/tests/providers/google/cloud/hooks/test_dataproc_metastore.py +++ b/providers/tests/google/cloud/hooks/test_dataproc_metastore.py @@ -23,7 +23,8 @@ from google.api_core.gapic_v1.method import DEFAULT from airflow.providers.google.cloud.hooks.dataproc_metastore import DataprocMetastoreHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/test_datastore.py b/providers/tests/google/cloud/hooks/test_datastore.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_datastore.py rename to providers/tests/google/cloud/hooks/test_datastore.py diff --git a/tests/providers/google/cloud/hooks/test_dlp.py b/providers/tests/google/cloud/hooks/test_dlp.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_dlp.py rename to providers/tests/google/cloud/hooks/test_dlp.py index 186d5de1732b3..f0c63fcd23625 100644 --- a/tests/providers/google/cloud/hooks/test_dlp.py +++ b/providers/tests/google/cloud/hooks/test_dlp.py @@ -33,7 +33,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.dlp import CloudDLPHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id API_RESPONSE: dict[Any, Any] = {} ORGANIZATION_ID = "test-org" diff --git a/tests/providers/google/cloud/hooks/test_functions.py b/providers/tests/google/cloud/hooks/test_functions.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_functions.py rename to providers/tests/google/cloud/hooks/test_functions.py index f7bb02b1e2c9e..4e3c48795117c 100644 --- a/tests/providers/google/cloud/hooks/test_functions.py +++ b/providers/tests/google/cloud/hooks/test_functions.py @@ -24,7 +24,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.functions import CloudFunctionsHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, get_open_mock, mock_base_gcp_hook_default_project_id, diff --git a/tests/providers/google/cloud/hooks/test_gcs.py b/providers/tests/google/cloud/hooks/test_gcs.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_gcs.py rename to providers/tests/google/cloud/hooks/test_gcs.py index 5d2735834a958..464534bd11d93 100644 --- a/tests/providers/google/cloud/hooks/test_gcs.py +++ b/providers/tests/google/cloud/hooks/test_gcs.py @@ -41,7 +41,8 @@ from airflow.providers.google.common.consts import CLIENT_INFO from airflow.utils import timezone from airflow.version import version -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id BASE_STRING = "airflow.providers.google.common.hooks.base_google.{}" GCS_STRING = "airflow.providers.google.cloud.hooks.gcs.{}" diff --git a/tests/providers/google/cloud/hooks/test_gdm.py b/providers/tests/google/cloud/hooks/test_gdm.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_gdm.py rename to providers/tests/google/cloud/hooks/test_gdm.py diff --git a/tests/providers/google/cloud/hooks/test_kms.py b/providers/tests/google/cloud/hooks/test_kms.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_kms.py rename to providers/tests/google/cloud/hooks/test_kms.py diff --git a/tests/providers/google/cloud/hooks/test_kms_system.py b/providers/tests/google/cloud/hooks/test_kms_system.py similarity index 96% rename from tests/providers/google/cloud/hooks/test_kms_system.py rename to providers/tests/google/cloud/hooks/test_kms_system.py index afb4de1d1d04f..374fabb5246dd 100644 --- a/tests/providers/google/cloud/hooks/test_kms_system.py +++ b/providers/tests/google/cloud/hooks/test_kms_system.py @@ -23,8 +23,9 @@ import pytest from airflow.providers.google.cloud.hooks.kms import CloudKMSHook -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_KMS_KEY -from tests.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context + +from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_KMS_KEY # To prevent resource name collisions, key ring and key resources CANNOT be deleted, so # to avoid cluttering the project, we only create the key once during project initialization. diff --git a/tests/providers/google/cloud/hooks/test_kubernetes_engine.py b/providers/tests/google/cloud/hooks/test_kubernetes_engine.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_kubernetes_engine.py rename to providers/tests/google/cloud/hooks/test_kubernetes_engine.py index 479cceed05815..e535323f4d4cf 100644 --- a/tests/providers/google/cloud/hooks/test_kubernetes_engine.py +++ b/providers/tests/google/cloud/hooks/test_kubernetes_engine.py @@ -34,7 +34,8 @@ GKEKubernetesHook, ) from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id TASK_ID = "test-gke-cluster-operator" CLUSTER_NAME = "test-cluster" diff --git a/tests/providers/google/cloud/hooks/test_life_sciences.py b/providers/tests/google/cloud/hooks/test_life_sciences.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_life_sciences.py rename to providers/tests/google/cloud/hooks/test_life_sciences.py index 582b856b22c8b..ca113ec648f55 100644 --- a/tests/providers/google/cloud/hooks/test_life_sciences.py +++ b/providers/tests/google/cloud/hooks/test_life_sciences.py @@ -28,7 +28,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.life_sciences import LifeSciencesHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, diff --git a/tests/providers/google/cloud/hooks/test_looker.py b/providers/tests/google/cloud/hooks/test_looker.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_looker.py rename to providers/tests/google/cloud/hooks/test_looker.py diff --git a/tests/providers/google/cloud/hooks/test_mlengine.py b/providers/tests/google/cloud/hooks/test_mlengine.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_mlengine.py rename to providers/tests/google/cloud/hooks/test_mlengine.py index 7e8d7d68032fb..85ac30327d4c7 100644 --- a/tests/providers/google/cloud/hooks/test_mlengine.py +++ b/providers/tests/google/cloud/hooks/test_mlengine.py @@ -31,7 +31,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks import mlengine as hook from airflow.providers.google.cloud.hooks.mlengine import MLEngineAsyncHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/test_natural_language.py b/providers/tests/google/cloud/hooks/test_natural_language.py similarity index 98% rename from tests/providers/google/cloud/hooks/test_natural_language.py rename to providers/tests/google/cloud/hooks/test_natural_language.py index 22ba3d87c10f5..a3228439b10eb 100644 --- a/tests/providers/google/cloud/hooks/test_natural_language.py +++ b/providers/tests/google/cloud/hooks/test_natural_language.py @@ -26,7 +26,8 @@ from airflow.providers.google.cloud.hooks.natural_language import CloudNaturalLanguageHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id API_RESPONSE: dict[Any, Any] = {} DOCUMENT = Document( diff --git a/tests/providers/google/cloud/hooks/test_os_login.py b/providers/tests/google/cloud/hooks/test_os_login.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_os_login.py rename to providers/tests/google/cloud/hooks/test_os_login.py index 568e96610c762..48861f6954692 100644 --- a/tests/providers/google/cloud/hooks/test_os_login.py +++ b/providers/tests/google/cloud/hooks/test_os_login.py @@ -24,7 +24,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.os_login import OSLoginHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/test_pubsub.py b/providers/tests/google/cloud/hooks/test_pubsub.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_pubsub.py rename to providers/tests/google/cloud/hooks/test_pubsub.py diff --git a/tests/providers/google/cloud/hooks/test_secret_manager.py b/providers/tests/google/cloud/hooks/test_secret_manager.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_secret_manager.py rename to providers/tests/google/cloud/hooks/test_secret_manager.py index a800f1f785b8c..e758f76a9dcd7 100644 --- a/tests/providers/google/cloud/hooks/test_secret_manager.py +++ b/providers/tests/google/cloud/hooks/test_secret_manager.py @@ -29,7 +29,8 @@ SecretsManagerHook, ) from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/test_secret_manager_system.py b/providers/tests/google/cloud/hooks/test_secret_manager_system.py similarity index 95% rename from tests/providers/google/cloud/hooks/test_secret_manager_system.py rename to providers/tests/google/cloud/hooks/test_secret_manager_system.py index 6aacc078b9a28..d13305f52b42e 100644 --- a/tests/providers/google/cloud/hooks/test_secret_manager_system.py +++ b/providers/tests/google/cloud/hooks/test_secret_manager_system.py @@ -21,8 +21,9 @@ import pytest from airflow.providers.google.cloud.hooks.secret_manager import SecretsManagerHook -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY -from tests.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context + +from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY TEST_SECRET_ID = os.environ.get("GCP_SECRET_MANAGER_SECRET_ID", "test-secret") TEST_SECRET_VALUE = os.environ.get("GCP_SECRET_MANAGER_SECRET_VALUE", "test-secret-value") diff --git a/tests/providers/google/cloud/hooks/test_spanner.py b/providers/tests/google/cloud/hooks/test_spanner.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_spanner.py rename to providers/tests/google/cloud/hooks/test_spanner.py index 1633d558166d4..3f6af9dcdcaf2 100644 --- a/tests/providers/google/cloud/hooks/test_spanner.py +++ b/providers/tests/google/cloud/hooks/test_spanner.py @@ -25,7 +25,8 @@ from airflow.providers.google.cloud.hooks.spanner import SpannerHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, diff --git a/tests/providers/google/cloud/hooks/test_speech_to_text.py b/providers/tests/google/cloud/hooks/test_speech_to_text.py similarity index 97% rename from tests/providers/google/cloud/hooks/test_speech_to_text.py rename to providers/tests/google/cloud/hooks/test_speech_to_text.py index 8cda071976245..97ef56f790b48 100644 --- a/tests/providers/google/cloud/hooks/test_speech_to_text.py +++ b/providers/tests/google/cloud/hooks/test_speech_to_text.py @@ -25,7 +25,8 @@ from airflow.providers.google.cloud.hooks.speech_to_text import CloudSpeechToTextHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id PROJECT_ID = "project-id" CONFIG = {"encoding": "LINEAR16"} diff --git a/tests/providers/google/cloud/hooks/test_stackdriver.py b/providers/tests/google/cloud/hooks/test_stackdriver.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_stackdriver.py rename to providers/tests/google/cloud/hooks/test_stackdriver.py diff --git a/tests/providers/google/cloud/hooks/test_tasks.py b/providers/tests/google/cloud/hooks/test_tasks.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_tasks.py rename to providers/tests/google/cloud/hooks/test_tasks.py index 9885248db45e0..71228ba7e7178 100644 --- a/tests/providers/google/cloud/hooks/test_tasks.py +++ b/providers/tests/google/cloud/hooks/test_tasks.py @@ -26,7 +26,8 @@ from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id API_RESPONSE: dict[Any, Any] = {} PROJECT_ID = "test-project" diff --git a/tests/providers/google/cloud/hooks/test_text_to_speech.py b/providers/tests/google/cloud/hooks/test_text_to_speech.py similarity index 97% rename from tests/providers/google/cloud/hooks/test_text_to_speech.py rename to providers/tests/google/cloud/hooks/test_text_to_speech.py index 573c56e098f58..4f3eb90684499 100644 --- a/tests/providers/google/cloud/hooks/test_text_to_speech.py +++ b/providers/tests/google/cloud/hooks/test_text_to_speech.py @@ -29,7 +29,8 @@ from airflow.providers.google.cloud.hooks.text_to_speech import CloudTextToSpeechHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id INPUT = {"text": "test text"} VOICE = {"language_code": "en-US", "ssml_gender": "FEMALE"} diff --git a/tests/providers/google/cloud/hooks/test_translate.py b/providers/tests/google/cloud/hooks/test_translate.py similarity index 98% rename from tests/providers/google/cloud/hooks/test_translate.py rename to providers/tests/google/cloud/hooks/test_translate.py index 068fe60fa373c..addc54d430e7a 100644 --- a/tests/providers/google/cloud/hooks/test_translate.py +++ b/providers/tests/google/cloud/hooks/test_translate.py @@ -23,7 +23,8 @@ from airflow.providers.google.cloud.hooks.translate import CloudTranslateHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id PROJECT_ID_TEST = "project-id" diff --git a/tests/providers/google/cloud/hooks/test_video_intelligence.py b/providers/tests/google/cloud/hooks/test_video_intelligence.py similarity index 98% rename from tests/providers/google/cloud/hooks/test_video_intelligence.py rename to providers/tests/google/cloud/hooks/test_video_intelligence.py index 75bb9b92f71c8..17e047e2da956 100644 --- a/tests/providers/google/cloud/hooks/test_video_intelligence.py +++ b/providers/tests/google/cloud/hooks/test_video_intelligence.py @@ -25,7 +25,8 @@ from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIntelligenceHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id INPUT_URI = "gs://bucket-name/input-file" OUTPUT_URI = "gs://bucket-name/output-file" diff --git a/tests/providers/google/cloud/hooks/test_vision.py b/providers/tests/google/cloud/hooks/test_vision.py similarity index 99% rename from tests/providers/google/cloud/hooks/test_vision.py rename to providers/tests/google/cloud/hooks/test_vision.py index 814e412ee0fc5..6a414d59e1a19 100644 --- a/tests/providers/google/cloud/hooks/test_vision.py +++ b/providers/tests/google/cloud/hooks/test_vision.py @@ -37,7 +37,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.vision import ERR_DIFF_NAMES, ERR_UNABLE_TO_CREATE, CloudVisionHook from airflow.providers.google.common.consts import CLIENT_INFO -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id PROJECT_ID_TEST = "project-id" PROJECT_ID_TEST_2 = "project-id-2" diff --git a/tests/providers/google/cloud/hooks/test_workflows.py b/providers/tests/google/cloud/hooks/test_workflows.py similarity index 100% rename from tests/providers/google/cloud/hooks/test_workflows.py rename to providers/tests/google/cloud/hooks/test_workflows.py diff --git a/tests/providers/databricks/plugins/__init__.py b/providers/tests/google/cloud/hooks/vertex_ai/__init__.py similarity index 100% rename from tests/providers/databricks/plugins/__init__.py rename to providers/tests/google/cloud/hooks/vertex_ai/__init__.py diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_auto_ml.py b/providers/tests/google/cloud/hooks/vertex_ai/test_auto_ml.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_auto_ml.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_auto_ml.py index 2e90cb7a1778b..6b64c966f54c9 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_auto_ml.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_auto_ml.py @@ -27,7 +27,8 @@ from google.api_core.gapic_v1.method import DEFAULT from airflow.providers.google.cloud.hooks.vertex_ai.auto_ml import AutoMLHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py b/providers/tests/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py index 1b2e0e1fbb9eb..15efe3e88bac4 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py @@ -33,7 +33,8 @@ BatchPredictionJobAsyncHook, BatchPredictionJobHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py b/providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py index c6f31c48470e2..a2cedce2a92a5 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_custom_job.py @@ -36,7 +36,8 @@ PipelineState, types, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_dataset.py b/providers/tests/google/cloud/hooks/vertex_ai/test_dataset.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_dataset.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_dataset.py index f20c9142a2c09..d54eb48c2d686 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_dataset.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_dataset.py @@ -27,7 +27,8 @@ from google.api_core.gapic_v1.method import DEFAULT from airflow.providers.google.cloud.hooks.vertex_ai.dataset import DatasetHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_endpoint_service.py b/providers/tests/google/cloud/hooks/vertex_ai/test_endpoint_service.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_endpoint_service.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_endpoint_service.py index 273a97fa19af9..a284eae13a41b 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_endpoint_service.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_endpoint_service.py @@ -27,7 +27,8 @@ from google.api_core.gapic_v1.method import DEFAULT from airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service import EndpointServiceHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_generative_model.py b/providers/tests/google/cloud/hooks/vertex_ai/test_generative_model.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_generative_model.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_generative_model.py index 19723a51b1ddb..52a8c417c6ac2 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_generative_model.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_generative_model.py @@ -33,7 +33,8 @@ from airflow.providers.google.cloud.hooks.vertex_ai.generative_model import ( GenerativeModelHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py b/providers/tests/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py index eca6fc17ebaf9..05fbf9c123447 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py @@ -37,7 +37,8 @@ HyperparameterTuningJobAsyncHook, HyperparameterTuningJobHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_model_service.py b/providers/tests/google/cloud/hooks/vertex_ai/test_model_service.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_model_service.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_model_service.py index 1505e8276d077..c95d3fecd492f 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_model_service.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_model_service.py @@ -27,7 +27,8 @@ from google.api_core.gapic_v1.method import DEFAULT from airflow.providers.google.cloud.hooks.vertex_ai.model_service import ModelServiceHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_pipeline_job.py b/providers/tests/google/cloud/hooks/vertex_ai/test_pipeline_job.py similarity index 99% rename from tests/providers/google/cloud/hooks/vertex_ai/test_pipeline_job.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_pipeline_job.py index 02068ec57051f..21937251c3c53 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_pipeline_job.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_pipeline_job.py @@ -34,7 +34,8 @@ PipelineState, types, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/google/cloud/hooks/vertex_ai/test_prediction_service.py b/providers/tests/google/cloud/hooks/vertex_ai/test_prediction_service.py similarity index 98% rename from tests/providers/google/cloud/hooks/vertex_ai/test_prediction_service.py rename to providers/tests/google/cloud/hooks/vertex_ai/test_prediction_service.py index 987578b7c11ea..7e56ca92813a5 100644 --- a/tests/providers/google/cloud/hooks/vertex_ai/test_prediction_service.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_prediction_service.py @@ -28,7 +28,8 @@ from airflow.providers.google.cloud.hooks.vertex_ai.prediction_service import ( PredictionServiceHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, ) diff --git a/tests/providers/databricks/sensors/__init__.py b/providers/tests/google/cloud/links/__init__.py similarity index 100% rename from tests/providers/databricks/sensors/__init__.py rename to providers/tests/google/cloud/links/__init__.py diff --git a/tests/providers/google/cloud/links/test_translate.py b/providers/tests/google/cloud/links/test_translate.py similarity index 100% rename from tests/providers/google/cloud/links/test_translate.py rename to providers/tests/google/cloud/links/test_translate.py diff --git a/tests/providers/databricks/utils/__init__.py b/providers/tests/google/cloud/log/__init__.py similarity index 100% rename from tests/providers/databricks/utils/__init__.py rename to providers/tests/google/cloud/log/__init__.py diff --git a/tests/providers/google/cloud/log/test_gcs_task_handler.py b/providers/tests/google/cloud/log/test_gcs_task_handler.py similarity index 98% rename from tests/providers/google/cloud/log/test_gcs_task_handler.py rename to providers/tests/google/cloud/log/test_gcs_task_handler.py index 7653c4d837be4..9f27c618de29d 100644 --- a/tests/providers/google/cloud/log/test_gcs_task_handler.py +++ b/providers/tests/google/cloud/log/test_gcs_task_handler.py @@ -27,8 +27,9 @@ from airflow.providers.google.cloud.log.gcs_task_handler import GCSTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs @pytest.mark.db_test diff --git a/tests/providers/google/cloud/log/test_gcs_task_handler_system.py b/providers/tests/google/cloud/log/test_gcs_task_handler_system.py similarity index 84% rename from tests/providers/google/cloud/log/test_gcs_task_handler_system.py rename to providers/tests/google/cloud/log/test_gcs_task_handler_system.py index 406bb0387df57..eb1ce08d14f5c 100644 --- a/tests/providers/google/cloud/log/test_gcs_task_handler_system.py +++ b/providers/tests/google/cloud/log/test_gcs_task_handler_system.py @@ -29,14 +29,15 @@ from airflow.models import DagBag, TaskInstance from airflow.utils.log.log_reader import TaskLogReader from airflow.utils.session import provide_session -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_connections, clear_db_runs -from tests.test_utils.gcp_system_helpers import ( + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_connections, clear_db_runs +from dev.tests_common.test_utils.gcp_system_helpers import ( GoogleSystemTest, provide_gcp_context, resolve_full_gcp_key_path, ) +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY @pytest.mark.system("google") @@ -84,12 +85,15 @@ def test_should_read_logs(self, session): self.assert_remote_logs("INFO - Task exited with return code 0", ti) def assert_remote_logs(self, expected_message, ti): - with provide_gcp_context(GCP_GCS_KEY), conf_vars( - { - ("logging", "remote_logging"): "True", - ("logging", "remote_base_log_folder"): f"gs://{self.bucket_name}/path/to/logs", - ("logging", "remote_log_conn_id"): "google_cloud_default", - } + with ( + provide_gcp_context(GCP_GCS_KEY), + conf_vars( + { + ("logging", "remote_logging"): "True", + ("logging", "remote_base_log_folder"): f"gs://{self.bucket_name}/path/to/logs", + ("logging", "remote_log_conn_id"): "google_cloud_default", + } + ), ): from airflow.config_templates import airflow_local_settings diff --git a/tests/providers/google/cloud/log/test_stackdriver_task_handler.py b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py similarity index 99% rename from tests/providers/google/cloud/log/test_stackdriver_task_handler.py rename to providers/tests/google/cloud/log/test_stackdriver_task_handler.py index 1ac8b91df5a65..783f1a34b762c 100644 --- a/tests/providers/google/cloud/log/test_stackdriver_task_handler.py +++ b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py @@ -29,9 +29,10 @@ from airflow.providers.google.cloud.log.stackdriver_task_handler import StackdriverTaskHandler from airflow.utils import timezone from airflow.utils.state import TaskInstanceState -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs def _create_list_log_entries_response_mock(messages, token): diff --git a/tests/providers/google/cloud/log/test_stackdriver_task_handler_system.py b/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py similarity index 88% rename from tests/providers/google/cloud/log/test_stackdriver_task_handler_system.py rename to providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py index b5daac0d81495..a53dd43d08feb 100644 --- a/tests/providers/google/cloud/log/test_stackdriver_task_handler_system.py +++ b/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py @@ -29,14 +29,15 @@ from airflow.models import TaskInstance from airflow.utils.log.log_reader import TaskLogReader from airflow.utils.session import provide_session -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_STACKDRIVER -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_runs -from tests.test_utils.gcp_system_helpers import ( + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_runs +from dev.tests_common.test_utils.gcp_system_helpers import ( GoogleSystemTest, provide_gcp_context, resolve_full_gcp_key_path, ) +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_STACKDRIVER @pytest.mark.system("google") @@ -86,11 +87,14 @@ def test_should_support_adc(self, session): self.assert_remote_logs("terminated with exit code 0", ti) def assert_remote_logs(self, expected_message, ti): - with provide_gcp_context(GCP_STACKDRIVER), conf_vars( - { - ("logging", "remote_logging"): "True", - ("logging", "remote_base_log_folder"): f"stackdriver://{self.log_name}", - } + with ( + provide_gcp_context(GCP_STACKDRIVER), + conf_vars( + { + ("logging", "remote_logging"): "True", + ("logging", "remote_base_log_folder"): f"stackdriver://{self.log_name}", + } + ), ): from airflow.config_templates import airflow_local_settings diff --git a/tests/providers/dbt/__init__.py b/providers/tests/google/cloud/openlineage/__init__.py similarity index 100% rename from tests/providers/dbt/__init__.py rename to providers/tests/google/cloud/openlineage/__init__.py diff --git a/tests/providers/google/cloud/openlineage/test_mixins.py b/providers/tests/google/cloud/openlineage/test_mixins.py similarity index 95% rename from tests/providers/google/cloud/openlineage/test_mixins.py rename to providers/tests/google/cloud/openlineage/test_mixins.py index f7feade65d361..5229db3a6071d 100644 --- a/tests/providers/google/cloud/openlineage/test_mixins.py +++ b/providers/tests/google/cloud/openlineage/test_mixins.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +import os from unittest.mock import MagicMock import pytest @@ -35,17 +36,17 @@ ) -def read_file_json(file): - with open(file=file) as f: - return json.loads(f.read()) +def read_common_json_file(rel: str): + with open(os.path.dirname(__file__) + "/../utils/" + rel) as f: + return json.load(f) class TableMock(MagicMock): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.inputs = [ - read_file_json("tests/providers/google/cloud/utils/table_details.json"), - read_file_json("tests/providers/google/cloud/utils/out_table_details.json"), + read_common_json_file("table_details.json"), + read_common_json_file("out_table_details.json"), ] @property @@ -55,8 +56,8 @@ def _properties(self): class TestBigQueryOpenLineageMixin: def setup_method(self): - self.job_details = read_file_json("tests/providers/google/cloud/utils/job_details.json") - self.script_job_details = read_file_json("tests/providers/google/cloud/utils/script_job_details.json") + self.job_details = read_common_json_file("job_details.json") + self.script_job_details = read_common_json_file("script_job_details.json") hook = MagicMock() self.client = MagicMock() diff --git a/tests/providers/google/cloud/openlineage/test_utils.py b/providers/tests/google/cloud/openlineage/test_utils.py similarity index 97% rename from tests/providers/google/cloud/openlineage/test_utils.py rename to providers/tests/google/cloud/openlineage/test_utils.py index e47f14332f455..4f2db0038b7b7 100644 --- a/tests/providers/google/cloud/openlineage/test_utils.py +++ b/providers/tests/google/cloud/openlineage/test_utils.py @@ -65,8 +65,8 @@ class TableMock(MagicMock): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.inputs = [ - read_file_json("tests/providers/google/cloud/utils/table_details.json"), - read_file_json("tests/providers/google/cloud/utils/out_table_details.json"), + read_file_json("providers/tests/google/cloud/utils/table_details.json"), + read_file_json("providers/tests/google/cloud/utils/out_table_details.json"), ] @property diff --git a/tests/providers/dbt/cloud/__init__.py b/providers/tests/google/cloud/operators/__init__.py similarity index 100% rename from tests/providers/dbt/cloud/__init__.py rename to providers/tests/google/cloud/operators/__init__.py diff --git a/tests/providers/dbt/cloud/hooks/__init__.py b/providers/tests/google/cloud/operators/source/__init__.py similarity index 100% rename from tests/providers/dbt/cloud/hooks/__init__.py rename to providers/tests/google/cloud/operators/source/__init__.py diff --git a/tests/providers/dbt/cloud/operators/__init__.py b/providers/tests/google/cloud/operators/source/source_prefix/__init__.py similarity index 100% rename from tests/providers/dbt/cloud/operators/__init__.py rename to providers/tests/google/cloud/operators/source/source_prefix/__init__.py diff --git a/tests/providers/google/cloud/operators/test_automl.py b/providers/tests/google/cloud/operators/test_automl.py similarity index 100% rename from tests/providers/google/cloud/operators/test_automl.py rename to providers/tests/google/cloud/operators/test_automl.py diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/providers/tests/google/cloud/operators/test_bigquery.py similarity index 99% rename from tests/providers/google/cloud/operators/test_bigquery.py rename to providers/tests/google/cloud/operators/test_bigquery.py index ab9e8e6ac019f..3836d737662fc 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/providers/tests/google/cloud/operators/test_bigquery.py @@ -18,6 +18,7 @@ from __future__ import annotations import json +import os from contextlib import suppress from unittest import mock from unittest.mock import ANY, MagicMock @@ -73,7 +74,13 @@ ) from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.timezone import datetime -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags, clear_db_xcom + +from dev.tests_common.test_utils.db import ( + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, + clear_db_xcom, +) pytestmark = pytest.mark.db_test @@ -1840,7 +1847,7 @@ def test_execute_openlineage_events(self, mock_hook): assert result == real_job_id - with open(file="tests/providers/google/cloud/utils/job_details.json") as f: + with open(os.path.dirname(__file__) + "/../utils/job_details.json") as f: job_details = json.loads(f.read()) mock_hook.return_value.get_client.return_value.get_job.return_value._properties = job_details mock_hook.return_value.get_client.return_value.get_table.side_effect = Exception() diff --git a/tests/providers/google/cloud/operators/test_bigquery_dts.py b/providers/tests/google/cloud/operators/test_bigquery_dts.py similarity index 98% rename from tests/providers/google/cloud/operators/test_bigquery_dts.py rename to providers/tests/google/cloud/operators/test_bigquery_dts.py index f44479bbce9e4..f50c42805b49d 100644 --- a/tests/providers/google/cloud/operators/test_bigquery_dts.py +++ b/providers/tests/google/cloud/operators/test_bigquery_dts.py @@ -27,7 +27,8 @@ BigQueryDataTransferServiceStartTransferRunsOperator, BigQueryDeleteDataTransferConfigOperator, ) -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS PROJECT_ID = "id" diff --git a/tests/providers/google/cloud/operators/test_bigtable.py b/providers/tests/google/cloud/operators/test_bigtable.py similarity index 100% rename from tests/providers/google/cloud/operators/test_bigtable.py rename to providers/tests/google/cloud/operators/test_bigtable.py diff --git a/tests/providers/google/cloud/operators/test_cloud_base.py b/providers/tests/google/cloud/operators/test_cloud_base.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_base.py rename to providers/tests/google/cloud/operators/test_cloud_base.py diff --git a/tests/providers/google/cloud/operators/test_cloud_batch.py b/providers/tests/google/cloud/operators/test_cloud_batch.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_batch.py rename to providers/tests/google/cloud/operators/test_cloud_batch.py diff --git a/tests/providers/google/cloud/operators/test_cloud_build.py b/providers/tests/google/cloud/operators/test_cloud_build.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_build.py rename to providers/tests/google/cloud/operators/test_cloud_build.py diff --git a/tests/providers/google/cloud/operators/test_cloud_composer.py b/providers/tests/google/cloud/operators/test_cloud_composer.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_composer.py rename to providers/tests/google/cloud/operators/test_cloud_composer.py diff --git a/tests/providers/google/cloud/operators/test_cloud_memorystore.py b/providers/tests/google/cloud/operators/test_cloud_memorystore.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_memorystore.py rename to providers/tests/google/cloud/operators/test_cloud_memorystore.py diff --git a/tests/providers/google/cloud/operators/test_cloud_run.py b/providers/tests/google/cloud/operators/test_cloud_run.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_run.py rename to providers/tests/google/cloud/operators/test_cloud_run.py diff --git a/tests/providers/google/cloud/operators/test_cloud_sql.py b/providers/tests/google/cloud/operators/test_cloud_sql.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_sql.py rename to providers/tests/google/cloud/operators/test_cloud_sql.py diff --git a/tests/providers/google/cloud/operators/test_cloud_storage_transfer_service.py b/providers/tests/google/cloud/operators/test_cloud_storage_transfer_service.py similarity index 100% rename from tests/providers/google/cloud/operators/test_cloud_storage_transfer_service.py rename to providers/tests/google/cloud/operators/test_cloud_storage_transfer_service.py diff --git a/tests/providers/google/cloud/operators/test_compute.py b/providers/tests/google/cloud/operators/test_compute.py similarity index 100% rename from tests/providers/google/cloud/operators/test_compute.py rename to providers/tests/google/cloud/operators/test_compute.py diff --git a/tests/providers/google/cloud/operators/test_datacatalog.py b/providers/tests/google/cloud/operators/test_datacatalog.py similarity index 100% rename from tests/providers/google/cloud/operators/test_datacatalog.py rename to providers/tests/google/cloud/operators/test_datacatalog.py diff --git a/tests/providers/google/cloud/operators/test_dataflow.py b/providers/tests/google/cloud/operators/test_dataflow.py similarity index 100% rename from tests/providers/google/cloud/operators/test_dataflow.py rename to providers/tests/google/cloud/operators/test_dataflow.py diff --git a/tests/providers/google/cloud/operators/test_dataform.py b/providers/tests/google/cloud/operators/test_dataform.py similarity index 100% rename from tests/providers/google/cloud/operators/test_dataform.py rename to providers/tests/google/cloud/operators/test_dataform.py diff --git a/tests/providers/google/cloud/operators/test_datafusion.py b/providers/tests/google/cloud/operators/test_datafusion.py similarity index 100% rename from tests/providers/google/cloud/operators/test_datafusion.py rename to providers/tests/google/cloud/operators/test_datafusion.py diff --git a/tests/providers/google/cloud/operators/test_datapipeline.py b/providers/tests/google/cloud/operators/test_datapipeline.py similarity index 100% rename from tests/providers/google/cloud/operators/test_datapipeline.py rename to providers/tests/google/cloud/operators/test_datapipeline.py diff --git a/tests/providers/google/cloud/operators/test_dataplex.py b/providers/tests/google/cloud/operators/test_dataplex.py similarity index 100% rename from tests/providers/google/cloud/operators/test_dataplex.py rename to providers/tests/google/cloud/operators/test_dataplex.py diff --git a/tests/providers/google/cloud/operators/test_dataprep.py b/providers/tests/google/cloud/operators/test_dataprep.py similarity index 100% rename from tests/providers/google/cloud/operators/test_dataprep.py rename to providers/tests/google/cloud/operators/test_dataprep.py diff --git a/tests/providers/google/cloud/operators/test_dataprep_system.py b/providers/tests/google/cloud/operators/test_dataprep_system.py similarity index 91% rename from tests/providers/google/cloud/operators/test_dataprep_system.py rename to providers/tests/google/cloud/operators/test_dataprep_system.py index fcba01fe599cc..96f47fa3e365d 100644 --- a/tests/providers/google/cloud/operators/test_dataprep_system.py +++ b/providers/tests/google/cloud/operators/test_dataprep_system.py @@ -24,8 +24,9 @@ from airflow.models import Connection from airflow.utils.session import create_session -from tests.test_utils.db import clear_db_connections -from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest + +from dev.tests_common.test_utils.db import clear_db_connections +from dev.tests_common.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest TOKEN = os.environ.get("DATAPREP_TOKEN") EXTRA = {"token": TOKEN} diff --git a/tests/providers/google/cloud/operators/test_dataproc.py b/providers/tests/google/cloud/operators/test_dataproc.py similarity index 99% rename from tests/providers/google/cloud/operators/test_dataproc.py rename to providers/tests/google/cloud/operators/test_dataproc.py index 58b38125ee1d3..cf4bffa3a0929 100644 --- a/tests/providers/google/cloud/operators/test_dataproc.py +++ b/providers/tests/google/cloud/operators/test_dataproc.py @@ -79,8 +79,9 @@ from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.timezone import datetime -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_VERSION -from tests.test_utils.db import clear_db_runs, clear_db_xcom + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_VERSION +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom AIRFLOW_VERSION_LABEL = "v" + str(AIRFLOW_VERSION).replace(".", "-").replace("+", "-") diff --git a/tests/providers/google/cloud/operators/test_dataproc_metastore.py b/providers/tests/google/cloud/operators/test_dataproc_metastore.py similarity index 100% rename from tests/providers/google/cloud/operators/test_dataproc_metastore.py rename to providers/tests/google/cloud/operators/test_dataproc_metastore.py diff --git a/tests/providers/google/cloud/operators/test_datastore.py b/providers/tests/google/cloud/operators/test_datastore.py similarity index 100% rename from tests/providers/google/cloud/operators/test_datastore.py rename to providers/tests/google/cloud/operators/test_datastore.py diff --git a/tests/providers/google/cloud/operators/test_datastore_system.py b/providers/tests/google/cloud/operators/test_datastore_system.py similarity index 89% rename from tests/providers/google/cloud/operators/test_datastore_system.py rename to providers/tests/google/cloud/operators/test_datastore_system.py index 5835a46673db1..a98215a5317a3 100644 --- a/tests/providers/google/cloud/operators/test_datastore_system.py +++ b/providers/tests/google/cloud/operators/test_datastore_system.py @@ -21,8 +21,12 @@ import pytest -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_DATASTORE_KEY -from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context +from dev.tests_common.test_utils.gcp_system_helpers import ( + CLOUD_DAG_FOLDER, + GoogleSystemTest, + provide_gcp_context, +) +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_DATASTORE_KEY BUCKET = os.environ.get("GCP_DATASTORE_BUCKET", "datastore-system-test") diff --git a/tests/providers/google/cloud/operators/test_dlp.py b/providers/tests/google/cloud/operators/test_dlp.py similarity index 100% rename from tests/providers/google/cloud/operators/test_dlp.py rename to providers/tests/google/cloud/operators/test_dlp.py diff --git a/tests/providers/google/cloud/operators/test_functions.py b/providers/tests/google/cloud/operators/test_functions.py similarity index 100% rename from tests/providers/google/cloud/operators/test_functions.py rename to providers/tests/google/cloud/operators/test_functions.py diff --git a/tests/providers/google/cloud/operators/test_gcs.py b/providers/tests/google/cloud/operators/test_gcs.py similarity index 100% rename from tests/providers/google/cloud/operators/test_gcs.py rename to providers/tests/google/cloud/operators/test_gcs.py diff --git a/tests/providers/google/cloud/operators/test_kubernetes_engine.py b/providers/tests/google/cloud/operators/test_kubernetes_engine.py similarity index 100% rename from tests/providers/google/cloud/operators/test_kubernetes_engine.py rename to providers/tests/google/cloud/operators/test_kubernetes_engine.py diff --git a/tests/providers/google/cloud/operators/test_life_sciences.py b/providers/tests/google/cloud/operators/test_life_sciences.py similarity index 100% rename from tests/providers/google/cloud/operators/test_life_sciences.py rename to providers/tests/google/cloud/operators/test_life_sciences.py diff --git a/tests/providers/google/cloud/operators/test_looker.py b/providers/tests/google/cloud/operators/test_looker.py similarity index 98% rename from tests/providers/google/cloud/operators/test_looker.py rename to providers/tests/google/cloud/operators/test_looker.py index 5dfea013a2516..b368259c503e0 100644 --- a/tests/providers/google/cloud/operators/test_looker.py +++ b/providers/tests/google/cloud/operators/test_looker.py @@ -25,7 +25,8 @@ from airflow.models import DAG, DagBag from airflow.providers.google.cloud.operators.looker import LookerStartPdtBuildOperator from airflow.utils.timezone import datetime -from tests.test_utils.db import clear_db_runs, clear_db_xcom + +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom OPERATOR_PATH = "airflow.providers.google.cloud.operators.looker.{}" diff --git a/tests/providers/google/cloud/operators/test_mlengine.py b/providers/tests/google/cloud/operators/test_mlengine.py similarity index 100% rename from tests/providers/google/cloud/operators/test_mlengine.py rename to providers/tests/google/cloud/operators/test_mlengine.py diff --git a/tests/providers/google/cloud/operators/test_natural_language.py b/providers/tests/google/cloud/operators/test_natural_language.py similarity index 100% rename from tests/providers/google/cloud/operators/test_natural_language.py rename to providers/tests/google/cloud/operators/test_natural_language.py diff --git a/tests/providers/google/cloud/operators/test_pubsub.py b/providers/tests/google/cloud/operators/test_pubsub.py similarity index 100% rename from tests/providers/google/cloud/operators/test_pubsub.py rename to providers/tests/google/cloud/operators/test_pubsub.py diff --git a/tests/providers/google/cloud/operators/test_spanner.py b/providers/tests/google/cloud/operators/test_spanner.py similarity index 100% rename from tests/providers/google/cloud/operators/test_spanner.py rename to providers/tests/google/cloud/operators/test_spanner.py diff --git a/tests/providers/google/cloud/operators/test_speech_to_text.py b/providers/tests/google/cloud/operators/test_speech_to_text.py similarity index 100% rename from tests/providers/google/cloud/operators/test_speech_to_text.py rename to providers/tests/google/cloud/operators/test_speech_to_text.py diff --git a/tests/providers/google/cloud/operators/test_stackdriver.py b/providers/tests/google/cloud/operators/test_stackdriver.py similarity index 100% rename from tests/providers/google/cloud/operators/test_stackdriver.py rename to providers/tests/google/cloud/operators/test_stackdriver.py diff --git a/tests/providers/google/cloud/operators/test_tasks.py b/providers/tests/google/cloud/operators/test_tasks.py similarity index 100% rename from tests/providers/google/cloud/operators/test_tasks.py rename to providers/tests/google/cloud/operators/test_tasks.py diff --git a/tests/providers/google/cloud/operators/test_text_to_speech.py b/providers/tests/google/cloud/operators/test_text_to_speech.py similarity index 100% rename from tests/providers/google/cloud/operators/test_text_to_speech.py rename to providers/tests/google/cloud/operators/test_text_to_speech.py diff --git a/tests/providers/google/cloud/operators/test_translate.py b/providers/tests/google/cloud/operators/test_translate.py similarity index 100% rename from tests/providers/google/cloud/operators/test_translate.py rename to providers/tests/google/cloud/operators/test_translate.py diff --git a/tests/providers/google/cloud/operators/test_translate_speech.py b/providers/tests/google/cloud/operators/test_translate_speech.py similarity index 100% rename from tests/providers/google/cloud/operators/test_translate_speech.py rename to providers/tests/google/cloud/operators/test_translate_speech.py diff --git a/tests/providers/google/cloud/operators/test_vertex_ai.py b/providers/tests/google/cloud/operators/test_vertex_ai.py similarity index 99% rename from tests/providers/google/cloud/operators/test_vertex_ai.py rename to providers/tests/google/cloud/operators/test_vertex_ai.py index 25deb188aaa4d..f79542006e26b 100644 --- a/tests/providers/google/cloud/operators/test_vertex_ai.py +++ b/providers/tests/google/cloud/operators/test_vertex_ai.py @@ -2178,9 +2178,12 @@ def test_execute_deferrable(self, mock_hook, mock_link_persist): deferrable=True, ) context = {"ti": mock.MagicMock()} - with pytest.raises(TaskDeferred) as exception_info, pytest.warns( - AirflowProviderDeprecationWarning, - match=SYNC_DEPRECATION_WARNING.format("28.08.2024"), + with ( + pytest.raises(TaskDeferred) as exception_info, + pytest.warns( + AirflowProviderDeprecationWarning, + match=SYNC_DEPRECATION_WARNING.format("28.08.2024"), + ), ): op.execute(context=context) @@ -2587,9 +2590,12 @@ def test_deferrable_sync_error(self): parallel_trial_count=3, deferrable=True, ) - with pytest.raises(AirflowException), pytest.warns( - AirflowProviderDeprecationWarning, - match=SYNC_DEPRECATION_WARNING.format("01.09.2024"), + with ( + pytest.raises(AirflowException), + pytest.warns( + AirflowProviderDeprecationWarning, + match=SYNC_DEPRECATION_WARNING.format("01.09.2024"), + ), ): op.execute(context={"ti": mock.MagicMock()}) diff --git a/tests/providers/google/cloud/operators/test_video_intelligence.py b/providers/tests/google/cloud/operators/test_video_intelligence.py similarity index 100% rename from tests/providers/google/cloud/operators/test_video_intelligence.py rename to providers/tests/google/cloud/operators/test_video_intelligence.py diff --git a/tests/providers/google/cloud/operators/test_vision.py b/providers/tests/google/cloud/operators/test_vision.py similarity index 100% rename from tests/providers/google/cloud/operators/test_vision.py rename to providers/tests/google/cloud/operators/test_vision.py diff --git a/tests/providers/google/cloud/operators/test_workflows.py b/providers/tests/google/cloud/operators/test_workflows.py similarity index 100% rename from tests/providers/google/cloud/operators/test_workflows.py rename to providers/tests/google/cloud/operators/test_workflows.py diff --git a/tests/providers/dbt/cloud/sensors/__init__.py b/providers/tests/google/cloud/operators/vertex_ai/__init__.py similarity index 100% rename from tests/providers/dbt/cloud/sensors/__init__.py rename to providers/tests/google/cloud/operators/vertex_ai/__init__.py diff --git a/tests/providers/google/cloud/operators/vertex_ai/test_generative_model.py b/providers/tests/google/cloud/operators/vertex_ai/test_generative_model.py similarity index 100% rename from tests/providers/google/cloud/operators/vertex_ai/test_generative_model.py rename to providers/tests/google/cloud/operators/vertex_ai/test_generative_model.py diff --git a/tests/providers/dbt/cloud/test_data/__init__.py b/providers/tests/google/cloud/secrets/__init__.py similarity index 100% rename from tests/providers/dbt/cloud/test_data/__init__.py rename to providers/tests/google/cloud/secrets/__init__.py diff --git a/tests/providers/google/cloud/secrets/test_secret_manager.py b/providers/tests/google/cloud/secrets/test_secret_manager.py similarity index 100% rename from tests/providers/google/cloud/secrets/test_secret_manager.py rename to providers/tests/google/cloud/secrets/test_secret_manager.py diff --git a/tests/providers/google/cloud/secrets/test_secret_manager_system.py b/providers/tests/google/cloud/secrets/test_secret_manager_system.py similarity index 95% rename from tests/providers/google/cloud/secrets/test_secret_manager_system.py rename to providers/tests/google/cloud/secrets/test_secret_manager_system.py index 6f0e88282dc92..b9b8b6d0cac82 100644 --- a/tests/providers/google/cloud/secrets/test_secret_manager_system.py +++ b/providers/tests/google/cloud/secrets/test_secret_manager_system.py @@ -23,8 +23,8 @@ import pytest -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY -from tests.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context +from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY BACKEND_IMPORT_PATH = "airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend" diff --git a/tests/providers/dbt/cloud/triggers/__init__.py b/providers/tests/google/cloud/sensors/__init__.py similarity index 100% rename from tests/providers/dbt/cloud/triggers/__init__.py rename to providers/tests/google/cloud/sensors/__init__.py diff --git a/tests/providers/google/cloud/sensors/test_bigquery.py b/providers/tests/google/cloud/sensors/test_bigquery.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_bigquery.py rename to providers/tests/google/cloud/sensors/test_bigquery.py diff --git a/tests/providers/google/cloud/sensors/test_bigquery_dts.py b/providers/tests/google/cloud/sensors/test_bigquery_dts.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_bigquery_dts.py rename to providers/tests/google/cloud/sensors/test_bigquery_dts.py diff --git a/tests/providers/google/cloud/sensors/test_bigtable.py b/providers/tests/google/cloud/sensors/test_bigtable.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_bigtable.py rename to providers/tests/google/cloud/sensors/test_bigtable.py diff --git a/tests/providers/google/cloud/sensors/test_cloud_composer.py b/providers/tests/google/cloud/sensors/test_cloud_composer.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_cloud_composer.py rename to providers/tests/google/cloud/sensors/test_cloud_composer.py diff --git a/tests/providers/google/cloud/sensors/test_cloud_storage_transfer_service.py b/providers/tests/google/cloud/sensors/test_cloud_storage_transfer_service.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_cloud_storage_transfer_service.py rename to providers/tests/google/cloud/sensors/test_cloud_storage_transfer_service.py diff --git a/tests/providers/google/cloud/sensors/test_dataflow.py b/providers/tests/google/cloud/sensors/test_dataflow.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_dataflow.py rename to providers/tests/google/cloud/sensors/test_dataflow.py diff --git a/tests/providers/google/cloud/sensors/test_datafusion.py b/providers/tests/google/cloud/sensors/test_datafusion.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_datafusion.py rename to providers/tests/google/cloud/sensors/test_datafusion.py diff --git a/tests/providers/google/cloud/sensors/test_dataplex.py b/providers/tests/google/cloud/sensors/test_dataplex.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_dataplex.py rename to providers/tests/google/cloud/sensors/test_dataplex.py diff --git a/tests/providers/google/cloud/sensors/test_dataprep.py b/providers/tests/google/cloud/sensors/test_dataprep.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_dataprep.py rename to providers/tests/google/cloud/sensors/test_dataprep.py diff --git a/tests/providers/google/cloud/sensors/test_dataproc.py b/providers/tests/google/cloud/sensors/test_dataproc.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_dataproc.py rename to providers/tests/google/cloud/sensors/test_dataproc.py diff --git a/tests/providers/google/cloud/sensors/test_dataproc_metastore.py b/providers/tests/google/cloud/sensors/test_dataproc_metastore.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_dataproc_metastore.py rename to providers/tests/google/cloud/sensors/test_dataproc_metastore.py diff --git a/tests/providers/google/cloud/sensors/test_gcs.py b/providers/tests/google/cloud/sensors/test_gcs.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_gcs.py rename to providers/tests/google/cloud/sensors/test_gcs.py diff --git a/tests/providers/google/cloud/sensors/test_looker.py b/providers/tests/google/cloud/sensors/test_looker.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_looker.py rename to providers/tests/google/cloud/sensors/test_looker.py diff --git a/tests/providers/google/cloud/sensors/test_pubsub.py b/providers/tests/google/cloud/sensors/test_pubsub.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_pubsub.py rename to providers/tests/google/cloud/sensors/test_pubsub.py diff --git a/tests/providers/google/cloud/sensors/test_tasks.py b/providers/tests/google/cloud/sensors/test_tasks.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_tasks.py rename to providers/tests/google/cloud/sensors/test_tasks.py diff --git a/tests/providers/google/cloud/sensors/test_workflows.py b/providers/tests/google/cloud/sensors/test_workflows.py similarity index 100% rename from tests/providers/google/cloud/sensors/test_workflows.py rename to providers/tests/google/cloud/sensors/test_workflows.py diff --git a/tests/providers/dbt/cloud/utils/__init__.py b/providers/tests/google/cloud/transfers/__init__.py similarity index 100% rename from tests/providers/dbt/cloud/utils/__init__.py rename to providers/tests/google/cloud/transfers/__init__.py diff --git a/tests/providers/google/cloud/transfers/test_adls_to_gcs.py b/providers/tests/google/cloud/transfers/test_adls_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_adls_to_gcs.py rename to providers/tests/google/cloud/transfers/test_adls_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_azure_blob_to_gcs.py b/providers/tests/google/cloud/transfers/test_azure_blob_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_azure_blob_to_gcs.py rename to providers/tests/google/cloud/transfers/test_azure_blob_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_azure_fileshare_to_gcs.py b/providers/tests/google/cloud/transfers/test_azure_fileshare_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_azure_fileshare_to_gcs.py rename to providers/tests/google/cloud/transfers/test_azure_fileshare_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery.py b/providers/tests/google/cloud/transfers/test_bigquery_to_bigquery.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_bigquery_to_bigquery.py rename to providers/tests/google/cloud/transfers/test_bigquery_to_bigquery.py diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_gcs.py b/providers/tests/google/cloud/transfers/test_bigquery_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_bigquery_to_gcs.py rename to providers/tests/google/cloud/transfers/test_bigquery_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_mssql.py b/providers/tests/google/cloud/transfers/test_bigquery_to_mssql.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_bigquery_to_mssql.py rename to providers/tests/google/cloud/transfers/test_bigquery_to_mssql.py diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_mysql.py b/providers/tests/google/cloud/transfers/test_bigquery_to_mysql.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_bigquery_to_mysql.py rename to providers/tests/google/cloud/transfers/test_bigquery_to_mysql.py diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_postgres.py b/providers/tests/google/cloud/transfers/test_bigquery_to_postgres.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_bigquery_to_postgres.py rename to providers/tests/google/cloud/transfers/test_bigquery_to_postgres.py diff --git a/tests/providers/google/cloud/transfers/test_calendar_to_gcs.py b/providers/tests/google/cloud/transfers/test_calendar_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_calendar_to_gcs.py rename to providers/tests/google/cloud/transfers/test_calendar_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_cassandra_to_gcs.py b/providers/tests/google/cloud/transfers/test_cassandra_to_gcs.py similarity index 91% rename from tests/providers/google/cloud/transfers/test_cassandra_to_gcs.py rename to providers/tests/google/cloud/transfers/test_cassandra_to_gcs.py index 75e3cbac1789b..caa9d5ea76a1a 100644 --- a/tests/providers/google/cloud/transfers/test_cassandra_to_gcs.py +++ b/providers/tests/google/cloud/transfers/test_cassandra_to_gcs.py @@ -46,13 +46,17 @@ def test_execute(self): except cassandra.DependencyException: pytest.skip("cassandra-driver not installed with libev support. Skipping test.") - with mock.patch( - "airflow.providers.google.cloud.transfers.cassandra_to_gcs.NamedTemporaryFile" - ) as mock_tempfile, mock.patch( - "airflow.providers.google.cloud.transfers.cassandra_to_gcs.GCSHook.upload" - ) as mock_upload, mock.patch( - "airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraHook" - ) as mock_hook: + with ( + mock.patch( + "airflow.providers.google.cloud.transfers.cassandra_to_gcs.NamedTemporaryFile" + ) as mock_tempfile, + mock.patch( + "airflow.providers.google.cloud.transfers.cassandra_to_gcs.GCSHook.upload" + ) as mock_upload, + mock.patch( + "airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraHook" + ) as mock_hook, + ): mock_tempfile.return_value.name = TMP_FILE_NAME operator = CassandraToGCSOperator( task_id=TASK_ID, diff --git a/tests/providers/google/cloud/transfers/test_facebook_ads_to_gcs.py b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_facebook_ads_to_gcs.py rename to providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_facebook_ads_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py similarity index 93% rename from tests/providers/google/cloud/transfers/test_facebook_ads_to_gcs_system.py rename to providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py index 85c6f69ae59ea..ba24a0c34da25 100644 --- a/tests/providers/google/cloud/transfers/test_facebook_ads_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py @@ -26,8 +26,13 @@ from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.utils.process_utils import patch_environ -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY -from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context + +from dev.tests_common.test_utils.gcp_system_helpers import ( + CLOUD_DAG_FOLDER, + GoogleSystemTest, + provide_gcp_context, +) +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") FACEBOOK_KEY = "facebook.json" diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py b/providers/tests/google/cloud/transfers/test_gcs_to_bigquery.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py rename to providers/tests/google/cloud/transfers/test_gcs_to_bigquery.py diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py b/providers/tests/google/cloud/transfers/test_gcs_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_gcs_to_gcs.py rename to providers/tests/google/cloud/transfers/test_gcs_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_local.py b/providers/tests/google/cloud/transfers/test_gcs_to_local.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_gcs_to_local.py rename to providers/tests/google/cloud/transfers/test_gcs_to_local.py diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_sftp.py b/providers/tests/google/cloud/transfers/test_gcs_to_sftp.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_gcs_to_sftp.py rename to providers/tests/google/cloud/transfers/test_gcs_to_sftp.py diff --git a/tests/providers/google/cloud/transfers/test_gdrive_to_gcs.py b/providers/tests/google/cloud/transfers/test_gdrive_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_gdrive_to_gcs.py rename to providers/tests/google/cloud/transfers/test_gdrive_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_gdrive_to_local.py b/providers/tests/google/cloud/transfers/test_gdrive_to_local.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_gdrive_to_local.py rename to providers/tests/google/cloud/transfers/test_gdrive_to_local.py diff --git a/tests/providers/google/cloud/transfers/test_local_to_gcs.py b/providers/tests/google/cloud/transfers/test_local_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_local_to_gcs.py rename to providers/tests/google/cloud/transfers/test_local_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_mssql_to_gcs.py b/providers/tests/google/cloud/transfers/test_mssql_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_mssql_to_gcs.py rename to providers/tests/google/cloud/transfers/test_mssql_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_mysql_to_gcs.py b/providers/tests/google/cloud/transfers/test_mysql_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_mysql_to_gcs.py rename to providers/tests/google/cloud/transfers/test_mysql_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_oracle_to_gcs.py b/providers/tests/google/cloud/transfers/test_oracle_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_oracle_to_gcs.py rename to providers/tests/google/cloud/transfers/test_oracle_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_postgres_to_gcs.py b/providers/tests/google/cloud/transfers/test_postgres_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_postgres_to_gcs.py rename to providers/tests/google/cloud/transfers/test_postgres_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_s3_to_gcs.py b/providers/tests/google/cloud/transfers/test_s3_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_s3_to_gcs.py rename to providers/tests/google/cloud/transfers/test_s3_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_salesforce_to_gcs.py b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_salesforce_to_gcs.py rename to providers/tests/google/cloud/transfers/test_salesforce_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_salesforce_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py similarity index 84% rename from tests/providers/google/cloud/transfers/test_salesforce_to_gcs_system.py rename to providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py index c2eee9bd09110..afd0856fad244 100644 --- a/tests/providers/google/cloud/transfers/test_salesforce_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py @@ -20,9 +20,13 @@ import pytest -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY -from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context -from tests.test_utils.salesforce_system_helpers import provide_salesforce_connection +from dev.tests_common.test_utils.gcp_system_helpers import ( + CLOUD_DAG_FOLDER, + GoogleSystemTest, + provide_gcp_context, +) +from dev.tests_common.test_utils.salesforce_system_helpers import provide_salesforce_connection +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") SALESFORCE_KEY = "salesforce.json" diff --git a/tests/providers/google/cloud/transfers/test_sftp_to_gcs.py b/providers/tests/google/cloud/transfers/test_sftp_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_sftp_to_gcs.py rename to providers/tests/google/cloud/transfers/test_sftp_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_sheets_to_gcs.py b/providers/tests/google/cloud/transfers/test_sheets_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_sheets_to_gcs.py rename to providers/tests/google/cloud/transfers/test_sheets_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_sql_to_gcs.py b/providers/tests/google/cloud/transfers/test_sql_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_sql_to_gcs.py rename to providers/tests/google/cloud/transfers/test_sql_to_gcs.py diff --git a/tests/providers/google/cloud/transfers/test_trino_to_gcs.py b/providers/tests/google/cloud/transfers/test_trino_to_gcs.py similarity index 100% rename from tests/providers/google/cloud/transfers/test_trino_to_gcs.py rename to providers/tests/google/cloud/transfers/test_trino_to_gcs.py diff --git a/tests/providers/discord/notifications/__init__.py b/providers/tests/google/cloud/triggers/__init__.py similarity index 100% rename from tests/providers/discord/notifications/__init__.py rename to providers/tests/google/cloud/triggers/__init__.py diff --git a/tests/providers/google/cloud/triggers/test_bigquery.py b/providers/tests/google/cloud/triggers/test_bigquery.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_bigquery.py rename to providers/tests/google/cloud/triggers/test_bigquery.py diff --git a/tests/providers/google/cloud/triggers/test_bigquery_dts.py b/providers/tests/google/cloud/triggers/test_bigquery_dts.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_bigquery_dts.py rename to providers/tests/google/cloud/triggers/test_bigquery_dts.py diff --git a/tests/providers/google/cloud/triggers/test_cloud_batch.py b/providers/tests/google/cloud/triggers/test_cloud_batch.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_cloud_batch.py rename to providers/tests/google/cloud/triggers/test_cloud_batch.py diff --git a/tests/providers/google/cloud/triggers/test_cloud_build.py b/providers/tests/google/cloud/triggers/test_cloud_build.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_cloud_build.py rename to providers/tests/google/cloud/triggers/test_cloud_build.py diff --git a/tests/providers/google/cloud/triggers/test_cloud_composer.py b/providers/tests/google/cloud/triggers/test_cloud_composer.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_cloud_composer.py rename to providers/tests/google/cloud/triggers/test_cloud_composer.py diff --git a/tests/providers/google/cloud/triggers/test_cloud_run.py b/providers/tests/google/cloud/triggers/test_cloud_run.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_cloud_run.py rename to providers/tests/google/cloud/triggers/test_cloud_run.py diff --git a/tests/providers/google/cloud/triggers/test_cloud_sql.py b/providers/tests/google/cloud/triggers/test_cloud_sql.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_cloud_sql.py rename to providers/tests/google/cloud/triggers/test_cloud_sql.py diff --git a/tests/providers/google/cloud/triggers/test_cloud_storage_transfer_service.py b/providers/tests/google/cloud/triggers/test_cloud_storage_transfer_service.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_cloud_storage_transfer_service.py rename to providers/tests/google/cloud/triggers/test_cloud_storage_transfer_service.py diff --git a/tests/providers/google/cloud/triggers/test_dataflow.py b/providers/tests/google/cloud/triggers/test_dataflow.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_dataflow.py rename to providers/tests/google/cloud/triggers/test_dataflow.py diff --git a/tests/providers/google/cloud/triggers/test_datafusion.py b/providers/tests/google/cloud/triggers/test_datafusion.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_datafusion.py rename to providers/tests/google/cloud/triggers/test_datafusion.py diff --git a/tests/providers/google/cloud/triggers/test_dataplex.py b/providers/tests/google/cloud/triggers/test_dataplex.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_dataplex.py rename to providers/tests/google/cloud/triggers/test_dataplex.py diff --git a/tests/providers/google/cloud/triggers/test_dataproc.py b/providers/tests/google/cloud/triggers/test_dataproc.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_dataproc.py rename to providers/tests/google/cloud/triggers/test_dataproc.py diff --git a/tests/providers/google/cloud/triggers/test_gcs.py b/providers/tests/google/cloud/triggers/test_gcs.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_gcs.py rename to providers/tests/google/cloud/triggers/test_gcs.py diff --git a/tests/providers/google/cloud/triggers/test_kubernetes_engine.py b/providers/tests/google/cloud/triggers/test_kubernetes_engine.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_kubernetes_engine.py rename to providers/tests/google/cloud/triggers/test_kubernetes_engine.py diff --git a/tests/providers/google/cloud/triggers/test_mlengine.py b/providers/tests/google/cloud/triggers/test_mlengine.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_mlengine.py rename to providers/tests/google/cloud/triggers/test_mlengine.py diff --git a/tests/providers/google/cloud/triggers/test_pubsub.py b/providers/tests/google/cloud/triggers/test_pubsub.py similarity index 100% rename from tests/providers/google/cloud/triggers/test_pubsub.py rename to providers/tests/google/cloud/triggers/test_pubsub.py diff --git a/tests/providers/google/cloud/triggers/test_vertex_ai.py b/providers/tests/google/cloud/triggers/test_vertex_ai.py similarity index 99% rename from tests/providers/google/cloud/triggers/test_vertex_ai.py rename to providers/tests/google/cloud/triggers/test_vertex_ai.py index 946ebf49487fb..e2e588a2bfdd9 100644 --- a/tests/providers/google/cloud/triggers/test_vertex_ai.py +++ b/providers/tests/google/cloud/triggers/test_vertex_ai.py @@ -46,7 +46,8 @@ RunPipelineJobTrigger, ) from airflow.triggers.base import TriggerEvent -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id TEST_CONN_ID = "test_connection" TEST_PROJECT_ID = "test_propject_id" diff --git a/tests/providers/ftp/hooks/__init__.py b/providers/tests/google/cloud/utils/__init__.py similarity index 100% rename from tests/providers/ftp/hooks/__init__.py rename to providers/tests/google/cloud/utils/__init__.py diff --git a/tests/providers/google/cloud/utils/airflow_util.py b/providers/tests/google/cloud/utils/airflow_util.py similarity index 100% rename from tests/providers/google/cloud/utils/airflow_util.py rename to providers/tests/google/cloud/utils/airflow_util.py diff --git a/tests/providers/google/cloud/utils/base_gcp_mock.py b/providers/tests/google/cloud/utils/base_gcp_mock.py similarity index 100% rename from tests/providers/google/cloud/utils/base_gcp_mock.py rename to providers/tests/google/cloud/utils/base_gcp_mock.py diff --git a/tests/providers/google/cloud/utils/gcp_authenticator.py b/providers/tests/google/cloud/utils/gcp_authenticator.py similarity index 98% rename from tests/providers/google/cloud/utils/gcp_authenticator.py rename to providers/tests/google/cloud/utils/gcp_authenticator.py index 7c95e57dc328f..6bb11b260644c 100644 --- a/tests/providers/google/cloud/utils/gcp_authenticator.py +++ b/providers/tests/google/cloud/utils/gcp_authenticator.py @@ -25,9 +25,10 @@ from airflow.exceptions import AirflowException from airflow.models import Connection +from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from dev.tests_common.test_utils.logging_command_executor import CommandExecutor + # Please keep these variables in alphabetical order. -from tests.test_utils import AIRFLOW_MAIN_FOLDER -from tests.test_utils.logging_command_executor import CommandExecutor GCP_AI_KEY = "gcp_ai.json" GCP_BIGQUERY_KEY = "gcp_bigquery.json" diff --git a/tests/providers/google/cloud/utils/job_details.json b/providers/tests/google/cloud/utils/job_details.json similarity index 100% rename from tests/providers/google/cloud/utils/job_details.json rename to providers/tests/google/cloud/utils/job_details.json diff --git a/tests/providers/google/cloud/utils/out_table_details.json b/providers/tests/google/cloud/utils/out_table_details.json similarity index 100% rename from tests/providers/google/cloud/utils/out_table_details.json rename to providers/tests/google/cloud/utils/out_table_details.json diff --git a/tests/providers/google/cloud/utils/script_job_details.json b/providers/tests/google/cloud/utils/script_job_details.json similarity index 100% rename from tests/providers/google/cloud/utils/script_job_details.json rename to providers/tests/google/cloud/utils/script_job_details.json diff --git a/tests/providers/google/cloud/utils/table_details.json b/providers/tests/google/cloud/utils/table_details.json similarity index 100% rename from tests/providers/google/cloud/utils/table_details.json rename to providers/tests/google/cloud/utils/table_details.json diff --git a/tests/providers/google/cloud/utils/test_credentials_provider.py b/providers/tests/google/cloud/utils/test_credentials_provider.py similarity index 98% rename from tests/providers/google/cloud/utils/test_credentials_provider.py rename to providers/tests/google/cloud/utils/test_credentials_provider.py index 3cab06cee8a46..2e6873775745a 100644 --- a/tests/providers/google/cloud/utils/test_credentials_provider.py +++ b/providers/tests/google/cloud/utils/test_credentials_provider.py @@ -319,9 +319,10 @@ def test_get_credentials_and_project_id_with_service_account_info( @mock.patch("google.auth.load_credentials_from_file", return_value=("CREDENTIALS", "PROJECT_ID")) def test_get_credentials_using_credential_config_file(self, mock_load_credentials_from_file, caplog): - with caplog.at_level( - level=logging.DEBUG, logger=CRED_PROVIDER_LOGGER_NAME - ), NamedTemporaryFile() as temp_file: + with ( + caplog.at_level(level=logging.DEBUG, logger=CRED_PROVIDER_LOGGER_NAME), + NamedTemporaryFile() as temp_file, + ): caplog.clear() result = get_credentials_and_project_id(credential_config_file=temp_file.name) mock_load_credentials_from_file.assert_called_once_with(temp_file.name, scopes=None) @@ -350,8 +351,9 @@ def test_get_credentials_using_credential_config_string(self, mock_load_credenti def test_get_credentials_using_credential_config_invalid_string(self, caplog): caplog.clear() - with pytest.raises(DefaultCredentialsError), caplog.at_level( - level=logging.DEBUG, logger=CRED_PROVIDER_LOGGER_NAME + with ( + pytest.raises(DefaultCredentialsError), + caplog.at_level(level=logging.DEBUG, logger=CRED_PROVIDER_LOGGER_NAME), ): get_credentials_and_project_id(credential_config_file="invalid json}}}}") assert "Getting connection using credential configuration string." in caplog.messages diff --git a/tests/providers/google/cloud/utils/test_datafusion.py b/providers/tests/google/cloud/utils/test_datafusion.py similarity index 100% rename from tests/providers/google/cloud/utils/test_datafusion.py rename to providers/tests/google/cloud/utils/test_datafusion.py diff --git a/tests/providers/google/cloud/utils/test_dataproc.py b/providers/tests/google/cloud/utils/test_dataproc.py similarity index 100% rename from tests/providers/google/cloud/utils/test_dataproc.py rename to providers/tests/google/cloud/utils/test_dataproc.py diff --git a/tests/providers/google/cloud/utils/test_external_token_supplier.py b/providers/tests/google/cloud/utils/test_external_token_supplier.py similarity index 100% rename from tests/providers/google/cloud/utils/test_external_token_supplier.py rename to providers/tests/google/cloud/utils/test_external_token_supplier.py diff --git a/tests/providers/google/cloud/utils/test_field_sanitizer.py b/providers/tests/google/cloud/utils/test_field_sanitizer.py similarity index 100% rename from tests/providers/google/cloud/utils/test_field_sanitizer.py rename to providers/tests/google/cloud/utils/test_field_sanitizer.py diff --git a/tests/providers/google/cloud/utils/test_field_validator.py b/providers/tests/google/cloud/utils/test_field_validator.py similarity index 100% rename from tests/providers/google/cloud/utils/test_field_validator.py rename to providers/tests/google/cloud/utils/test_field_validator.py diff --git a/tests/providers/google/cloud/utils/test_helpers.py b/providers/tests/google/cloud/utils/test_helpers.py similarity index 100% rename from tests/providers/google/cloud/utils/test_helpers.py rename to providers/tests/google/cloud/utils/test_helpers.py diff --git a/tests/providers/google/cloud/utils/test_mlengine_operator_utils.py b/providers/tests/google/cloud/utils/test_mlengine_operator_utils.py similarity index 100% rename from tests/providers/google/cloud/utils/test_mlengine_operator_utils.py rename to providers/tests/google/cloud/utils/test_mlengine_operator_utils.py diff --git a/tests/providers/google/cloud/utils/test_mlengine_prediction_summary.py b/providers/tests/google/cloud/utils/test_mlengine_prediction_summary.py similarity index 91% rename from tests/providers/google/cloud/utils/test_mlengine_prediction_summary.py rename to providers/tests/google/cloud/utils/test_mlengine_prediction_summary.py index 7d28cf1e68413..984c4847d9424 100644 --- a/tests/providers/google/cloud/utils/test_mlengine_prediction_summary.py +++ b/providers/tests/google/cloud/utils/test_mlengine_prediction_summary.py @@ -87,13 +87,11 @@ def test_run_should_fail_if_enc_fn_is_not_callable(self): ) def test_run_should_not_fail_with_valid_fn(self): - with mock.patch.object( - mlengine_prediction_summary.beam.pipeline, "PipelineOptions" - ) as pipeline_mock, mock.patch.object( - mlengine_prediction_summary.beam, "Pipeline" - ) as pipeline_obj_mock, mock.patch.object( - mlengine_prediction_summary.beam.io, "ReadFromText" - ) as io_mock: + with ( + mock.patch.object(mlengine_prediction_summary.beam.pipeline, "PipelineOptions") as pipeline_mock, + mock.patch.object(mlengine_prediction_summary.beam, "Pipeline") as pipeline_obj_mock, + mock.patch.object(mlengine_prediction_summary.beam.io, "ReadFromText") as io_mock, + ): def metric_function(): return 1 diff --git a/tests/providers/docker/decorators/__init__.py b/providers/tests/google/common/__init__.py similarity index 100% rename from tests/providers/docker/decorators/__init__.py rename to providers/tests/google/common/__init__.py diff --git a/tests/providers/elasticsearch/log/__init__.py b/providers/tests/google/common/auth_backend/__init__.py similarity index 100% rename from tests/providers/elasticsearch/log/__init__.py rename to providers/tests/google/common/auth_backend/__init__.py diff --git a/tests/providers/google/common/auth_backend/test_google_openid.py b/providers/tests/google/common/auth_backend/test_google_openid.py similarity index 95% rename from tests/providers/google/common/auth_backend/test_google_openid.py rename to providers/tests/google/common/auth_backend/test_google_openid.py index 260ae0d6fb5e1..67b0ff2003d22 100644 --- a/tests/providers/google/common/auth_backend/test_google_openid.py +++ b/providers/tests/google/common/auth_backend/test_google_openid.py @@ -22,10 +22,11 @@ from google.auth.exceptions import GoogleAuthError from airflow.www.app import create_app -from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_pools -from tests.test_utils.decorators import dont_initialize_flask_app_submodules + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_pools +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules @pytest.fixture(scope="module") diff --git a/tests/providers/fab/auth_manager/api/auth/__init__.py b/providers/tests/google/common/hooks/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/api/auth/__init__.py rename to providers/tests/google/common/hooks/__init__.py diff --git a/tests/providers/google/common/hooks/test_base_google.py b/providers/tests/google/common/hooks/test_base_google.py similarity index 99% rename from tests/providers/google/common/hooks/test_base_google.py rename to providers/tests/google/common/hooks/test_base_google.py index 2b67b8844ecbc..a4317c7e8b9ef 100644 --- a/tests/providers/google/common/hooks/test_base_google.py +++ b/providers/tests/google/common/hooks/test_base_google.py @@ -38,7 +38,8 @@ from airflow.providers.google.cloud.utils.credentials_provider import _DEFAULT_SCOPES from airflow.providers.google.common.hooks import base_google as hook from airflow.providers.google.common.hooks.base_google import GoogleBaseHook, is_refresh_credentials_exception -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id default_creds_available = True default_project = None diff --git a/tests/providers/google/common/hooks/test_discovery_api.py b/providers/tests/google/common/hooks/test_discovery_api.py similarity index 100% rename from tests/providers/google/common/hooks/test_discovery_api.py rename to providers/tests/google/common/hooks/test_discovery_api.py diff --git a/tests/providers/google/common/test_deprecated.py b/providers/tests/google/common/test_deprecated.py similarity index 100% rename from tests/providers/google/common/test_deprecated.py rename to providers/tests/google/common/test_deprecated.py diff --git a/tests/providers/fab/auth_manager/api/auth/backend/__init__.py b/providers/tests/google/common/utils/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/api/auth/backend/__init__.py rename to providers/tests/google/common/utils/__init__.py diff --git a/tests/providers/google/common/utils/test_id_token_credentials.py b/providers/tests/google/common/utils/test_id_token_credentials.py similarity index 100% rename from tests/providers/google/common/utils/test_id_token_credentials.py rename to providers/tests/google/common/utils/test_id_token_credentials.py diff --git a/tests/providers/fab/auth_manager/api_endpoints/__init__.py b/providers/tests/google/firebase/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/api_endpoints/__init__.py rename to providers/tests/google/firebase/__init__.py diff --git a/tests/providers/fab/auth_manager/cli_commands/__init__.py b/providers/tests/google/firebase/hooks/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/cli_commands/__init__.py rename to providers/tests/google/firebase/hooks/__init__.py diff --git a/tests/providers/google/firebase/hooks/test_firestore.py b/providers/tests/google/firebase/hooks/test_firestore.py similarity index 99% rename from tests/providers/google/firebase/hooks/test_firestore.py rename to providers/tests/google/firebase/hooks/test_firestore.py index e6c17105a60fc..b7308c5f9b9ce 100644 --- a/tests/providers/google/firebase/hooks/test_firestore.py +++ b/providers/tests/google/firebase/hooks/test_firestore.py @@ -27,7 +27,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.firebase.hooks.firestore import CloudFirestoreHook -from tests.providers.google.cloud.utils.base_gcp_mock import ( + +from providers.tests.google.cloud.utils.base_gcp_mock import ( GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id, mock_base_gcp_hook_no_default_project_id, diff --git a/tests/providers/fab/auth_manager/decorators/__init__.py b/providers/tests/google/firebase/operators/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/decorators/__init__.py rename to providers/tests/google/firebase/operators/__init__.py diff --git a/tests/providers/google/firebase/operators/test_firestore.py b/providers/tests/google/firebase/operators/test_firestore.py similarity index 100% rename from tests/providers/google/firebase/operators/test_firestore.py rename to providers/tests/google/firebase/operators/test_firestore.py diff --git a/tests/providers/ftp/sensors/__init__.py b/providers/tests/google/leveldb/__init__.py similarity index 100% rename from tests/providers/ftp/sensors/__init__.py rename to providers/tests/google/leveldb/__init__.py diff --git a/tests/providers/google/cloud/utils/__init__.py b/providers/tests/google/leveldb/hooks/__init__.py similarity index 100% rename from tests/providers/google/cloud/utils/__init__.py rename to providers/tests/google/leveldb/hooks/__init__.py diff --git a/tests/providers/google/leveldb/hooks/test_leveldb.py b/providers/tests/google/leveldb/hooks/test_leveldb.py similarity index 100% rename from tests/providers/google/leveldb/hooks/test_leveldb.py rename to providers/tests/google/leveldb/hooks/test_leveldb.py diff --git a/tests/providers/google/leveldb/__init__.py b/providers/tests/google/leveldb/operators/__init__.py similarity index 100% rename from tests/providers/google/leveldb/__init__.py rename to providers/tests/google/leveldb/operators/__init__.py diff --git a/tests/providers/google/leveldb/operators/test_leveldb.py b/providers/tests/google/leveldb/operators/test_leveldb.py similarity index 100% rename from tests/providers/google/leveldb/operators/test_leveldb.py rename to providers/tests/google/leveldb/operators/test_leveldb.py diff --git a/tests/providers/fab/auth_manager/security_manager/__init__.py b/providers/tests/google/marketing_platform/__init__.py similarity index 100% rename from tests/providers/fab/auth_manager/security_manager/__init__.py rename to providers/tests/google/marketing_platform/__init__.py diff --git a/tests/providers/facebook/__init__.py b/providers/tests/google/marketing_platform/hooks/__init__.py similarity index 100% rename from tests/providers/facebook/__init__.py rename to providers/tests/google/marketing_platform/hooks/__init__.py diff --git a/tests/providers/google/marketing_platform/hooks/test_analytics.py b/providers/tests/google/marketing_platform/hooks/test_analytics.py similarity index 96% rename from tests/providers/google/marketing_platform/hooks/test_analytics.py rename to providers/tests/google/marketing_platform/hooks/test_analytics.py index e88e8fe862a35..9944e2a9947ac 100644 --- a/tests/providers/google/marketing_platform/hooks/test_analytics.py +++ b/providers/tests/google/marketing_platform/hooks/test_analytics.py @@ -24,7 +24,8 @@ from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.google.marketing_platform.hooks.analytics import GoogleAnalyticsHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id WEB_PROPERTY_AD_WORDS_LINK_ID = "AAIIRRFFLLOOWW" WEB_PROPERTY_ID = "web_property_id" @@ -38,10 +39,13 @@ class TestGoogleAnalyticsHook: def setup_method(self): - with mock.patch( - "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", - new=mock_base_gcp_hook_default_project_id, - ), warnings.catch_warnings(): + with ( + mock.patch( + "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", + new=mock_base_gcp_hook_default_project_id, + ), + warnings.catch_warnings(), + ): warnings.simplefilter("ignore", AirflowProviderDeprecationWarning) self.hook = GoogleAnalyticsHook(API_VERSION, GCP_CONN_ID) diff --git a/tests/providers/google/marketing_platform/hooks/test_analytics_admin.py b/providers/tests/google/marketing_platform/hooks/test_analytics_admin.py similarity index 99% rename from tests/providers/google/marketing_platform/hooks/test_analytics_admin.py rename to providers/tests/google/marketing_platform/hooks/test_analytics_admin.py index 81a5210d499ab..9887dac4db240 100644 --- a/tests/providers/google/marketing_platform/hooks/test_analytics_admin.py +++ b/providers/tests/google/marketing_platform/hooks/test_analytics_admin.py @@ -20,7 +20,8 @@ from unittest import mock from airflow.providers.google.marketing_platform.hooks.analytics_admin import GoogleAnalyticsAdminHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id GCP_CONN_ID = "test_gcp_conn_id" IMPERSONATION_CHAIN = ["ACCOUNT_1", "ACCOUNT_2", "ACCOUNT_3"] diff --git a/tests/providers/google/marketing_platform/hooks/test_campaign_manager.py b/providers/tests/google/marketing_platform/hooks/test_campaign_manager.py similarity index 99% rename from tests/providers/google/marketing_platform/hooks/test_campaign_manager.py rename to providers/tests/google/marketing_platform/hooks/test_campaign_manager.py index 2e7cd821b3c25..0c16e1979592b 100644 --- a/tests/providers/google/marketing_platform/hooks/test_campaign_manager.py +++ b/providers/tests/google/marketing_platform/hooks/test_campaign_manager.py @@ -20,7 +20,8 @@ from unittest import mock from airflow.providers.google.marketing_platform.hooks.campaign_manager import GoogleCampaignManagerHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id API_VERSION = "v4" GCP_CONN_ID = "google_cloud_default" diff --git a/tests/providers/google/marketing_platform/hooks/test_display_video.py b/providers/tests/google/marketing_platform/hooks/test_display_video.py similarity index 99% rename from tests/providers/google/marketing_platform/hooks/test_display_video.py rename to providers/tests/google/marketing_platform/hooks/test_display_video.py index c6eae2aa8f9b1..2c476428e89f9 100644 --- a/tests/providers/google/marketing_platform/hooks/test_display_video.py +++ b/providers/tests/google/marketing_platform/hooks/test_display_video.py @@ -20,7 +20,8 @@ from unittest import mock from airflow.providers.google.marketing_platform.hooks.display_video import GoogleDisplayVideo360Hook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id API_VERSION = "v2" GCP_CONN_ID = "google_cloud_default" diff --git a/tests/providers/google/marketing_platform/hooks/test_search_ads.py b/providers/tests/google/marketing_platform/hooks/test_search_ads.py similarity index 99% rename from tests/providers/google/marketing_platform/hooks/test_search_ads.py rename to providers/tests/google/marketing_platform/hooks/test_search_ads.py index 1fc08237f1458..c3b585364cea5 100644 --- a/tests/providers/google/marketing_platform/hooks/test_search_ads.py +++ b/providers/tests/google/marketing_platform/hooks/test_search_ads.py @@ -25,7 +25,8 @@ GoogleSearchAdsHook, GoogleSearchAdsReportingHook, ) -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id GCP_CONN_ID = "google_cloud_default" API_VERSION = "v0" diff --git a/tests/providers/google/leveldb/hooks/__init__.py b/providers/tests/google/marketing_platform/links/__init__.py similarity index 100% rename from tests/providers/google/leveldb/hooks/__init__.py rename to providers/tests/google/marketing_platform/links/__init__.py diff --git a/tests/providers/google/marketing_platform/links/test_analytics_admin.py b/providers/tests/google/marketing_platform/links/test_analytics_admin.py similarity index 100% rename from tests/providers/google/marketing_platform/links/test_analytics_admin.py rename to providers/tests/google/marketing_platform/links/test_analytics_admin.py diff --git a/tests/providers/facebook/ads/__init__.py b/providers/tests/google/marketing_platform/operators/__init__.py similarity index 100% rename from tests/providers/facebook/ads/__init__.py rename to providers/tests/google/marketing_platform/operators/__init__.py diff --git a/tests/providers/google/marketing_platform/operators/test_analytics.py b/providers/tests/google/marketing_platform/operators/test_analytics.py similarity index 100% rename from tests/providers/google/marketing_platform/operators/test_analytics.py rename to providers/tests/google/marketing_platform/operators/test_analytics.py diff --git a/tests/providers/google/marketing_platform/operators/test_analytics_admin.py b/providers/tests/google/marketing_platform/operators/test_analytics_admin.py similarity index 100% rename from tests/providers/google/marketing_platform/operators/test_analytics_admin.py rename to providers/tests/google/marketing_platform/operators/test_analytics_admin.py diff --git a/tests/providers/google/marketing_platform/operators/test_campaign_manager.py b/providers/tests/google/marketing_platform/operators/test_campaign_manager.py similarity index 100% rename from tests/providers/google/marketing_platform/operators/test_campaign_manager.py rename to providers/tests/google/marketing_platform/operators/test_campaign_manager.py diff --git a/tests/providers/google/marketing_platform/operators/test_display_video.py b/providers/tests/google/marketing_platform/operators/test_display_video.py similarity index 100% rename from tests/providers/google/marketing_platform/operators/test_display_video.py rename to providers/tests/google/marketing_platform/operators/test_display_video.py diff --git a/tests/providers/google/marketing_platform/operators/test_display_video_system.py b/providers/tests/google/marketing_platform/operators/test_display_video_system.py similarity index 91% rename from tests/providers/google/marketing_platform/operators/test_display_video_system.py rename to providers/tests/google/marketing_platform/operators/test_display_video_system.py index 10422e8ccb912..78f5d4ee021f9 100644 --- a/tests/providers/google/marketing_platform/operators/test_display_video_system.py +++ b/providers/tests/google/marketing_platform/operators/test_display_video_system.py @@ -20,8 +20,13 @@ from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.providers.google.marketing_platform.example_dags.example_display_video import BUCKET -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GMP_KEY -from tests.test_utils.gcp_system_helpers import MARKETING_DAG_FOLDER, GoogleSystemTest, provide_gcp_context + +from dev.tests_common.test_utils.gcp_system_helpers import ( + MARKETING_DAG_FOLDER, + GoogleSystemTest, + provide_gcp_context, +) +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GMP_KEY # Requires the following scope: SCOPES = [ diff --git a/tests/providers/google/marketing_platform/operators/test_search_ads.py b/providers/tests/google/marketing_platform/operators/test_search_ads.py similarity index 100% rename from tests/providers/google/marketing_platform/operators/test_search_ads.py rename to providers/tests/google/marketing_platform/operators/test_search_ads.py diff --git a/tests/providers/facebook/ads/hooks/__init__.py b/providers/tests/google/marketing_platform/sensors/__init__.py similarity index 100% rename from tests/providers/facebook/ads/hooks/__init__.py rename to providers/tests/google/marketing_platform/sensors/__init__.py diff --git a/tests/providers/google/marketing_platform/sensors/test_campaign_manager.py b/providers/tests/google/marketing_platform/sensors/test_campaign_manager.py similarity index 100% rename from tests/providers/google/marketing_platform/sensors/test_campaign_manager.py rename to providers/tests/google/marketing_platform/sensors/test_campaign_manager.py diff --git a/tests/providers/google/marketing_platform/sensors/test_display_video.py b/providers/tests/google/marketing_platform/sensors/test_display_video.py similarity index 100% rename from tests/providers/google/marketing_platform/sensors/test_display_video.py rename to providers/tests/google/marketing_platform/sensors/test_display_video.py diff --git a/tests/providers/google/leveldb/operators/__init__.py b/providers/tests/google/suite/__init__.py similarity index 100% rename from tests/providers/google/leveldb/operators/__init__.py rename to providers/tests/google/suite/__init__.py diff --git a/tests/providers/google/marketing_platform/links/__init__.py b/providers/tests/google/suite/hooks/__init__.py similarity index 100% rename from tests/providers/google/marketing_platform/links/__init__.py rename to providers/tests/google/suite/hooks/__init__.py diff --git a/tests/providers/google/suite/hooks/test_calendar.py b/providers/tests/google/suite/hooks/test_calendar.py similarity index 98% rename from tests/providers/google/suite/hooks/test_calendar.py rename to providers/tests/google/suite/hooks/test_calendar.py index e072043f6ea5d..dfcf3e10626f3 100644 --- a/tests/providers/google/suite/hooks/test_calendar.py +++ b/providers/tests/google/suite/hooks/test_calendar.py @@ -24,7 +24,8 @@ from unittest import mock from airflow.providers.google.suite.hooks.calendar import GoogleCalendarHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id API_VERSION = "api_version" GCP_CONN_ID = "test" diff --git a/tests/providers/google/suite/hooks/test_drive.py b/providers/tests/google/suite/hooks/test_drive.py similarity index 99% rename from tests/providers/google/suite/hooks/test_drive.py rename to providers/tests/google/suite/hooks/test_drive.py index 00423f263401c..a6fb0391d2c8c 100644 --- a/tests/providers/google/suite/hooks/test_drive.py +++ b/providers/tests/google/suite/hooks/test_drive.py @@ -22,7 +22,8 @@ import pytest from airflow.providers.google.suite.hooks.drive import GoogleDriveHook -from tests.providers.google.cloud.utils.base_gcp_mock import GCP_CONNECTION_WITH_PROJECT_ID + +from providers.tests.google.cloud.utils.base_gcp_mock import GCP_CONNECTION_WITH_PROJECT_ID @pytest.mark.db_test diff --git a/tests/providers/google/suite/hooks/test_sheets.py b/providers/tests/google/suite/hooks/test_sheets.py similarity index 99% rename from tests/providers/google/suite/hooks/test_sheets.py rename to providers/tests/google/suite/hooks/test_sheets.py index ca3159d7d1c93..1be75426a336a 100644 --- a/tests/providers/google/suite/hooks/test_sheets.py +++ b/providers/tests/google/suite/hooks/test_sheets.py @@ -27,7 +27,8 @@ from airflow.exceptions import AirflowException from airflow.providers.google.suite.hooks.sheets import GSheetsHook -from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id + +from providers.tests.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id GCP_CONN_ID = "test" SPREADSHEET_ID = "1234567890" diff --git a/tests/providers/google/suite/__init__.py b/providers/tests/google/suite/operators/__init__.py similarity index 100% rename from tests/providers/google/suite/__init__.py rename to providers/tests/google/suite/operators/__init__.py diff --git a/tests/providers/google/suite/operators/test_sheets.py b/providers/tests/google/suite/operators/test_sheets.py similarity index 100% rename from tests/providers/google/suite/operators/test_sheets.py rename to providers/tests/google/suite/operators/test_sheets.py diff --git a/tests/providers/google/suite/hooks/__init__.py b/providers/tests/google/suite/sensors/__init__.py similarity index 100% rename from tests/providers/google/suite/hooks/__init__.py rename to providers/tests/google/suite/sensors/__init__.py diff --git a/tests/providers/google/suite/sensors/test_drive.py b/providers/tests/google/suite/sensors/test_drive.py similarity index 100% rename from tests/providers/google/suite/sensors/test_drive.py rename to providers/tests/google/suite/sensors/test_drive.py diff --git a/tests/providers/ftp/operators/__init__.py b/providers/tests/google/suite/transfers/__init__.py similarity index 100% rename from tests/providers/ftp/operators/__init__.py rename to providers/tests/google/suite/transfers/__init__.py diff --git a/tests/providers/google/suite/transfers/test_gcs_to_gdrive.py b/providers/tests/google/suite/transfers/test_gcs_to_gdrive.py similarity index 100% rename from tests/providers/google/suite/transfers/test_gcs_to_gdrive.py rename to providers/tests/google/suite/transfers/test_gcs_to_gdrive.py diff --git a/tests/providers/google/suite/transfers/test_gcs_to_sheets.py b/providers/tests/google/suite/transfers/test_gcs_to_sheets.py similarity index 100% rename from tests/providers/google/suite/transfers/test_gcs_to_sheets.py rename to providers/tests/google/suite/transfers/test_gcs_to_sheets.py diff --git a/tests/providers/google/suite/transfers/test_local_to_drive.py b/providers/tests/google/suite/transfers/test_local_to_drive.py similarity index 100% rename from tests/providers/google/suite/transfers/test_local_to_drive.py rename to providers/tests/google/suite/transfers/test_local_to_drive.py diff --git a/tests/providers/google/suite/transfers/test_sql_to_sheets.py b/providers/tests/google/suite/transfers/test_sql_to_sheets.py similarity index 100% rename from tests/providers/google/suite/transfers/test_sql_to_sheets.py rename to providers/tests/google/suite/transfers/test_sql_to_sheets.py diff --git a/tests/providers/google/test_go_module.py b/providers/tests/google/test_go_module.py similarity index 100% rename from tests/providers/google/test_go_module.py rename to providers/tests/google/test_go_module.py diff --git a/tests/providers/google/suite/operators/__init__.py b/providers/tests/grpc/__init__.py similarity index 100% rename from tests/providers/google/suite/operators/__init__.py rename to providers/tests/grpc/__init__.py diff --git a/tests/providers/google/suite/sensors/__init__.py b/providers/tests/grpc/hooks/__init__.py similarity index 100% rename from tests/providers/google/suite/sensors/__init__.py rename to providers/tests/grpc/hooks/__init__.py diff --git a/tests/providers/grpc/hooks/test_grpc.py b/providers/tests/grpc/hooks/test_grpc.py similarity index 100% rename from tests/providers/grpc/hooks/test_grpc.py rename to providers/tests/grpc/hooks/test_grpc.py diff --git a/tests/providers/grpc/__init__.py b/providers/tests/grpc/operators/__init__.py similarity index 100% rename from tests/providers/grpc/__init__.py rename to providers/tests/grpc/operators/__init__.py diff --git a/tests/providers/grpc/operators/test_grpc.py b/providers/tests/grpc/operators/test_grpc.py similarity index 100% rename from tests/providers/grpc/operators/test_grpc.py rename to providers/tests/grpc/operators/test_grpc.py diff --git a/tests/providers/github/__init__.py b/providers/tests/hashicorp/__init__.py similarity index 100% rename from tests/providers/github/__init__.py rename to providers/tests/hashicorp/__init__.py diff --git a/tests/providers/github/hooks/__init__.py b/providers/tests/hashicorp/_internal_client/__init__.py similarity index 100% rename from tests/providers/github/hooks/__init__.py rename to providers/tests/hashicorp/_internal_client/__init__.py diff --git a/tests/providers/hashicorp/_internal_client/test_vault_client.py b/providers/tests/hashicorp/_internal_client/test_vault_client.py similarity index 100% rename from tests/providers/hashicorp/_internal_client/test_vault_client.py rename to providers/tests/hashicorp/_internal_client/test_vault_client.py diff --git a/tests/providers/github/operators/__init__.py b/providers/tests/hashicorp/hooks/__init__.py similarity index 100% rename from tests/providers/github/operators/__init__.py rename to providers/tests/hashicorp/hooks/__init__.py diff --git a/tests/providers/hashicorp/hooks/test_vault.py b/providers/tests/hashicorp/hooks/test_vault.py similarity index 99% rename from tests/providers/hashicorp/hooks/test_vault.py rename to providers/tests/hashicorp/hooks/test_vault.py index 1880be99749e6..442a656d20064 100644 --- a/tests/providers/hashicorp/hooks/test_vault.py +++ b/providers/tests/hashicorp/hooks/test_vault.py @@ -26,7 +26,8 @@ from airflow.configuration import AirflowConfigParser from airflow.exceptions import AirflowConfigException from airflow.providers.hashicorp.hooks.vault import VaultHook -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars class TestVaultHook: diff --git a/tests/providers/github/sensors/__init__.py b/providers/tests/hashicorp/secrets/__init__.py similarity index 100% rename from tests/providers/github/sensors/__init__.py rename to providers/tests/hashicorp/secrets/__init__.py diff --git a/tests/providers/hashicorp/secrets/test_vault.py b/providers/tests/hashicorp/secrets/test_vault.py similarity index 100% rename from tests/providers/hashicorp/secrets/test_vault.py rename to providers/tests/hashicorp/secrets/test_vault.py diff --git a/tests/providers/grpc/hooks/__init__.py b/providers/tests/http/__init__.py similarity index 100% rename from tests/providers/grpc/hooks/__init__.py rename to providers/tests/http/__init__.py diff --git a/tests/providers/grpc/operators/__init__.py b/providers/tests/http/hooks/__init__.py similarity index 100% rename from tests/providers/grpc/operators/__init__.py rename to providers/tests/http/hooks/__init__.py diff --git a/tests/providers/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py similarity index 95% rename from tests/providers/http/hooks/test_http.py rename to providers/tests/http/hooks/test_http.py index 5b631df8a4d60..e09fd2d034e60 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -487,11 +487,15 @@ def test_default_auth_not_initialized(self, auth): auth.assert_not_called() def test_keep_alive_enabled(self): - with mock.patch( - "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port - ), mock.patch( - "requests_toolbelt.adapters.socket_options.TCPKeepAliveAdapter.send" - ) as tcp_keep_alive_send, mock.patch("requests.adapters.HTTPAdapter.send") as http_send: + with ( + mock.patch( + "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port + ), + mock.patch( + "requests_toolbelt.adapters.socket_options.TCPKeepAliveAdapter.send" + ) as tcp_keep_alive_send, + mock.patch("requests.adapters.HTTPAdapter.send") as http_send, + ): hook = HttpHook(method="GET") response = Response() response.status_code = HTTPStatus.OK @@ -502,11 +506,15 @@ def test_keep_alive_enabled(self): http_send.assert_not_called() def test_keep_alive_disabled(self): - with mock.patch( - "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port - ), mock.patch( - "requests_toolbelt.adapters.socket_options.TCPKeepAliveAdapter.send" - ) as tcp_keep_alive_send, mock.patch("requests.adapters.HTTPAdapter.send") as http_send: + with ( + mock.patch( + "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port + ), + mock.patch( + "requests_toolbelt.adapters.socket_options.TCPKeepAliveAdapter.send" + ) as tcp_keep_alive_send, + mock.patch("requests.adapters.HTTPAdapter.send") as http_send, + ): hook = HttpHook(method="GET", tcp_keep_alive=False) response = Response() response.status_code = HTTPStatus.OK @@ -536,9 +544,12 @@ async def test_do_api_call_async_non_retryable_error(self, aioresponse): hook = HttpAsyncHook(method="GET") aioresponse.get("http://httpbin.org/non_existent_endpoint", status=400) - with pytest.raises(AirflowException, match="400:Bad Request"), mock.patch.dict( - "os.environ", - AIRFLOW_CONN_HTTP_DEFAULT="http://httpbin.org/", + with ( + pytest.raises(AirflowException, match="400:Bad Request"), + mock.patch.dict( + "os.environ", + AIRFLOW_CONN_HTTP_DEFAULT="http://httpbin.org/", + ), ): await hook.run(endpoint="non_existent_endpoint") @@ -549,9 +560,12 @@ async def test_do_api_call_async_retryable_error(self, caplog, aioresponse): hook = HttpAsyncHook(method="GET") aioresponse.get("http://httpbin.org/non_existent_endpoint", status=500, repeat=True) - with pytest.raises(AirflowException, match="500:Internal Server Error"), mock.patch.dict( - "os.environ", - AIRFLOW_CONN_HTTP_DEFAULT="http://httpbin.org/", + with ( + pytest.raises(AirflowException, match="500:Internal Server Error"), + mock.patch.dict( + "os.environ", + AIRFLOW_CONN_HTTP_DEFAULT="http://httpbin.org/", + ), ): await hook.run(endpoint="non_existent_endpoint") @@ -700,8 +714,9 @@ def get_empty_conn(conn_id: str = "http_default"): return Connection(conn_id=conn_id, conn_type="http") hook = HttpAsyncHook() - with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_empty_conn), mock.patch( - "aiohttp.ClientSession.post", new_callable=mock.AsyncMock - ) as mocked_function: + with ( + mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_empty_conn), + mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function, + ): await hook.run("test.com:8080/v1/test") assert mocked_function.call_args.args[0] == "http://test.com:8080/v1/test" diff --git a/tests/providers/http/__init__.py b/providers/tests/http/operators/__init__.py similarity index 100% rename from tests/providers/http/__init__.py rename to providers/tests/http/operators/__init__.py diff --git a/tests/providers/http/operators/test_http.py b/providers/tests/http/operators/test_http.py similarity index 100% rename from tests/providers/http/operators/test_http.py rename to providers/tests/http/operators/test_http.py diff --git a/tests/providers/http/hooks/__init__.py b/providers/tests/http/sensors/__init__.py similarity index 100% rename from tests/providers/http/hooks/__init__.py rename to providers/tests/http/sensors/__init__.py diff --git a/tests/providers/http/sensors/test_http.py b/providers/tests/http/sensors/test_http.py similarity index 100% rename from tests/providers/http/sensors/test_http.py rename to providers/tests/http/sensors/test_http.py diff --git a/tests/providers/http/operators/__init__.py b/providers/tests/http/triggers/__init__.py similarity index 100% rename from tests/providers/http/operators/__init__.py rename to providers/tests/http/triggers/__init__.py diff --git a/tests/providers/http/triggers/test_http.py b/providers/tests/http/triggers/test_http.py similarity index 100% rename from tests/providers/http/triggers/test_http.py rename to providers/tests/http/triggers/test_http.py diff --git a/tests/providers/http/sensors/__init__.py b/providers/tests/imap/__init__.py similarity index 100% rename from tests/providers/http/sensors/__init__.py rename to providers/tests/imap/__init__.py diff --git a/tests/providers/http/triggers/__init__.py b/providers/tests/imap/hooks/__init__.py similarity index 100% rename from tests/providers/http/triggers/__init__.py rename to providers/tests/imap/hooks/__init__.py diff --git a/tests/providers/imap/hooks/test_imap.py b/providers/tests/imap/hooks/test_imap.py similarity index 99% rename from tests/providers/imap/hooks/test_imap.py rename to providers/tests/imap/hooks/test_imap.py index 2430d3f664882..2971cfd1924c6 100644 --- a/tests/providers/imap/hooks/test_imap.py +++ b/providers/tests/imap/hooks/test_imap.py @@ -27,7 +27,8 @@ from airflow.models import Connection from airflow.providers.imap.hooks.imap import ImapHook from airflow.utils import db -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/imap/__init__.py b/providers/tests/imap/sensors/__init__.py similarity index 100% rename from tests/providers/imap/__init__.py rename to providers/tests/imap/sensors/__init__.py diff --git a/tests/providers/imap/sensors/test_imap_attachment.py b/providers/tests/imap/sensors/test_imap_attachment.py similarity index 100% rename from tests/providers/imap/sensors/test_imap_attachment.py rename to providers/tests/imap/sensors/test_imap_attachment.py diff --git a/tests/providers/google/__init__.py b/providers/tests/influxdb/__init__.py similarity index 100% rename from tests/providers/google/__init__.py rename to providers/tests/influxdb/__init__.py diff --git a/tests/providers/google/ads/__init__.py b/providers/tests/influxdb/hooks/__init__.py similarity index 100% rename from tests/providers/google/ads/__init__.py rename to providers/tests/influxdb/hooks/__init__.py diff --git a/tests/providers/influxdb/hooks/test_influxdb.py b/providers/tests/influxdb/hooks/test_influxdb.py similarity index 100% rename from tests/providers/influxdb/hooks/test_influxdb.py rename to providers/tests/influxdb/hooks/test_influxdb.py diff --git a/tests/providers/google/ads/hooks/__init__.py b/providers/tests/influxdb/operators/__init__.py similarity index 100% rename from tests/providers/google/ads/hooks/__init__.py rename to providers/tests/influxdb/operators/__init__.py diff --git a/tests/providers/influxdb/operators/test_influxdb.py b/providers/tests/influxdb/operators/test_influxdb.py similarity index 100% rename from tests/providers/influxdb/operators/test_influxdb.py rename to providers/tests/influxdb/operators/test_influxdb.py diff --git a/tests/providers/google/ads/operators/__init__.py b/providers/tests/integration/__init__.py similarity index 100% rename from tests/providers/google/ads/operators/__init__.py rename to providers/tests/integration/__init__.py diff --git a/tests/providers/google/ads/transfers/__init__.py b/providers/tests/integration/apache/__init__.py similarity index 100% rename from tests/providers/google/ads/transfers/__init__.py rename to providers/tests/integration/apache/__init__.py diff --git a/tests/providers/google/cloud/__init__.py b/providers/tests/integration/apache/cassandra/__init__.py similarity index 100% rename from tests/providers/google/cloud/__init__.py rename to providers/tests/integration/apache/cassandra/__init__.py diff --git a/tests/providers/google/cloud/_internal_client/__init__.py b/providers/tests/integration/apache/cassandra/hooks/__init__.py similarity index 100% rename from tests/providers/google/cloud/_internal_client/__init__.py rename to providers/tests/integration/apache/cassandra/hooks/__init__.py diff --git a/tests/integration/providers/apache/cassandra/hooks/test_cassandra.py b/providers/tests/integration/apache/cassandra/hooks/test_cassandra.py similarity index 100% rename from tests/integration/providers/apache/cassandra/hooks/test_cassandra.py rename to providers/tests/integration/apache/cassandra/hooks/test_cassandra.py diff --git a/tests/providers/google/cloud/hooks/__init__.py b/providers/tests/integration/apache/drill/__init__.py similarity index 100% rename from tests/providers/google/cloud/hooks/__init__.py rename to providers/tests/integration/apache/drill/__init__.py diff --git a/tests/providers/google/cloud/hooks/vertex_ai/__init__.py b/providers/tests/integration/apache/drill/hooks/__init__.py similarity index 100% rename from tests/providers/google/cloud/hooks/vertex_ai/__init__.py rename to providers/tests/integration/apache/drill/hooks/__init__.py diff --git a/tests/integration/providers/apache/drill/hooks/test_drill.py b/providers/tests/integration/apache/drill/hooks/test_drill.py similarity index 100% rename from tests/integration/providers/apache/drill/hooks/test_drill.py rename to providers/tests/integration/apache/drill/hooks/test_drill.py diff --git a/tests/providers/google/cloud/links/__init__.py b/providers/tests/integration/apache/drill/operators/__init__.py similarity index 100% rename from tests/providers/google/cloud/links/__init__.py rename to providers/tests/integration/apache/drill/operators/__init__.py diff --git a/tests/providers/google/cloud/log/__init__.py b/providers/tests/integration/apache/hive/__init__.py similarity index 100% rename from tests/providers/google/cloud/log/__init__.py rename to providers/tests/integration/apache/hive/__init__.py diff --git a/tests/providers/google/cloud/openlineage/__init__.py b/providers/tests/integration/apache/hive/transfers/__init__.py similarity index 100% rename from tests/providers/google/cloud/openlineage/__init__.py rename to providers/tests/integration/apache/hive/transfers/__init__.py diff --git a/tests/integration/providers/apache/hive/transfers/test_mssql_to_hive.py b/providers/tests/integration/apache/hive/transfers/test_mssql_to_hive.py similarity index 100% rename from tests/integration/providers/apache/hive/transfers/test_mssql_to_hive.py rename to providers/tests/integration/apache/hive/transfers/test_mssql_to_hive.py diff --git a/tests/providers/google/cloud/operators/__init__.py b/providers/tests/integration/apache/kafka/__init__.py similarity index 100% rename from tests/providers/google/cloud/operators/__init__.py rename to providers/tests/integration/apache/kafka/__init__.py diff --git a/tests/providers/google/cloud/operators/source/__init__.py b/providers/tests/integration/apache/kafka/hooks/__init__.py similarity index 100% rename from tests/providers/google/cloud/operators/source/__init__.py rename to providers/tests/integration/apache/kafka/hooks/__init__.py diff --git a/tests/integration/providers/apache/kafka/hooks/test_admin_client.py b/providers/tests/integration/apache/kafka/hooks/test_admin_client.py similarity index 100% rename from tests/integration/providers/apache/kafka/hooks/test_admin_client.py rename to providers/tests/integration/apache/kafka/hooks/test_admin_client.py diff --git a/tests/integration/providers/apache/kafka/hooks/test_consumer.py b/providers/tests/integration/apache/kafka/hooks/test_consumer.py similarity index 100% rename from tests/integration/providers/apache/kafka/hooks/test_consumer.py rename to providers/tests/integration/apache/kafka/hooks/test_consumer.py diff --git a/tests/integration/providers/apache/kafka/hooks/test_producer.py b/providers/tests/integration/apache/kafka/hooks/test_producer.py similarity index 100% rename from tests/integration/providers/apache/kafka/hooks/test_producer.py rename to providers/tests/integration/apache/kafka/hooks/test_producer.py diff --git a/tests/providers/google/cloud/operators/source/source_prefix/__init__.py b/providers/tests/integration/apache/kafka/operators/__init__.py similarity index 100% rename from tests/providers/google/cloud/operators/source/source_prefix/__init__.py rename to providers/tests/integration/apache/kafka/operators/__init__.py diff --git a/tests/integration/providers/apache/kafka/operators/test_consume.py b/providers/tests/integration/apache/kafka/operators/test_consume.py similarity index 100% rename from tests/integration/providers/apache/kafka/operators/test_consume.py rename to providers/tests/integration/apache/kafka/operators/test_consume.py diff --git a/tests/integration/providers/apache/kafka/operators/test_produce.py b/providers/tests/integration/apache/kafka/operators/test_produce.py similarity index 100% rename from tests/integration/providers/apache/kafka/operators/test_produce.py rename to providers/tests/integration/apache/kafka/operators/test_produce.py diff --git a/tests/providers/google/cloud/operators/vertex_ai/__init__.py b/providers/tests/integration/apache/kafka/sensors/__init__.py similarity index 100% rename from tests/providers/google/cloud/operators/vertex_ai/__init__.py rename to providers/tests/integration/apache/kafka/sensors/__init__.py diff --git a/tests/providers/google/cloud/secrets/__init__.py b/providers/tests/integration/apache/kafka/triggers/__init__.py similarity index 100% rename from tests/providers/google/cloud/secrets/__init__.py rename to providers/tests/integration/apache/kafka/triggers/__init__.py diff --git a/tests/integration/providers/apache/kafka/triggers/test_await_message.py b/providers/tests/integration/apache/kafka/triggers/test_await_message.py similarity index 100% rename from tests/integration/providers/apache/kafka/triggers/test_await_message.py rename to providers/tests/integration/apache/kafka/triggers/test_await_message.py diff --git a/tests/providers/google/cloud/sensors/__init__.py b/providers/tests/integration/apache/pinot/__init__.py similarity index 100% rename from tests/providers/google/cloud/sensors/__init__.py rename to providers/tests/integration/apache/pinot/__init__.py diff --git a/tests/providers/google/cloud/transfers/__init__.py b/providers/tests/integration/apache/pinot/hooks/__init__.py similarity index 100% rename from tests/providers/google/cloud/transfers/__init__.py rename to providers/tests/integration/apache/pinot/hooks/__init__.py diff --git a/tests/integration/providers/apache/pinot/hooks/test_pinot.py b/providers/tests/integration/apache/pinot/hooks/test_pinot.py similarity index 100% rename from tests/integration/providers/apache/pinot/hooks/test_pinot.py rename to providers/tests/integration/apache/pinot/hooks/test_pinot.py diff --git a/tests/providers/google/cloud/triggers/__init__.py b/providers/tests/integration/google/__init__.py similarity index 100% rename from tests/providers/google/cloud/triggers/__init__.py rename to providers/tests/integration/google/__init__.py diff --git a/tests/providers/google/common/__init__.py b/providers/tests/integration/google/cloud/__init__.py similarity index 100% rename from tests/providers/google/common/__init__.py rename to providers/tests/integration/google/cloud/__init__.py diff --git a/tests/providers/google/common/auth_backend/__init__.py b/providers/tests/integration/google/cloud/transfers/__init__.py similarity index 100% rename from tests/providers/google/common/auth_backend/__init__.py rename to providers/tests/integration/google/cloud/transfers/__init__.py diff --git a/tests/integration/providers/google/cloud/transfers/test_bigquery_to_mssql.py b/providers/tests/integration/google/cloud/transfers/test_bigquery_to_mssql.py similarity index 100% rename from tests/integration/providers/google/cloud/transfers/test_bigquery_to_mssql.py rename to providers/tests/integration/google/cloud/transfers/test_bigquery_to_mssql.py diff --git a/tests/integration/providers/google/cloud/transfers/test_mssql_to_gcs.py b/providers/tests/integration/google/cloud/transfers/test_mssql_to_gcs.py similarity index 100% rename from tests/integration/providers/google/cloud/transfers/test_mssql_to_gcs.py rename to providers/tests/integration/google/cloud/transfers/test_mssql_to_gcs.py diff --git a/tests/integration/providers/google/cloud/transfers/test_trino_to_gcs.py b/providers/tests/integration/google/cloud/transfers/test_trino_to_gcs.py similarity index 100% rename from tests/integration/providers/google/cloud/transfers/test_trino_to_gcs.py rename to providers/tests/integration/google/cloud/transfers/test_trino_to_gcs.py diff --git a/tests/providers/google/common/hooks/__init__.py b/providers/tests/integration/microsoft/__init__.py similarity index 100% rename from tests/providers/google/common/hooks/__init__.py rename to providers/tests/integration/microsoft/__init__.py diff --git a/tests/providers/google/common/utils/__init__.py b/providers/tests/integration/microsoft/mssql/__init__.py similarity index 100% rename from tests/providers/google/common/utils/__init__.py rename to providers/tests/integration/microsoft/mssql/__init__.py diff --git a/tests/providers/google/datasets/__init__.py b/providers/tests/integration/microsoft/mssql/hooks/__init__.py similarity index 100% rename from tests/providers/google/datasets/__init__.py rename to providers/tests/integration/microsoft/mssql/hooks/__init__.py diff --git a/tests/integration/providers/microsoft/mssql/hooks/test_mssql.py b/providers/tests/integration/microsoft/mssql/hooks/test_mssql.py similarity index 100% rename from tests/integration/providers/microsoft/mssql/hooks/test_mssql.py rename to providers/tests/integration/microsoft/mssql/hooks/test_mssql.py diff --git a/tests/providers/google/firebase/__init__.py b/providers/tests/integration/mongo/__init__.py similarity index 100% rename from tests/providers/google/firebase/__init__.py rename to providers/tests/integration/mongo/__init__.py diff --git a/tests/providers/google/firebase/hooks/__init__.py b/providers/tests/integration/mongo/sensors/__init__.py similarity index 100% rename from tests/providers/google/firebase/hooks/__init__.py rename to providers/tests/integration/mongo/sensors/__init__.py diff --git a/tests/integration/providers/mongo/sensors/test_mongo.py b/providers/tests/integration/mongo/sensors/test_mongo.py similarity index 100% rename from tests/integration/providers/mongo/sensors/test_mongo.py rename to providers/tests/integration/mongo/sensors/test_mongo.py diff --git a/tests/providers/google/firebase/operators/__init__.py b/providers/tests/integration/openlineage/__init__.py similarity index 100% rename from tests/providers/google/firebase/operators/__init__.py rename to providers/tests/integration/openlineage/__init__.py diff --git a/tests/providers/google/marketing_platform/__init__.py b/providers/tests/integration/openlineage/operators/__init__.py similarity index 100% rename from tests/providers/google/marketing_platform/__init__.py rename to providers/tests/integration/openlineage/operators/__init__.py diff --git a/tests/providers/google/marketing_platform/hooks/__init__.py b/providers/tests/integration/qdrant/__init__.py similarity index 100% rename from tests/providers/google/marketing_platform/hooks/__init__.py rename to providers/tests/integration/qdrant/__init__.py diff --git a/tests/providers/google/marketing_platform/operators/__init__.py b/providers/tests/integration/qdrant/hooks/__init__.py similarity index 100% rename from tests/providers/google/marketing_platform/operators/__init__.py rename to providers/tests/integration/qdrant/hooks/__init__.py diff --git a/tests/integration/providers/qdrant/hooks/test_qdrant.py b/providers/tests/integration/qdrant/hooks/test_qdrant.py similarity index 100% rename from tests/integration/providers/qdrant/hooks/test_qdrant.py rename to providers/tests/integration/qdrant/hooks/test_qdrant.py diff --git a/tests/providers/google/marketing_platform/sensors/__init__.py b/providers/tests/integration/qdrant/operators/__init__.py similarity index 100% rename from tests/providers/google/marketing_platform/sensors/__init__.py rename to providers/tests/integration/qdrant/operators/__init__.py diff --git a/tests/integration/providers/qdrant/operators/test_qdrant_ingest.py b/providers/tests/integration/qdrant/operators/test_qdrant_ingest.py similarity index 100% rename from tests/integration/providers/qdrant/operators/test_qdrant_ingest.py rename to providers/tests/integration/qdrant/operators/test_qdrant_ingest.py diff --git a/tests/providers/google/suite/transfers/__init__.py b/providers/tests/integration/redis/__init__.py similarity index 100% rename from tests/providers/google/suite/transfers/__init__.py rename to providers/tests/integration/redis/__init__.py diff --git a/tests/providers/hashicorp/__init__.py b/providers/tests/integration/redis/hooks/__init__.py similarity index 100% rename from tests/providers/hashicorp/__init__.py rename to providers/tests/integration/redis/hooks/__init__.py diff --git a/tests/integration/providers/redis/hooks/test_redis.py b/providers/tests/integration/redis/hooks/test_redis.py similarity index 100% rename from tests/integration/providers/redis/hooks/test_redis.py rename to providers/tests/integration/redis/hooks/test_redis.py diff --git a/tests/providers/hashicorp/_internal_client/__init__.py b/providers/tests/integration/redis/operators/__init__.py similarity index 100% rename from tests/providers/hashicorp/_internal_client/__init__.py rename to providers/tests/integration/redis/operators/__init__.py diff --git a/tests/integration/providers/redis/operators/test_redis_publish.py b/providers/tests/integration/redis/operators/test_redis_publish.py similarity index 100% rename from tests/integration/providers/redis/operators/test_redis_publish.py rename to providers/tests/integration/redis/operators/test_redis_publish.py diff --git a/tests/providers/hashicorp/hooks/__init__.py b/providers/tests/integration/redis/sensors/__init__.py similarity index 100% rename from tests/providers/hashicorp/hooks/__init__.py rename to providers/tests/integration/redis/sensors/__init__.py diff --git a/tests/integration/providers/redis/sensors/test_redis_key.py b/providers/tests/integration/redis/sensors/test_redis_key.py similarity index 100% rename from tests/integration/providers/redis/sensors/test_redis_key.py rename to providers/tests/integration/redis/sensors/test_redis_key.py diff --git a/tests/integration/providers/redis/sensors/test_redis_pub_sub.py b/providers/tests/integration/redis/sensors/test_redis_pub_sub.py similarity index 100% rename from tests/integration/providers/redis/sensors/test_redis_pub_sub.py rename to providers/tests/integration/redis/sensors/test_redis_pub_sub.py diff --git a/tests/providers/hashicorp/secrets/__init__.py b/providers/tests/integration/trino/__init__.py similarity index 100% rename from tests/providers/hashicorp/secrets/__init__.py rename to providers/tests/integration/trino/__init__.py diff --git a/tests/providers/influxdb/__init__.py b/providers/tests/integration/trino/hooks/__init__.py similarity index 100% rename from tests/providers/influxdb/__init__.py rename to providers/tests/integration/trino/hooks/__init__.py diff --git a/tests/integration/providers/trino/hooks/test_trino.py b/providers/tests/integration/trino/hooks/test_trino.py similarity index 100% rename from tests/integration/providers/trino/hooks/test_trino.py rename to providers/tests/integration/trino/hooks/test_trino.py diff --git a/tests/providers/influxdb/hooks/__init__.py b/providers/tests/integration/ydb/__init__.py similarity index 100% rename from tests/providers/influxdb/hooks/__init__.py rename to providers/tests/integration/ydb/__init__.py diff --git a/tests/providers/influxdb/operators/__init__.py b/providers/tests/integration/ydb/hooks/__init__.py similarity index 100% rename from tests/providers/influxdb/operators/__init__.py rename to providers/tests/integration/ydb/hooks/__init__.py diff --git a/tests/providers/jenkins/hooks/__init__.py b/providers/tests/integration/ydb/operators/__init__.py similarity index 100% rename from tests/providers/jenkins/hooks/__init__.py rename to providers/tests/integration/ydb/operators/__init__.py diff --git a/tests/integration/providers/ydb/operators/test_ydb.py b/providers/tests/integration/ydb/operators/test_ydb.py similarity index 100% rename from tests/integration/providers/ydb/operators/test_ydb.py rename to providers/tests/integration/ydb/operators/test_ydb.py diff --git a/tests/providers/imap/hooks/__init__.py b/providers/tests/jdbc/__init__.py similarity index 100% rename from tests/providers/imap/hooks/__init__.py rename to providers/tests/jdbc/__init__.py diff --git a/tests/providers/imap/sensors/__init__.py b/providers/tests/jdbc/hooks/__init__.py similarity index 100% rename from tests/providers/imap/sensors/__init__.py rename to providers/tests/jdbc/hooks/__init__.py diff --git a/tests/providers/jdbc/hooks/test_jdbc.py b/providers/tests/jdbc/hooks/test_jdbc.py similarity index 97% rename from tests/providers/jdbc/hooks/test_jdbc.py rename to providers/tests/jdbc/hooks/test_jdbc.py index f26a9d7ffb5b3..cfb27934d86da 100644 --- a/tests/providers/jdbc/hooks/test_jdbc.py +++ b/providers/tests/jdbc/hooks/test_jdbc.py @@ -157,9 +157,10 @@ def test_driver_extra_works_when_allow_driver_extra(self): assert hook.driver_class == "Blah driver class" def test_default_driver_set(self): - with patch.object(JdbcHook, "default_driver_path", "Blah driver path") as _, patch.object( - JdbcHook, "default_driver_class", "Blah driver class" - ) as _: + with ( + patch.object(JdbcHook, "default_driver_path", "Blah driver path") as _, + patch.object(JdbcHook, "default_driver_class", "Blah driver class") as _, + ): hook = get_hook() assert hook.driver_path == "Blah driver path" assert hook.driver_class == "Blah driver class" diff --git a/tests/providers/jdbc/__init__.py b/providers/tests/jdbc/operators/__init__.py similarity index 100% rename from tests/providers/jdbc/__init__.py rename to providers/tests/jdbc/operators/__init__.py diff --git a/tests/providers/jdbc/operators/test_jdbc.py b/providers/tests/jdbc/operators/test_jdbc.py similarity index 100% rename from tests/providers/jdbc/operators/test_jdbc.py rename to providers/tests/jdbc/operators/test_jdbc.py diff --git a/tests/providers/jdbc/hooks/__init__.py b/providers/tests/jenkins/__init__.py similarity index 100% rename from tests/providers/jdbc/hooks/__init__.py rename to providers/tests/jenkins/__init__.py diff --git a/tests/providers/microsoft/azure/log/__init__.py b/providers/tests/jenkins/hooks/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/log/__init__.py rename to providers/tests/jenkins/hooks/__init__.py diff --git a/tests/providers/jenkins/hooks/test_jenkins.py b/providers/tests/jenkins/hooks/test_jenkins.py similarity index 100% rename from tests/providers/jenkins/hooks/test_jenkins.py rename to providers/tests/jenkins/hooks/test_jenkins.py diff --git a/tests/providers/jdbc/operators/__init__.py b/providers/tests/jenkins/operators/__init__.py similarity index 100% rename from tests/providers/jdbc/operators/__init__.py rename to providers/tests/jenkins/operators/__init__.py diff --git a/tests/providers/jenkins/operators/test_jenkins_job_trigger.py b/providers/tests/jenkins/operators/test_jenkins_job_trigger.py similarity index 85% rename from tests/providers/jenkins/operators/test_jenkins_job_trigger.py rename to providers/tests/jenkins/operators/test_jenkins_job_trigger.py index d21cdbedc007a..2d740da1fe9f1 100644 --- a/tests/providers/jenkins/operators/test_jenkins_job_trigger.py +++ b/providers/tests/jenkins/operators/test_jenkins_job_trigger.py @@ -46,13 +46,16 @@ def test_execute(self, parameters, mocker): hook_mock = Mock(spec=JenkinsHook) hook_mock.get_jenkins_server.return_value = jenkins_mock - with patch.object( - JenkinsJobTriggerOperator, - "hook", - new_callable=mocker.PropertyMock, - ) as hook_mocked, patch( - "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" - ) as mock_make_request: + with ( + patch.object( + JenkinsJobTriggerOperator, + "hook", + new_callable=mocker.PropertyMock, + ) as hook_mocked, + patch( + "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" + ) as mock_make_request, + ): mock_make_request.side_effect = [ {"body": "", "headers": {"Location": "http://what-a-strange.url/18"}}, {"body": '{"executable":{"number":"1"}}', "headers": {}}, @@ -86,13 +89,16 @@ def test_execute_job_polling_loop(self, parameters, mocker): hook_mock = Mock(spec=JenkinsHook) hook_mock.get_jenkins_server.return_value = jenkins_mock - with patch.object( - JenkinsJobTriggerOperator, - "hook", - new_callable=mocker.PropertyMock, - ) as hook_mocked, patch( - "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" - ) as mock_make_request: + with ( + patch.object( + JenkinsJobTriggerOperator, + "hook", + new_callable=mocker.PropertyMock, + ) as hook_mocked, + patch( + "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" + ) as mock_make_request, + ): mock_make_request.side_effect = [ {"body": "", "headers": {"Location": "http://what-a-strange.url/18"}}, {"body": '{"executable":{"number":"1"}}', "headers": {}}, @@ -124,13 +130,16 @@ def test_execute_job_failure(self, parameters, mocker): hook_mock = Mock(spec=JenkinsHook) hook_mock.get_jenkins_server.return_value = jenkins_mock - with patch.object( - JenkinsJobTriggerOperator, - "hook", - new_callable=mocker.PropertyMock, - ) as hook_mocked, patch( - "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" - ) as mock_make_request: + with ( + patch.object( + JenkinsJobTriggerOperator, + "hook", + new_callable=mocker.PropertyMock, + ) as hook_mocked, + patch( + "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" + ) as mock_make_request, + ): mock_make_request.side_effect = [ {"body": "", "headers": {"Location": "http://what-a-strange.url/18"}}, {"body": '{"executable":{"number":"1"}}', "headers": {}}, @@ -182,13 +191,16 @@ def test_allowed_jenkins_states(self, state, allowed_jenkins_states, mocker): hook_mock = Mock(spec=JenkinsHook) hook_mock.get_jenkins_server.return_value = jenkins_mock - with patch.object( - JenkinsJobTriggerOperator, - "hook", - new_callable=mocker.PropertyMock, - ) as hook_mocked, patch( - "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers", - ) as mock_make_request: + with ( + patch.object( + JenkinsJobTriggerOperator, + "hook", + new_callable=mocker.PropertyMock, + ) as hook_mocked, + patch( + "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers", + ) as mock_make_request, + ): mock_make_request.side_effect = [ {"body": "", "headers": {"Location": "http://what-a-strange.url/18"}}, {"body": '{"executable":{"number":"1"}}', "headers": {}}, @@ -246,13 +258,16 @@ def test_allowed_jenkins_states_failure(self, state, allowed_jenkins_states, moc hook_mock = Mock(spec=JenkinsHook) hook_mock.get_jenkins_server.return_value = jenkins_mock - with patch.object( - JenkinsJobTriggerOperator, - "hook", - new_callable=mocker.PropertyMock, - ) as hook_mocked, patch( - "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" - ) as mock_make_request: + with ( + patch.object( + JenkinsJobTriggerOperator, + "hook", + new_callable=mocker.PropertyMock, + ) as hook_mocked, + patch( + "airflow.providers.jenkins.operators.jenkins_job_trigger.jenkins_request_with_headers" + ) as mock_make_request, + ): mock_make_request.side_effect = [ {"body": "", "headers": {"Location": "http://what-a-strange.url/18"}}, {"body": '{"executable":{"number":"1"}}', "headers": {}}, diff --git a/tests/providers/jenkins/__init__.py b/providers/tests/jenkins/sensors/__init__.py similarity index 100% rename from tests/providers/jenkins/__init__.py rename to providers/tests/jenkins/sensors/__init__.py diff --git a/tests/providers/jenkins/sensors/test_jenkins.py b/providers/tests/jenkins/sensors/test_jenkins.py similarity index 100% rename from tests/providers/jenkins/sensors/test_jenkins.py rename to providers/tests/jenkins/sensors/test_jenkins.py diff --git a/tests/providers/jenkins/operators/__init__.py b/providers/tests/microsoft/__init__.py similarity index 100% rename from tests/providers/jenkins/operators/__init__.py rename to providers/tests/microsoft/__init__.py diff --git a/tests/providers/jenkins/sensors/__init__.py b/providers/tests/microsoft/azure/__init__.py similarity index 100% rename from tests/providers/jenkins/sensors/__init__.py rename to providers/tests/microsoft/azure/__init__.py diff --git a/tests/providers/microsoft/azure/base.py b/providers/tests/microsoft/azure/base.py similarity index 91% rename from tests/providers/microsoft/azure/base.py rename to providers/tests/microsoft/azure/base.py index cad6c1449fdca..98c0a59867ea4 100644 --- a/tests/providers/microsoft/azure/base.py +++ b/providers/tests/microsoft/azure/base.py @@ -26,7 +26,8 @@ from airflow.exceptions import TaskDeferred from airflow.providers.microsoft.azure.hooks.msgraph import KiotaRequestAdapterHook -from tests.providers.microsoft.conftest import get_airflow_connection, mock_context + +from providers.tests.microsoft.conftest import get_airflow_connection, mock_context if TYPE_CHECKING: from airflow.models import Operator @@ -39,9 +40,10 @@ def teardown_method(self, method): @contextmanager def patch_hook_and_request_adapter(self, response): - with patch( - "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection - ), patch.object(HttpxRequestAdapter, "get_http_response_message") as mock_get_http_response: + with ( + patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection), + patch.object(HttpxRequestAdapter, "get_http_response_message") as mock_get_http_response, + ): if isinstance(response, Exception): mock_get_http_response.side_effect = response else: diff --git a/tests/providers/microsoft/__init__.py b/providers/tests/microsoft/azure/fs/__init__.py similarity index 100% rename from tests/providers/microsoft/__init__.py rename to providers/tests/microsoft/azure/fs/__init__.py diff --git a/tests/providers/microsoft/azure/fs/test_adls.py b/providers/tests/microsoft/azure/fs/test_adls.py similarity index 100% rename from tests/providers/microsoft/azure/fs/test_adls.py rename to providers/tests/microsoft/azure/fs/test_adls.py diff --git a/tests/providers/microsoft/azure/__init__.py b/providers/tests/microsoft/azure/hooks/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/__init__.py rename to providers/tests/microsoft/azure/hooks/__init__.py diff --git a/tests/providers/microsoft/azure/hooks/test_adx.py b/providers/tests/microsoft/azure/hooks/test_adx.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_adx.py rename to providers/tests/microsoft/azure/hooks/test_adx.py diff --git a/tests/providers/microsoft/azure/hooks/test_asb.py b/providers/tests/microsoft/azure/hooks/test_asb.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_asb.py rename to providers/tests/microsoft/azure/hooks/test_asb.py diff --git a/tests/providers/microsoft/azure/hooks/test_base_azure.py b/providers/tests/microsoft/azure/hooks/test_base_azure.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_base_azure.py rename to providers/tests/microsoft/azure/hooks/test_base_azure.py diff --git a/tests/providers/microsoft/azure/hooks/test_batch.py b/providers/tests/microsoft/azure/hooks/test_batch.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_batch.py rename to providers/tests/microsoft/azure/hooks/test_batch.py diff --git a/tests/providers/microsoft/azure/hooks/test_container_instance.py b/providers/tests/microsoft/azure/hooks/test_container_instance.py similarity index 95% rename from tests/providers/microsoft/azure/hooks/test_container_instance.py rename to providers/tests/microsoft/azure/hooks/test_container_instance.py index 09dfc167883b0..38b2c743d57f3 100644 --- a/tests/providers/microsoft/azure/hooks/test_container_instance.py +++ b/providers/tests/microsoft/azure/hooks/test_container_instance.py @@ -60,10 +60,13 @@ def setup_test_cases(self, create_mock_connection): ) self.resources = ResourceRequirements(requests=ResourceRequests(memory_in_gb="4", cpu="1")) self.hook = AzureContainerInstanceHook(azure_conn_id=mock_connection.conn_id) - with patch("azure.mgmt.containerinstance.ContainerInstanceManagementClient"), patch( - "azure.common.credentials.ServicePrincipalCredentials.__init__", - autospec=True, - return_value=None, + with ( + patch("azure.mgmt.containerinstance.ContainerInstanceManagementClient"), + patch( + "azure.common.credentials.ServicePrincipalCredentials.__init__", + autospec=True, + return_value=None, + ), ): yield diff --git a/tests/providers/microsoft/azure/hooks/test_container_registry.py b/providers/tests/microsoft/azure/hooks/test_container_registry.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_container_registry.py rename to providers/tests/microsoft/azure/hooks/test_container_registry.py diff --git a/tests/providers/microsoft/azure/hooks/test_container_volume.py b/providers/tests/microsoft/azure/hooks/test_container_volume.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_container_volume.py rename to providers/tests/microsoft/azure/hooks/test_container_volume.py diff --git a/tests/providers/microsoft/azure/hooks/test_cosmos.py b/providers/tests/microsoft/azure/hooks/test_cosmos.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_cosmos.py rename to providers/tests/microsoft/azure/hooks/test_cosmos.py diff --git a/tests/providers/microsoft/azure/hooks/test_data_factory.py b/providers/tests/microsoft/azure/hooks/test_data_factory.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_data_factory.py rename to providers/tests/microsoft/azure/hooks/test_data_factory.py diff --git a/tests/providers/microsoft/azure/hooks/test_data_lake.py b/providers/tests/microsoft/azure/hooks/test_data_lake.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_data_lake.py rename to providers/tests/microsoft/azure/hooks/test_data_lake.py diff --git a/tests/providers/microsoft/azure/hooks/test_fileshare.py b/providers/tests/microsoft/azure/hooks/test_fileshare.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_fileshare.py rename to providers/tests/microsoft/azure/hooks/test_fileshare.py diff --git a/tests/providers/microsoft/azure/hooks/test_msgraph.py b/providers/tests/microsoft/azure/hooks/test_msgraph.py similarity index 99% rename from tests/providers/microsoft/azure/hooks/test_msgraph.py rename to providers/tests/microsoft/azure/hooks/test_msgraph.py index 04e85525616bc..0ecad98548b51 100644 --- a/tests/providers/microsoft/azure/hooks/test_msgraph.py +++ b/providers/tests/microsoft/azure/hooks/test_msgraph.py @@ -30,7 +30,8 @@ DefaultResponseHandler, KiotaRequestAdapterHook, ) -from tests.providers.microsoft.conftest import ( + +from providers.tests.microsoft.conftest import ( get_airflow_connection, load_file, load_json, diff --git a/tests/providers/microsoft/azure/hooks/test_powerbi.py b/providers/tests/microsoft/azure/hooks/test_powerbi.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_powerbi.py rename to providers/tests/microsoft/azure/hooks/test_powerbi.py diff --git a/tests/providers/microsoft/azure/hooks/test_synapse.py b/providers/tests/microsoft/azure/hooks/test_synapse.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_synapse.py rename to providers/tests/microsoft/azure/hooks/test_synapse.py diff --git a/tests/providers/microsoft/azure/hooks/test_synapse_pipeline.py b/providers/tests/microsoft/azure/hooks/test_synapse_pipeline.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_synapse_pipeline.py rename to providers/tests/microsoft/azure/hooks/test_synapse_pipeline.py diff --git a/tests/providers/microsoft/azure/hooks/test_wasb.py b/providers/tests/microsoft/azure/hooks/test_wasb.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/test_wasb.py rename to providers/tests/microsoft/azure/hooks/test_wasb.py diff --git a/tests/providers/microsoft/azure/resources/__init__.py b/providers/tests/microsoft/azure/log/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/resources/__init__.py rename to providers/tests/microsoft/azure/log/__init__.py diff --git a/tests/providers/microsoft/azure/log/test_wasb_task_handler.py b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py similarity index 98% rename from tests/providers/microsoft/azure/log/test_wasb_task_handler.py rename to providers/tests/microsoft/azure/log/test_wasb_task_handler.py index 7de68ec63a798..224961efe4336 100644 --- a/tests/providers/microsoft/azure/log/test_wasb_task_handler.py +++ b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py @@ -29,8 +29,9 @@ from airflow.providers.microsoft.azure.log.wasb_task_handler import WasbTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/providers/microsoft/azure/fs/__init__.py b/providers/tests/microsoft/azure/operators/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/fs/__init__.py rename to providers/tests/microsoft/azure/operators/__init__.py diff --git a/tests/providers/microsoft/azure/operators/test_adls_create.py b/providers/tests/microsoft/azure/operators/test_adls_create.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_adls_create.py rename to providers/tests/microsoft/azure/operators/test_adls_create.py diff --git a/tests/providers/microsoft/azure/operators/test_adls_delete.py b/providers/tests/microsoft/azure/operators/test_adls_delete.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_adls_delete.py rename to providers/tests/microsoft/azure/operators/test_adls_delete.py diff --git a/tests/providers/microsoft/azure/operators/test_adls_list.py b/providers/tests/microsoft/azure/operators/test_adls_list.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_adls_list.py rename to providers/tests/microsoft/azure/operators/test_adls_list.py diff --git a/tests/providers/microsoft/azure/operators/test_adx.py b/providers/tests/microsoft/azure/operators/test_adx.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_adx.py rename to providers/tests/microsoft/azure/operators/test_adx.py diff --git a/tests/providers/microsoft/azure/operators/test_asb.py b/providers/tests/microsoft/azure/operators/test_asb.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_asb.py rename to providers/tests/microsoft/azure/operators/test_asb.py diff --git a/tests/providers/microsoft/azure/operators/test_batch.py b/providers/tests/microsoft/azure/operators/test_batch.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_batch.py rename to providers/tests/microsoft/azure/operators/test_batch.py diff --git a/tests/providers/microsoft/azure/operators/test_container_instances.py b/providers/tests/microsoft/azure/operators/test_container_instances.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_container_instances.py rename to providers/tests/microsoft/azure/operators/test_container_instances.py diff --git a/tests/providers/microsoft/azure/operators/test_cosmos.py b/providers/tests/microsoft/azure/operators/test_cosmos.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_cosmos.py rename to providers/tests/microsoft/azure/operators/test_cosmos.py diff --git a/tests/providers/microsoft/azure/operators/test_data_factory.py b/providers/tests/microsoft/azure/operators/test_data_factory.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_data_factory.py rename to providers/tests/microsoft/azure/operators/test_data_factory.py diff --git a/tests/providers/microsoft/azure/operators/test_msgraph.py b/providers/tests/microsoft/azure/operators/test_msgraph.py similarity index 98% rename from tests/providers/microsoft/azure/operators/test_msgraph.py rename to providers/tests/microsoft/azure/operators/test_msgraph.py index 754b653ccdaf0..372152fe979ee 100644 --- a/tests/providers/microsoft/azure/operators/test_msgraph.py +++ b/providers/tests/microsoft/azure/operators/test_msgraph.py @@ -25,8 +25,9 @@ from airflow.exceptions import AirflowException from airflow.providers.microsoft.azure.operators.msgraph import MSGraphAsyncOperator from airflow.triggers.base import TriggerEvent -from tests.providers.microsoft.azure.base import Base -from tests.providers.microsoft.conftest import ( + +from providers.tests.microsoft.azure.base import Base +from providers.tests.microsoft.conftest import ( load_file, load_json, mock_context, diff --git a/tests/providers/microsoft/azure/operators/test_powerbi.py b/providers/tests/microsoft/azure/operators/test_powerbi.py similarity index 97% rename from tests/providers/microsoft/azure/operators/test_powerbi.py rename to providers/tests/microsoft/azure/operators/test_powerbi.py index 35bb76f782ce3..a9171cff39a6e 100644 --- a/tests/providers/microsoft/azure/operators/test_powerbi.py +++ b/providers/tests/microsoft/azure/operators/test_powerbi.py @@ -30,8 +30,9 @@ from airflow.providers.microsoft.azure.operators.powerbi import PowerBIDatasetRefreshOperator from airflow.providers.microsoft.azure.triggers.powerbi import PowerBITrigger from airflow.utils import timezone -from tests.providers.microsoft.azure.base import Base -from tests.providers.microsoft.conftest import get_airflow_connection, mock_context + +from providers.tests.microsoft.azure.base import Base +from providers.tests.microsoft.conftest import get_airflow_connection, mock_context DEFAULT_CONNECTION_CLIENT_SECRET = "powerbi_conn_id" TASK_ID = "run_powerbi_operator" diff --git a/tests/providers/microsoft/azure/operators/test_synapse.py b/providers/tests/microsoft/azure/operators/test_synapse.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_synapse.py rename to providers/tests/microsoft/azure/operators/test_synapse.py diff --git a/tests/providers/microsoft/azure/operators/test_wasb_delete_blob.py b/providers/tests/microsoft/azure/operators/test_wasb_delete_blob.py similarity index 100% rename from tests/providers/microsoft/azure/operators/test_wasb_delete_blob.py rename to providers/tests/microsoft/azure/operators/test_wasb_delete_blob.py diff --git a/tests/providers/microsoft/azure/secrets/__init__.py b/providers/tests/microsoft/azure/resources/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/secrets/__init__.py rename to providers/tests/microsoft/azure/resources/__init__.py diff --git a/tests/providers/microsoft/azure/resources/dummy.pdf b/providers/tests/microsoft/azure/resources/dummy.pdf similarity index 100% rename from tests/providers/microsoft/azure/resources/dummy.pdf rename to providers/tests/microsoft/azure/resources/dummy.pdf diff --git a/tests/providers/microsoft/azure/resources/next_users.json b/providers/tests/microsoft/azure/resources/next_users.json similarity index 100% rename from tests/providers/microsoft/azure/resources/next_users.json rename to providers/tests/microsoft/azure/resources/next_users.json diff --git a/tests/providers/microsoft/azure/resources/status.json b/providers/tests/microsoft/azure/resources/status.json similarity index 100% rename from tests/providers/microsoft/azure/resources/status.json rename to providers/tests/microsoft/azure/resources/status.json diff --git a/tests/providers/microsoft/azure/resources/users.json b/providers/tests/microsoft/azure/resources/users.json similarity index 100% rename from tests/providers/microsoft/azure/resources/users.json rename to providers/tests/microsoft/azure/resources/users.json diff --git a/tests/providers/microsoft/azure/transfers/__init__.py b/providers/tests/microsoft/azure/secrets/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/transfers/__init__.py rename to providers/tests/microsoft/azure/secrets/__init__.py diff --git a/tests/providers/microsoft/azure/secrets/test_key_vault.py b/providers/tests/microsoft/azure/secrets/test_key_vault.py similarity index 100% rename from tests/providers/microsoft/azure/secrets/test_key_vault.py rename to providers/tests/microsoft/azure/secrets/test_key_vault.py diff --git a/tests/providers/microsoft/azure/hooks/__init__.py b/providers/tests/microsoft/azure/sensors/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/hooks/__init__.py rename to providers/tests/microsoft/azure/sensors/__init__.py diff --git a/tests/providers/microsoft/azure/sensors/test_cosmos.py b/providers/tests/microsoft/azure/sensors/test_cosmos.py similarity index 100% rename from tests/providers/microsoft/azure/sensors/test_cosmos.py rename to providers/tests/microsoft/azure/sensors/test_cosmos.py diff --git a/tests/providers/microsoft/azure/sensors/test_data_factory.py b/providers/tests/microsoft/azure/sensors/test_data_factory.py similarity index 100% rename from tests/providers/microsoft/azure/sensors/test_data_factory.py rename to providers/tests/microsoft/azure/sensors/test_data_factory.py diff --git a/tests/providers/microsoft/azure/sensors/test_msgraph.py b/providers/tests/microsoft/azure/sensors/test_msgraph.py similarity index 95% rename from tests/providers/microsoft/azure/sensors/test_msgraph.py rename to providers/tests/microsoft/azure/sensors/test_msgraph.py index e257984affb1a..ba5ba35478861 100644 --- a/tests/providers/microsoft/azure/sensors/test_msgraph.py +++ b/providers/tests/microsoft/azure/sensors/test_msgraph.py @@ -20,8 +20,9 @@ from airflow.providers.microsoft.azure.sensors.msgraph import MSGraphSensor from airflow.triggers.base import TriggerEvent -from tests.providers.microsoft.azure.base import Base -from tests.providers.microsoft.conftest import load_json, mock_json_response + +from providers.tests.microsoft.azure.base import Base +from providers.tests.microsoft.conftest import load_json, mock_json_response class TestMSGraphSensor(Base): diff --git a/tests/providers/microsoft/azure/sensors/test_wasb.py b/providers/tests/microsoft/azure/sensors/test_wasb.py similarity index 100% rename from tests/providers/microsoft/azure/sensors/test_wasb.py rename to providers/tests/microsoft/azure/sensors/test_wasb.py diff --git a/tests/providers/microsoft/azure/test_utils.py b/providers/tests/microsoft/azure/test_utils.py similarity index 100% rename from tests/providers/microsoft/azure/test_utils.py rename to providers/tests/microsoft/azure/test_utils.py diff --git a/tests/providers/microsoft/azure/triggers/__init__.py b/providers/tests/microsoft/azure/transfers/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/triggers/__init__.py rename to providers/tests/microsoft/azure/transfers/__init__.py diff --git a/tests/providers/microsoft/azure/transfers/test_local_to_adls.py b/providers/tests/microsoft/azure/transfers/test_local_to_adls.py similarity index 100% rename from tests/providers/microsoft/azure/transfers/test_local_to_adls.py rename to providers/tests/microsoft/azure/transfers/test_local_to_adls.py diff --git a/tests/providers/microsoft/azure/transfers/test_local_to_wasb.py b/providers/tests/microsoft/azure/transfers/test_local_to_wasb.py similarity index 100% rename from tests/providers/microsoft/azure/transfers/test_local_to_wasb.py rename to providers/tests/microsoft/azure/transfers/test_local_to_wasb.py diff --git a/tests/providers/microsoft/azure/transfers/test_oracle_to_azure_data_lake.py b/providers/tests/microsoft/azure/transfers/test_oracle_to_azure_data_lake.py similarity index 100% rename from tests/providers/microsoft/azure/transfers/test_oracle_to_azure_data_lake.py rename to providers/tests/microsoft/azure/transfers/test_oracle_to_azure_data_lake.py diff --git a/tests/providers/microsoft/azure/transfers/test_s3_to_wasb.py b/providers/tests/microsoft/azure/transfers/test_s3_to_wasb.py similarity index 100% rename from tests/providers/microsoft/azure/transfers/test_s3_to_wasb.py rename to providers/tests/microsoft/azure/transfers/test_s3_to_wasb.py diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/providers/tests/microsoft/azure/transfers/test_sftp_to_wasb.py similarity index 100% rename from tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py rename to providers/tests/microsoft/azure/transfers/test_sftp_to_wasb.py diff --git a/tests/providers/microsoft/mssql/operators/__init__.py b/providers/tests/microsoft/azure/triggers/__init__.py similarity index 100% rename from tests/providers/microsoft/mssql/operators/__init__.py rename to providers/tests/microsoft/azure/triggers/__init__.py diff --git a/tests/providers/microsoft/azure/triggers/test_data_factory.py b/providers/tests/microsoft/azure/triggers/test_data_factory.py similarity index 100% rename from tests/providers/microsoft/azure/triggers/test_data_factory.py rename to providers/tests/microsoft/azure/triggers/test_data_factory.py diff --git a/tests/providers/microsoft/azure/triggers/test_msgraph.py b/providers/tests/microsoft/azure/triggers/test_msgraph.py similarity index 98% rename from tests/providers/microsoft/azure/triggers/test_msgraph.py rename to providers/tests/microsoft/azure/triggers/test_msgraph.py index 23085563cf8f9..0784d8d83177c 100644 --- a/tests/providers/microsoft/azure/triggers/test_msgraph.py +++ b/providers/tests/microsoft/azure/triggers/test_msgraph.py @@ -31,8 +31,9 @@ ResponseSerializer, ) from airflow.triggers.base import TriggerEvent -from tests.providers.microsoft.azure.base import Base -from tests.providers.microsoft.conftest import ( + +from providers.tests.microsoft.azure.base import Base +from providers.tests.microsoft.conftest import ( get_airflow_connection, load_file, load_json, diff --git a/tests/providers/microsoft/azure/triggers/test_powerbi.py b/providers/tests/microsoft/azure/triggers/test_powerbi.py similarity index 99% rename from tests/providers/microsoft/azure/triggers/test_powerbi.py rename to providers/tests/microsoft/azure/triggers/test_powerbi.py index c3276e258b3da..8f5a3e945ffcd 100644 --- a/tests/providers/microsoft/azure/triggers/test_powerbi.py +++ b/providers/tests/microsoft/azure/triggers/test_powerbi.py @@ -25,7 +25,8 @@ from airflow.providers.microsoft.azure.hooks.powerbi import PowerBIDatasetRefreshStatus from airflow.providers.microsoft.azure.triggers.powerbi import PowerBITrigger from airflow.triggers.base import TriggerEvent -from tests.providers.microsoft.conftest import get_airflow_connection + +from providers.tests.microsoft.conftest import get_airflow_connection POWERBI_CONN_ID = "powerbi_default" DATASET_ID = "dataset_id" diff --git a/tests/providers/microsoft/azure/triggers/test_wasb.py b/providers/tests/microsoft/azure/triggers/test_wasb.py similarity index 100% rename from tests/providers/microsoft/azure/triggers/test_wasb.py rename to providers/tests/microsoft/azure/triggers/test_wasb.py diff --git a/tests/providers/microsoft/conftest.py b/providers/tests/microsoft/conftest.py similarity index 100% rename from tests/providers/microsoft/conftest.py rename to providers/tests/microsoft/conftest.py diff --git a/tests/providers/microsoft/azure/operators/__init__.py b/providers/tests/microsoft/mssql/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/operators/__init__.py rename to providers/tests/microsoft/mssql/__init__.py diff --git a/tests/providers/microsoft/azure/sensors/__init__.py b/providers/tests/microsoft/mssql/hooks/__init__.py similarity index 100% rename from tests/providers/microsoft/azure/sensors/__init__.py rename to providers/tests/microsoft/mssql/hooks/__init__.py diff --git a/tests/providers/microsoft/mssql/hooks/test_mssql.py b/providers/tests/microsoft/mssql/hooks/test_mssql.py similarity index 99% rename from tests/providers/microsoft/mssql/hooks/test_mssql.py rename to providers/tests/microsoft/mssql/hooks/test_mssql.py index c629bf7b0d9a4..1b43bb787835b 100644 --- a/tests/providers/microsoft/mssql/hooks/test_mssql.py +++ b/providers/tests/microsoft/mssql/hooks/test_mssql.py @@ -23,7 +23,8 @@ import pytest from airflow.models import Connection -from tests.providers.microsoft.conftest import load_file + +from providers.tests.microsoft.conftest import load_file try: from airflow.providers.microsoft.mssql.hooks.mssql import MsSqlHook diff --git a/tests/providers/mysql/assets/__init__.py b/providers/tests/microsoft/mssql/operators/__init__.py similarity index 100% rename from tests/providers/mysql/assets/__init__.py rename to providers/tests/microsoft/mssql/operators/__init__.py diff --git a/tests/providers/microsoft/mssql/operators/test_mssql.py b/providers/tests/microsoft/mssql/operators/test_mssql.py similarity index 100% rename from tests/providers/microsoft/mssql/operators/test_mssql.py rename to providers/tests/microsoft/mssql/operators/test_mssql.py diff --git a/tests/providers/microsoft/mssql/__init__.py b/providers/tests/microsoft/mssql/resources/__init__.py similarity index 100% rename from tests/providers/microsoft/mssql/__init__.py rename to providers/tests/microsoft/mssql/resources/__init__.py diff --git a/tests/providers/microsoft/mssql/resources/replace.sql b/providers/tests/microsoft/mssql/resources/replace.sql similarity index 100% rename from tests/providers/microsoft/mssql/resources/replace.sql rename to providers/tests/microsoft/mssql/resources/replace.sql diff --git a/tests/providers/microsoft/mssql/hooks/__init__.py b/providers/tests/microsoft/psrp/__init__.py similarity index 100% rename from tests/providers/microsoft/mssql/hooks/__init__.py rename to providers/tests/microsoft/psrp/__init__.py diff --git a/tests/providers/microsoft/mssql/resources/__init__.py b/providers/tests/microsoft/psrp/hooks/__init__.py similarity index 100% rename from tests/providers/microsoft/mssql/resources/__init__.py rename to providers/tests/microsoft/psrp/hooks/__init__.py diff --git a/tests/providers/microsoft/psrp/hooks/test_psrp.py b/providers/tests/microsoft/psrp/hooks/test_psrp.py similarity index 96% rename from tests/providers/microsoft/psrp/hooks/test_psrp.py rename to providers/tests/microsoft/psrp/hooks/test_psrp.py index fe271f5c39e55..1f56e67a7a689 100644 --- a/tests/providers/microsoft/psrp/hooks/test_psrp.py +++ b/providers/tests/microsoft/psrp/hooks/test_psrp.py @@ -139,13 +139,16 @@ def test_invoke(self, runspace_pool, powershell, ws_man, logging_level): on_output_callback = Mock() - with PsrpHook( - CONNECTION_ID, - runspace_options=runspace_options, - wsman_options=wsman_options, - on_output_callback=on_output_callback, - **options, - ) as hook, patch.object(type(hook), "log") as logger: + with ( + PsrpHook( + CONNECTION_ID, + runspace_options=runspace_options, + wsman_options=wsman_options, + on_output_callback=on_output_callback, + **options, + ) as hook, + patch.object(type(hook), "log") as logger, + ): error_match = "Process had one or more errors" with pytest.raises(AirflowException, match=error_match): # noqa: PT012 error happen on context exit with hook.invoke() as ps: diff --git a/tests/providers/microsoft/psrp/__init__.py b/providers/tests/microsoft/psrp/operators/__init__.py similarity index 100% rename from tests/providers/microsoft/psrp/__init__.py rename to providers/tests/microsoft/psrp/operators/__init__.py diff --git a/tests/providers/microsoft/psrp/operators/test_psrp.py b/providers/tests/microsoft/psrp/operators/test_psrp.py similarity index 100% rename from tests/providers/microsoft/psrp/operators/test_psrp.py rename to providers/tests/microsoft/psrp/operators/test_psrp.py diff --git a/tests/providers/microsoft/psrp/hooks/__init__.py b/providers/tests/microsoft/winrm/__init__.py similarity index 100% rename from tests/providers/microsoft/psrp/hooks/__init__.py rename to providers/tests/microsoft/winrm/__init__.py diff --git a/tests/providers/microsoft/psrp/operators/__init__.py b/providers/tests/microsoft/winrm/hooks/__init__.py similarity index 100% rename from tests/providers/microsoft/psrp/operators/__init__.py rename to providers/tests/microsoft/winrm/hooks/__init__.py diff --git a/tests/providers/microsoft/winrm/hooks/test_winrm.py b/providers/tests/microsoft/winrm/hooks/test_winrm.py similarity index 100% rename from tests/providers/microsoft/winrm/hooks/test_winrm.py rename to providers/tests/microsoft/winrm/hooks/test_winrm.py diff --git a/tests/providers/microsoft/winrm/__init__.py b/providers/tests/microsoft/winrm/operators/__init__.py similarity index 100% rename from tests/providers/microsoft/winrm/__init__.py rename to providers/tests/microsoft/winrm/operators/__init__.py diff --git a/tests/providers/microsoft/winrm/operators/test_winrm.py b/providers/tests/microsoft/winrm/operators/test_winrm.py similarity index 100% rename from tests/providers/microsoft/winrm/operators/test_winrm.py rename to providers/tests/microsoft/winrm/operators/test_winrm.py diff --git a/tests/providers/microsoft/winrm/hooks/__init__.py b/providers/tests/mongo/__init__.py similarity index 100% rename from tests/providers/microsoft/winrm/hooks/__init__.py rename to providers/tests/mongo/__init__.py diff --git a/tests/providers/microsoft/winrm/operators/__init__.py b/providers/tests/mongo/hooks/__init__.py similarity index 100% rename from tests/providers/microsoft/winrm/operators/__init__.py rename to providers/tests/mongo/hooks/__init__.py diff --git a/tests/providers/mongo/hooks/test_mongo.py b/providers/tests/mongo/hooks/test_mongo.py similarity index 99% rename from tests/providers/mongo/hooks/test_mongo.py rename to providers/tests/mongo/hooks/test_mongo.py index 86b49535fa49b..78756b0cc4890 100644 --- a/tests/providers/mongo/hooks/test_mongo.py +++ b/providers/tests/mongo/hooks/test_mongo.py @@ -27,7 +27,8 @@ from airflow.exceptions import AirflowConfigException, AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.mongo.hooks.mongo import MongoHook -from tests.test_utils.compat import connection_as_json + +from dev.tests_common.test_utils.compat import connection_as_json pytestmark = pytest.mark.db_test diff --git a/tests/providers/mongo/__init__.py b/providers/tests/mongo/sensors/__init__.py similarity index 100% rename from tests/providers/mongo/__init__.py rename to providers/tests/mongo/sensors/__init__.py diff --git a/tests/providers/mongo/sensors/test_mongo.py b/providers/tests/mongo/sensors/test_mongo.py similarity index 100% rename from tests/providers/mongo/sensors/test_mongo.py rename to providers/tests/mongo/sensors/test_mongo.py diff --git a/tests/providers/mongo/hooks/__init__.py b/providers/tests/mysql/__init__.py similarity index 100% rename from tests/providers/mongo/hooks/__init__.py rename to providers/tests/mysql/__init__.py diff --git a/tests/providers/mysql/transfers/__init__.py b/providers/tests/mysql/assets/__init__.py similarity index 100% rename from tests/providers/mysql/transfers/__init__.py rename to providers/tests/mysql/assets/__init__.py diff --git a/tests/providers/mysql/assets/test_mysql.py b/providers/tests/mysql/assets/test_mysql.py similarity index 100% rename from tests/providers/mysql/assets/test_mysql.py rename to providers/tests/mysql/assets/test_mysql.py diff --git a/tests/providers/mongo/sensors/__init__.py b/providers/tests/mysql/hooks/__init__.py similarity index 100% rename from tests/providers/mongo/sensors/__init__.py rename to providers/tests/mysql/hooks/__init__.py diff --git a/tests/providers/mysql/hooks/test_mysql.py b/providers/tests/mysql/hooks/test_mysql.py similarity index 99% rename from tests/providers/mysql/hooks/test_mysql.py rename to providers/tests/mysql/hooks/test_mysql.py index 48fc62fe2c220..23748ae9cac19 100644 --- a/tests/providers/mysql/hooks/test_mysql.py +++ b/providers/tests/mysql/hooks/test_mysql.py @@ -37,7 +37,8 @@ from airflow.utils import timezone -from tests.test_utils.asserts import assert_equal_ignore_multiple_spaces + +from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces SSL_DICT = {"cert": "/tmp/client-cert.pem", "ca": "/tmp/server-ca.pem", "key": "/tmp/client-key.pem"} diff --git a/tests/providers/mysql/hooks/test_mysql_connector_python.py b/providers/tests/mysql/hooks/test_mysql_connector_python.py similarity index 100% rename from tests/providers/mysql/hooks/test_mysql_connector_python.py rename to providers/tests/mysql/hooks/test_mysql_connector_python.py diff --git a/tests/providers/mysql/__init__.py b/providers/tests/mysql/operators/__init__.py similarity index 100% rename from tests/providers/mysql/__init__.py rename to providers/tests/mysql/operators/__init__.py diff --git a/tests/providers/mysql/operators/test_mysql.py b/providers/tests/mysql/operators/test_mysql.py similarity index 99% rename from tests/providers/mysql/operators/test_mysql.py rename to providers/tests/mysql/operators/test_mysql.py index 10a1fcc151a82..75f0aed2935f3 100644 --- a/tests/providers/mysql/operators/test_mysql.py +++ b/providers/tests/mysql/operators/test_mysql.py @@ -34,7 +34,8 @@ from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.mysql.hooks.mysql import MySqlHook from airflow.utils import timezone -from tests.providers.mysql.hooks.test_mysql import MySqlContext + +from providers.tests.mysql.hooks.test_mysql import MySqlContext DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() diff --git a/tests/providers/odbc/__init__.py b/providers/tests/mysql/transfers/__init__.py similarity index 100% rename from tests/providers/odbc/__init__.py rename to providers/tests/mysql/transfers/__init__.py diff --git a/tests/providers/mysql/transfers/test_presto_to_mysql.py b/providers/tests/mysql/transfers/test_presto_to_mysql.py similarity index 100% rename from tests/providers/mysql/transfers/test_presto_to_mysql.py rename to providers/tests/mysql/transfers/test_presto_to_mysql.py diff --git a/tests/providers/mysql/transfers/test_s3_to_mysql.py b/providers/tests/mysql/transfers/test_s3_to_mysql.py similarity index 100% rename from tests/providers/mysql/transfers/test_s3_to_mysql.py rename to providers/tests/mysql/transfers/test_s3_to_mysql.py diff --git a/tests/providers/mysql/transfers/test_trino_to_mysql.py b/providers/tests/mysql/transfers/test_trino_to_mysql.py similarity index 100% rename from tests/providers/mysql/transfers/test_trino_to_mysql.py rename to providers/tests/mysql/transfers/test_trino_to_mysql.py diff --git a/tests/providers/mysql/transfers/test_vertica_to_mysql.py b/providers/tests/mysql/transfers/test_vertica_to_mysql.py similarity index 100% rename from tests/providers/mysql/transfers/test_vertica_to_mysql.py rename to providers/tests/mysql/transfers/test_vertica_to_mysql.py diff --git a/tests/providers/mysql/hooks/__init__.py b/providers/tests/neo4j/__init__.py similarity index 100% rename from tests/providers/mysql/hooks/__init__.py rename to providers/tests/neo4j/__init__.py diff --git a/tests/providers/mysql/operators/__init__.py b/providers/tests/neo4j/hooks/__init__.py similarity index 100% rename from tests/providers/mysql/operators/__init__.py rename to providers/tests/neo4j/hooks/__init__.py diff --git a/tests/providers/neo4j/hooks/test_neo4j.py b/providers/tests/neo4j/hooks/test_neo4j.py similarity index 100% rename from tests/providers/neo4j/hooks/test_neo4j.py rename to providers/tests/neo4j/hooks/test_neo4j.py diff --git a/tests/providers/neo4j/__init__.py b/providers/tests/neo4j/operators/__init__.py similarity index 100% rename from tests/providers/neo4j/__init__.py rename to providers/tests/neo4j/operators/__init__.py diff --git a/tests/providers/neo4j/operators/test_neo4j.py b/providers/tests/neo4j/operators/test_neo4j.py similarity index 100% rename from tests/providers/neo4j/operators/test_neo4j.py rename to providers/tests/neo4j/operators/test_neo4j.py diff --git a/tests/providers/odbc/hooks/__init__.py b/providers/tests/odbc/__init__.py similarity index 100% rename from tests/providers/odbc/hooks/__init__.py rename to providers/tests/odbc/__init__.py diff --git a/tests/providers/openai/__init__.py b/providers/tests/odbc/hooks/__init__.py similarity index 100% rename from tests/providers/openai/__init__.py rename to providers/tests/odbc/hooks/__init__.py diff --git a/tests/providers/odbc/hooks/test_odbc.py b/providers/tests/odbc/hooks/test_odbc.py similarity index 99% rename from tests/providers/odbc/hooks/test_odbc.py rename to providers/tests/odbc/hooks/test_odbc.py index bddd2ffd996b6..8f749aa4f765d 100644 --- a/tests/providers/odbc/hooks/test_odbc.py +++ b/providers/tests/odbc/hooks/test_odbc.py @@ -28,7 +28,8 @@ import pytest from airflow.providers.odbc.hooks.odbc import OdbcHook -from tests.providers.common.sql.test_utils import mock_hook + +from providers.tests.common.sql.test_utils import mock_hook @pytest.fixture diff --git a/tests/providers/openai/hooks/__init__.py b/providers/tests/openai/__init__.py similarity index 100% rename from tests/providers/openai/hooks/__init__.py rename to providers/tests/openai/__init__.py diff --git a/tests/providers/openai/operators/__init__.py b/providers/tests/openai/hooks/__init__.py similarity index 100% rename from tests/providers/openai/operators/__init__.py rename to providers/tests/openai/hooks/__init__.py diff --git a/tests/providers/openai/hooks/test_openai.py b/providers/tests/openai/hooks/test_openai.py similarity index 100% rename from tests/providers/openai/hooks/test_openai.py rename to providers/tests/openai/hooks/test_openai.py diff --git a/tests/providers/openai/triggers/__init__.py b/providers/tests/openai/operators/__init__.py similarity index 100% rename from tests/providers/openai/triggers/__init__.py rename to providers/tests/openai/operators/__init__.py diff --git a/tests/providers/openai/operators/test_openai.py b/providers/tests/openai/operators/test_openai.py similarity index 100% rename from tests/providers/openai/operators/test_openai.py rename to providers/tests/openai/operators/test_openai.py diff --git a/tests/providers/openai/test_exceptions.py b/providers/tests/openai/test_exceptions.py similarity index 100% rename from tests/providers/openai/test_exceptions.py rename to providers/tests/openai/test_exceptions.py diff --git a/tests/providers/openlineage/__init__.py b/providers/tests/openai/triggers/__init__.py similarity index 100% rename from tests/providers/openlineage/__init__.py rename to providers/tests/openai/triggers/__init__.py diff --git a/tests/providers/openai/triggers/test_openai.py b/providers/tests/openai/triggers/test_openai.py similarity index 100% rename from tests/providers/openai/triggers/test_openai.py rename to providers/tests/openai/triggers/test_openai.py diff --git a/tests/providers/neo4j/hooks/__init__.py b/providers/tests/openfaas/__init__.py similarity index 100% rename from tests/providers/neo4j/hooks/__init__.py rename to providers/tests/openfaas/__init__.py diff --git a/tests/providers/neo4j/operators/__init__.py b/providers/tests/openfaas/hooks/__init__.py similarity index 100% rename from tests/providers/neo4j/operators/__init__.py rename to providers/tests/openfaas/hooks/__init__.py diff --git a/tests/providers/openfaas/hooks/test_openfaas.py b/providers/tests/openfaas/hooks/test_openfaas.py similarity index 100% rename from tests/providers/openfaas/hooks/test_openfaas.py rename to providers/tests/openfaas/hooks/test_openfaas.py diff --git a/tests/providers/openlineage/extractors/__init__.py b/providers/tests/openlineage/__init__.py similarity index 100% rename from tests/providers/openlineage/extractors/__init__.py rename to providers/tests/openlineage/__init__.py diff --git a/tests/providers/openlineage/plugins/__init__.py b/providers/tests/openlineage/extractors/__init__.py similarity index 100% rename from tests/providers/openlineage/plugins/__init__.py rename to providers/tests/openlineage/extractors/__init__.py diff --git a/tests/providers/openlineage/extractors/test_base.py b/providers/tests/openlineage/extractors/test_base.py similarity index 99% rename from tests/providers/openlineage/extractors/test_base.py rename to providers/tests/openlineage/extractors/test_base.py index 88234d317472e..15c96ac675530 100644 --- a/tests/providers/openlineage/extractors/test_base.py +++ b/providers/tests/openlineage/extractors/test_base.py @@ -277,7 +277,7 @@ def test_extract_on_failure(task_state, is_airflow_2_10_or_higher, should_call_o @mock.patch("airflow.providers.openlineage.conf.custom_extractors") def test_extractors_env_var(custom_extractors): - custom_extractors.return_value = {"tests.providers.openlineage.extractors.test_base.ExampleExtractor"} + custom_extractors.return_value = {"providers.tests.openlineage.extractors.test_base.ExampleExtractor"} extractor = ExtractorManager().get_extractor_class(ExampleOperator(task_id="example")) assert extractor is ExampleExtractor diff --git a/tests/providers/openlineage/extractors/test_bash.py b/providers/tests/openlineage/extractors/test_bash.py similarity index 98% rename from tests/providers/openlineage/extractors/test_bash.py rename to providers/tests/openlineage/extractors/test_bash.py index fc862e5ee30b9..d4fcdf7af2b7a 100644 --- a/tests/providers/openlineage/extractors/test_bash.py +++ b/providers/tests/openlineage/extractors/test_bash.py @@ -27,7 +27,8 @@ from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.openlineage.extractors.bash import BashExtractor -from tests.test_utils.compat import BashOperator + +from dev.tests_common.test_utils.compat import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/extractors/test_manager.py b/providers/tests/openlineage/extractors/test_manager.py similarity index 99% rename from tests/providers/openlineage/extractors/test_manager.py rename to providers/tests/openlineage/extractors/test_manager.py index 601a456604843..6bbf303e32857 100644 --- a/tests/providers/openlineage/extractors/test_manager.py +++ b/providers/tests/openlineage/extractors/test_manager.py @@ -34,7 +34,8 @@ from airflow.providers.openlineage.extractors.manager import ExtractorManager from airflow.providers.openlineage.utils.utils import Asset from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/tests/providers/openlineage/extractors/test_python.py b/providers/tests/openlineage/extractors/test_python.py similarity index 98% rename from tests/providers/openlineage/extractors/test_python.py rename to providers/tests/openlineage/extractors/test_python.py index 44c5503b712d8..ef4fc0b861e7c 100644 --- a/tests/providers/openlineage/extractors/test_python.py +++ b/providers/tests/openlineage/extractors/test_python.py @@ -30,7 +30,8 @@ from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.operators.python import PythonOperator from airflow.providers.openlineage.extractors.python import PythonExtractor -from tests.test_utils.compat import BashOperator + +from dev.tests_common.test_utils.compat import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/log_config.py b/providers/tests/openlineage/log_config.py similarity index 100% rename from tests/providers/openlineage/log_config.py rename to providers/tests/openlineage/log_config.py diff --git a/tests/providers/openlineage/plugins/openlineage_configs/__init__.py b/providers/tests/openlineage/plugins/__init__.py similarity index 100% rename from tests/providers/openlineage/plugins/openlineage_configs/__init__.py rename to providers/tests/openlineage/plugins/__init__.py diff --git a/tests/providers/openlineage/utils/__init__.py b/providers/tests/openlineage/plugins/openlineage_configs/__init__.py similarity index 100% rename from tests/providers/openlineage/utils/__init__.py rename to providers/tests/openlineage/plugins/openlineage_configs/__init__.py diff --git a/tests/providers/openlineage/plugins/openlineage_configs/http.yaml b/providers/tests/openlineage/plugins/openlineage_configs/http.yaml similarity index 100% rename from tests/providers/openlineage/plugins/openlineage_configs/http.yaml rename to providers/tests/openlineage/plugins/openlineage_configs/http.yaml diff --git a/tests/providers/openlineage/plugins/test_adapter.py b/providers/tests/openlineage/plugins/test_adapter.py similarity index 99% rename from tests/providers/openlineage/plugins/test_adapter.py rename to providers/tests/openlineage/plugins/test_adapter.py index b01fe46fdca13..88f2250c638ef 100644 --- a/tests/providers/openlineage/plugins/test_adapter.py +++ b/providers/tests/openlineage/plugins/test_adapter.py @@ -51,8 +51,9 @@ ) from airflow.providers.openlineage.utils.utils import get_airflow_job_facet from airflow.utils.task_group import TaskGroup -from tests.test_utils.compat import BashOperator -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import BashOperator +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/plugins/test_execution.py b/providers/tests/openlineage/plugins/test_execution.py similarity index 98% rename from tests/providers/openlineage/plugins/test_execution.py rename to providers/tests/openlineage/plugins/test_execution.py index 8c0bdd55a1f96..c308047b1bc0d 100644 --- a/tests/providers/openlineage/plugins/test_execution.py +++ b/providers/tests/openlineage/plugins/test_execution.py @@ -36,8 +36,9 @@ from airflow.task.task_runner.standard_task_runner import StandardTaskRunner from airflow.utils import timezone from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/openlineage/plugins/test_facets.py b/providers/tests/openlineage/plugins/test_facets.py similarity index 100% rename from tests/providers/openlineage/plugins/test_facets.py rename to providers/tests/openlineage/plugins/test_facets.py diff --git a/tests/providers/openlineage/plugins/test_listener.py b/providers/tests/openlineage/plugins/test_listener.py similarity index 99% rename from tests/providers/openlineage/plugins/test_listener.py rename to providers/tests/openlineage/plugins/test_listener.py index 57c0134f79d82..0c1651ccf0277 100644 --- a/tests/providers/openlineage/plugins/test_listener.py +++ b/providers/tests/openlineage/plugins/test_listener.py @@ -38,8 +38,9 @@ from airflow.providers.openlineage.plugins.listener import OpenLineageListener from airflow.providers.openlineage.utils.selective_enable import disable_lineage, enable_lineage from airflow.utils.state import DagRunState, State -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/openlineage/plugins/test_macros.py b/providers/tests/openlineage/plugins/test_macros.py similarity index 100% rename from tests/providers/openlineage/plugins/test_macros.py rename to providers/tests/openlineage/plugins/test_macros.py diff --git a/tests/providers/openlineage/plugins/test_openlineage.py b/providers/tests/openlineage/plugins/test_openlineage.py similarity index 97% rename from tests/providers/openlineage/plugins/test_openlineage.py rename to providers/tests/openlineage/plugins/test_openlineage.py index dcb8198ceccad..8685326a29432 100644 --- a/tests/providers/openlineage/plugins/test_openlineage.py +++ b/providers/tests/openlineage/plugins/test_openlineage.py @@ -23,8 +23,8 @@ import pytest -from tests.conftest import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from tests.test_utils.config import conf_vars +from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from dev.tests_common.test_utils.config import conf_vars @pytest.mark.skipif( diff --git a/tests/providers/openlineage/plugins/test_utils.py b/providers/tests/openlineage/plugins/test_utils.py similarity index 99% rename from tests/providers/openlineage/plugins/test_utils.py rename to providers/tests/openlineage/plugins/test_utils.py index 65874a5ecebf3..444839ccaef30 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/providers/tests/openlineage/plugins/test_utils.py @@ -43,7 +43,8 @@ from airflow.utils import timezone from airflow.utils.log.secrets_masker import _secrets_masker from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/openlineage/test_conf.py b/providers/tests/openlineage/test_conf.py similarity index 99% rename from tests/providers/openlineage/test_conf.py rename to providers/tests/openlineage/test_conf.py index f3c483d4ce142..7f78a6a4c2eea 100644 --- a/tests/providers/openlineage/test_conf.py +++ b/providers/tests/openlineage/test_conf.py @@ -38,7 +38,8 @@ selective_enable, transport, ) -from tests.test_utils.config import conf_vars, env_vars + +from dev.tests_common.test_utils.config import conf_vars, env_vars _CONFIG_SECTION = "openlineage" _VAR_CONFIG_PATH = "OPENLINEAGE_CONFIG" diff --git a/tests/providers/openlineage/test_sqlparser.py b/providers/tests/openlineage/test_sqlparser.py similarity index 100% rename from tests/providers/openlineage/test_sqlparser.py rename to providers/tests/openlineage/test_sqlparser.py diff --git a/tests/providers/opensearch/__init__.py b/providers/tests/openlineage/utils/__init__.py similarity index 100% rename from tests/providers/opensearch/__init__.py rename to providers/tests/openlineage/utils/__init__.py diff --git a/tests/providers/openlineage/utils/custom_facet_fixture.py b/providers/tests/openlineage/utils/custom_facet_fixture.py similarity index 100% rename from tests/providers/openlineage/utils/custom_facet_fixture.py rename to providers/tests/openlineage/utils/custom_facet_fixture.py diff --git a/tests/providers/openlineage/utils/test_selective_enable.py b/providers/tests/openlineage/utils/test_selective_enable.py similarity index 100% rename from tests/providers/openlineage/utils/test_selective_enable.py rename to providers/tests/openlineage/utils/test_selective_enable.py diff --git a/tests/providers/openlineage/utils/test_sql.py b/providers/tests/openlineage/utils/test_sql.py similarity index 100% rename from tests/providers/openlineage/utils/test_sql.py rename to providers/tests/openlineage/utils/test_sql.py diff --git a/tests/providers/openlineage/utils/test_utils.py b/providers/tests/openlineage/utils/test_utils.py similarity index 96% rename from tests/providers/openlineage/utils/test_utils.py rename to providers/tests/openlineage/utils/test_utils.py index 20eba76adeb18..6cf7904546aba 100644 --- a/tests/providers/openlineage/utils/test_utils.py +++ b/providers/tests/openlineage/utils/test_utils.py @@ -43,8 +43,9 @@ from airflow.serialization.serialized_objects import SerializedBaseOperator from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator +from dev.tests_common.test_utils.mock_operators import MockOperator BASH_OPERATOR_PATH = "airflow.providers.standard.operators.bash" if not AIRFLOW_V_2_10_PLUS: @@ -190,7 +191,7 @@ def test_get_fully_qualified_class_name_mapped_operator(): mapped = MockOperator.partial(task_id="task_2").expand(arg2=["a", "b", "c"]) assert isinstance(mapped, MappedOperator) mapped_op_path = get_fully_qualified_class_name(mapped) - assert mapped_op_path == "tests.test_utils.mock_operators.MockOperator" + assert mapped_op_path == "dev.tests_common.test_utils.mock_operators.MockOperator" def test_get_fully_qualified_class_name_bash_operator(): @@ -309,7 +310,7 @@ def sum_values(values: list[int]) -> int: "downstream_task_ids": [], }, "task": { - "operator": "tests.providers.openlineage.utils.test_utils.CustomOperatorForTest", + "operator": "providers.tests.openlineage.utils.test_utils.CustomOperatorForTest", "task_group": None, "emits_ol_events": True, "ui_color": CustomOperatorForTest.ui_color, @@ -337,7 +338,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_1": { - "operator": "tests.providers.openlineage.utils.test_utils.CustomOperatorFromEmpty", + "operator": "providers.tests.openlineage.utils.test_utils.CustomOperatorFromEmpty", "task_group": None, "emits_ol_events": False, "ui_color": CustomOperatorFromEmpty.ui_color, @@ -406,7 +407,7 @@ def sum_values(values: list[int]) -> int: "emits_ol_events": True, "is_setup": False, "is_teardown": False, - "operator": "tests.providers.openlineage.utils.test_utils.TestMappedOperator", + "operator": "providers.tests.openlineage.utils.test_utils.TestMappedOperator", "task_group": None, "ui_color": "#fff", "ui_fgcolor": "#000", @@ -597,7 +598,7 @@ def test_get_user_provided_run_facets_with_no_function_definition(mock_custom_fa @patch( "airflow.providers.openlineage.conf.custom_run_facets", - return_value={"tests.providers.openlineage.utils.custom_facet_fixture.get_additional_test_facet"}, + return_value={"providers.tests.openlineage.utils.custom_facet_fixture.get_additional_test_facet"}, ) def test_get_user_provided_run_facets_with_function_definition(mock_custom_facet_funcs): sample_ti = TaskInstance( @@ -615,7 +616,7 @@ def test_get_user_provided_run_facets_with_function_definition(mock_custom_facet @patch( "airflow.providers.openlineage.conf.custom_run_facets", return_value={ - "tests.providers.openlineage.utils.custom_facet_fixture.get_additional_test_facet", + "providers.tests.openlineage.utils.custom_facet_fixture.get_additional_test_facet", }, ) def test_get_user_provided_run_facets_with_return_value_as_none(mock_custom_facet_funcs): @@ -635,9 +636,9 @@ def test_get_user_provided_run_facets_with_return_value_as_none(mock_custom_face "airflow.providers.openlineage.conf.custom_run_facets", return_value={ "invalid_function", - "tests.providers.openlineage.utils.custom_facet_fixture.get_additional_test_facet", - "tests.providers.openlineage.utils.custom_facet_fixture.return_type_is_not_dict", - "tests.providers.openlineage.utils.custom_facet_fixture.get_another_test_facet", + "providers.tests.openlineage.utils.custom_facet_fixture.get_additional_test_facet", + "providers.tests.openlineage.utils.custom_facet_fixture.return_type_is_not_dict", + "providers.tests.openlineage.utils.custom_facet_fixture.get_another_test_facet", }, ) def test_get_user_provided_run_facets_with_multiple_function_definition(mock_custom_facet_funcs): @@ -657,8 +658,8 @@ def test_get_user_provided_run_facets_with_multiple_function_definition(mock_cus @patch( "airflow.providers.openlineage.conf.custom_run_facets", return_value={ - "tests.providers.openlineage.utils.custom_facet_fixture.get_additional_test_facet", - "tests.providers.openlineage.utils.custom_facet_fixture.get_duplicate_test_facet_key", + "providers.tests.openlineage.utils.custom_facet_fixture.get_additional_test_facet", + "providers.tests.openlineage.utils.custom_facet_fixture.get_duplicate_test_facet_key", }, ) def test_get_user_provided_run_facets_with_duplicate_facet_keys(mock_custom_facet_funcs): @@ -691,7 +692,7 @@ def test_get_user_provided_run_facets_with_invalid_function_definition(mock_cust @patch( "airflow.providers.openlineage.conf.custom_run_facets", - return_value={"tests.providers.openlineage.utils.custom_facet_fixture.return_type_is_not_dict"}, + return_value={"providers.tests.openlineage.utils.custom_facet_fixture.return_type_is_not_dict"}, ) def test_get_user_provided_run_facets_with_wrong_return_type_function(mock_custom_facet_funcs): sample_ti = TaskInstance( @@ -706,7 +707,7 @@ def test_get_user_provided_run_facets_with_wrong_return_type_function(mock_custo @patch( "airflow.providers.openlineage.conf.custom_run_facets", - return_value={"tests.providers.openlineage.utils.custom_facet_fixture.get_custom_facet_throws_exception"}, + return_value={"providers.tests.openlineage.utils.custom_facet_fixture.get_custom_facet_throws_exception"}, ) def test_get_user_provided_run_facets_with_exception(mock_custom_facet_funcs): sample_ti = TaskInstance( diff --git a/tests/providers/opensearch/hooks/__init__.py b/providers/tests/opensearch/__init__.py similarity index 100% rename from tests/providers/opensearch/hooks/__init__.py rename to providers/tests/opensearch/__init__.py diff --git a/tests/providers/opensearch/conftest.py b/providers/tests/opensearch/conftest.py similarity index 100% rename from tests/providers/opensearch/conftest.py rename to providers/tests/opensearch/conftest.py diff --git a/tests/providers/opensearch/log/__init__.py b/providers/tests/opensearch/hooks/__init__.py similarity index 100% rename from tests/providers/opensearch/log/__init__.py rename to providers/tests/opensearch/hooks/__init__.py diff --git a/tests/providers/opensearch/hooks/test_opensearch.py b/providers/tests/opensearch/hooks/test_opensearch.py similarity index 100% rename from tests/providers/opensearch/hooks/test_opensearch.py rename to providers/tests/opensearch/hooks/test_opensearch.py diff --git a/tests/providers/opensearch/operators/__init__.py b/providers/tests/opensearch/log/__init__.py similarity index 100% rename from tests/providers/opensearch/operators/__init__.py rename to providers/tests/opensearch/log/__init__.py diff --git a/tests/providers/opensearch/log/test_os_json_formatter.py b/providers/tests/opensearch/log/test_os_json_formatter.py similarity index 100% rename from tests/providers/opensearch/log/test_os_json_formatter.py rename to providers/tests/opensearch/log/test_os_json_formatter.py diff --git a/tests/providers/opensearch/log/test_os_response.py b/providers/tests/opensearch/log/test_os_response.py similarity index 100% rename from tests/providers/opensearch/log/test_os_response.py rename to providers/tests/opensearch/log/test_os_response.py diff --git a/tests/providers/opensearch/log/test_os_task_handler.py b/providers/tests/opensearch/log/test_os_task_handler.py similarity index 99% rename from tests/providers/opensearch/log/test_os_task_handler.py rename to providers/tests/opensearch/log/test_os_task_handler.py index d23249ba9e6d9..1ffa0f5251191 100644 --- a/tests/providers/opensearch/log/test_os_task_handler.py +++ b/providers/tests/opensearch/log/test_os_task_handler.py @@ -43,9 +43,10 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import datetime -from tests.providers.opensearch.conftest import MockClient -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from providers.tests.opensearch.conftest import MockClient pytestmark = pytest.mark.db_test diff --git a/tests/providers/opsgenie/notifications/__init__.py b/providers/tests/opensearch/operators/__init__.py similarity index 100% rename from tests/providers/opsgenie/notifications/__init__.py rename to providers/tests/opensearch/operators/__init__.py diff --git a/tests/providers/opensearch/operators/test_opensearch.py b/providers/tests/opensearch/operators/test_opensearch.py similarity index 100% rename from tests/providers/opensearch/operators/test_opensearch.py rename to providers/tests/opensearch/operators/test_opensearch.py diff --git a/tests/providers/openfaas/__init__.py b/providers/tests/opsgenie/__init__.py similarity index 100% rename from tests/providers/openfaas/__init__.py rename to providers/tests/opsgenie/__init__.py diff --git a/tests/providers/openfaas/hooks/__init__.py b/providers/tests/opsgenie/hooks/__init__.py similarity index 100% rename from tests/providers/openfaas/hooks/__init__.py rename to providers/tests/opsgenie/hooks/__init__.py diff --git a/tests/providers/opsgenie/hooks/test_opsgenie.py b/providers/tests/opsgenie/hooks/test_opsgenie.py similarity index 100% rename from tests/providers/opsgenie/hooks/test_opsgenie.py rename to providers/tests/opsgenie/hooks/test_opsgenie.py diff --git a/tests/providers/opsgenie/typing/__init__.py b/providers/tests/opsgenie/notifications/__init__.py similarity index 100% rename from tests/providers/opsgenie/typing/__init__.py rename to providers/tests/opsgenie/notifications/__init__.py diff --git a/tests/providers/opsgenie/notifications/test_opsgenie.py b/providers/tests/opsgenie/notifications/test_opsgenie.py similarity index 100% rename from tests/providers/opsgenie/notifications/test_opsgenie.py rename to providers/tests/opsgenie/notifications/test_opsgenie.py diff --git a/tests/providers/opsgenie/__init__.py b/providers/tests/opsgenie/operators/__init__.py similarity index 100% rename from tests/providers/opsgenie/__init__.py rename to providers/tests/opsgenie/operators/__init__.py diff --git a/tests/providers/opsgenie/operators/test_opsgenie.py b/providers/tests/opsgenie/operators/test_opsgenie.py similarity index 100% rename from tests/providers/opsgenie/operators/test_opsgenie.py rename to providers/tests/opsgenie/operators/test_opsgenie.py diff --git a/tests/providers/oracle/operators/__init__.py b/providers/tests/opsgenie/typing/__init__.py similarity index 100% rename from tests/providers/oracle/operators/__init__.py rename to providers/tests/opsgenie/typing/__init__.py diff --git a/tests/providers/opsgenie/typing/test_opsgenie.py b/providers/tests/opsgenie/typing/test_opsgenie.py similarity index 100% rename from tests/providers/opsgenie/typing/test_opsgenie.py rename to providers/tests/opsgenie/typing/test_opsgenie.py diff --git a/tests/providers/opsgenie/hooks/__init__.py b/providers/tests/oracle/__init__.py similarity index 100% rename from tests/providers/opsgenie/hooks/__init__.py rename to providers/tests/oracle/__init__.py diff --git a/tests/providers/opsgenie/operators/__init__.py b/providers/tests/oracle/hooks/__init__.py similarity index 100% rename from tests/providers/opsgenie/operators/__init__.py rename to providers/tests/oracle/hooks/__init__.py diff --git a/tests/providers/oracle/hooks/test_oracle.py b/providers/tests/oracle/hooks/test_oracle.py similarity index 100% rename from tests/providers/oracle/hooks/test_oracle.py rename to providers/tests/oracle/hooks/test_oracle.py diff --git a/tests/providers/pagerduty/notifications/__init__.py b/providers/tests/oracle/operators/__init__.py similarity index 100% rename from tests/providers/pagerduty/notifications/__init__.py rename to providers/tests/oracle/operators/__init__.py diff --git a/tests/providers/oracle/operators/test_oracle.py b/providers/tests/oracle/operators/test_oracle.py similarity index 100% rename from tests/providers/oracle/operators/test_oracle.py rename to providers/tests/oracle/operators/test_oracle.py diff --git a/tests/providers/oracle/__init__.py b/providers/tests/oracle/transfers/__init__.py similarity index 100% rename from tests/providers/oracle/__init__.py rename to providers/tests/oracle/transfers/__init__.py diff --git a/tests/providers/oracle/transfers/test_oracle_to_oracle.py b/providers/tests/oracle/transfers/test_oracle_to_oracle.py similarity index 100% rename from tests/providers/oracle/transfers/test_oracle_to_oracle.py rename to providers/tests/oracle/transfers/test_oracle_to_oracle.py diff --git a/tests/providers/oracle/hooks/__init__.py b/providers/tests/pagerduty/__init__.py similarity index 100% rename from tests/providers/oracle/hooks/__init__.py rename to providers/tests/pagerduty/__init__.py diff --git a/tests/providers/oracle/transfers/__init__.py b/providers/tests/pagerduty/hooks/__init__.py similarity index 100% rename from tests/providers/oracle/transfers/__init__.py rename to providers/tests/pagerduty/hooks/__init__.py diff --git a/tests/providers/pagerduty/hooks/test_pagerduty.py b/providers/tests/pagerduty/hooks/test_pagerduty.py similarity index 100% rename from tests/providers/pagerduty/hooks/test_pagerduty.py rename to providers/tests/pagerduty/hooks/test_pagerduty.py diff --git a/tests/providers/pagerduty/hooks/test_pagerduty_events.py b/providers/tests/pagerduty/hooks/test_pagerduty_events.py similarity index 100% rename from tests/providers/pagerduty/hooks/test_pagerduty_events.py rename to providers/tests/pagerduty/hooks/test_pagerduty_events.py diff --git a/tests/providers/papermill/hooks/__init__.py b/providers/tests/pagerduty/notifications/__init__.py similarity index 100% rename from tests/providers/papermill/hooks/__init__.py rename to providers/tests/pagerduty/notifications/__init__.py diff --git a/tests/providers/pagerduty/notifications/test_pagerduty.py b/providers/tests/pagerduty/notifications/test_pagerduty.py similarity index 100% rename from tests/providers/pagerduty/notifications/test_pagerduty.py rename to providers/tests/pagerduty/notifications/test_pagerduty.py diff --git a/tests/providers/pagerduty/__init__.py b/providers/tests/papermill/__init__.py similarity index 100% rename from tests/providers/pagerduty/__init__.py rename to providers/tests/papermill/__init__.py diff --git a/tests/providers/pgvector/__init__.py b/providers/tests/papermill/hooks/__init__.py similarity index 100% rename from tests/providers/pgvector/__init__.py rename to providers/tests/papermill/hooks/__init__.py diff --git a/tests/providers/papermill/hooks/test_kernel.py b/providers/tests/papermill/hooks/test_kernel.py similarity index 100% rename from tests/providers/papermill/hooks/test_kernel.py rename to providers/tests/papermill/hooks/test_kernel.py diff --git a/tests/providers/pagerduty/hooks/__init__.py b/providers/tests/papermill/operators/__init__.py similarity index 100% rename from tests/providers/pagerduty/hooks/__init__.py rename to providers/tests/papermill/operators/__init__.py diff --git a/tests/providers/papermill/operators/test_papermill.py b/providers/tests/papermill/operators/test_papermill.py similarity index 100% rename from tests/providers/papermill/operators/test_papermill.py rename to providers/tests/papermill/operators/test_papermill.py diff --git a/tests/providers/pgvector/hooks/__init__.py b/providers/tests/pgvector/__init__.py similarity index 100% rename from tests/providers/pgvector/hooks/__init__.py rename to providers/tests/pgvector/__init__.py diff --git a/tests/providers/pgvector/operators/__init__.py b/providers/tests/pgvector/hooks/__init__.py similarity index 100% rename from tests/providers/pgvector/operators/__init__.py rename to providers/tests/pgvector/hooks/__init__.py diff --git a/tests/providers/pgvector/hooks/test_pgvector.py b/providers/tests/pgvector/hooks/test_pgvector.py similarity index 100% rename from tests/providers/pgvector/hooks/test_pgvector.py rename to providers/tests/pgvector/hooks/test_pgvector.py diff --git a/tests/providers/pinecone/__init__.py b/providers/tests/pgvector/operators/__init__.py similarity index 100% rename from tests/providers/pinecone/__init__.py rename to providers/tests/pgvector/operators/__init__.py diff --git a/tests/providers/pgvector/operators/test_pgvector.py b/providers/tests/pgvector/operators/test_pgvector.py similarity index 100% rename from tests/providers/pgvector/operators/test_pgvector.py rename to providers/tests/pgvector/operators/test_pgvector.py diff --git a/tests/providers/pinecone/hooks/__init__.py b/providers/tests/pinecone/__init__.py similarity index 100% rename from tests/providers/pinecone/hooks/__init__.py rename to providers/tests/pinecone/__init__.py diff --git a/tests/providers/pinecone/operators/__init__.py b/providers/tests/pinecone/hooks/__init__.py similarity index 100% rename from tests/providers/pinecone/operators/__init__.py rename to providers/tests/pinecone/hooks/__init__.py diff --git a/tests/providers/pinecone/hooks/test_pinecone.py b/providers/tests/pinecone/hooks/test_pinecone.py similarity index 100% rename from tests/providers/pinecone/hooks/test_pinecone.py rename to providers/tests/pinecone/hooks/test_pinecone.py diff --git a/tests/providers/postgres/assets/__init__.py b/providers/tests/pinecone/operators/__init__.py similarity index 100% rename from tests/providers/postgres/assets/__init__.py rename to providers/tests/pinecone/operators/__init__.py diff --git a/tests/providers/pinecone/operators/test_pinecone.py b/providers/tests/pinecone/operators/test_pinecone.py similarity index 100% rename from tests/providers/pinecone/operators/test_pinecone.py rename to providers/tests/pinecone/operators/test_pinecone.py diff --git a/tests/providers/papermill/__init__.py b/providers/tests/postgres/__init__.py similarity index 100% rename from tests/providers/papermill/__init__.py rename to providers/tests/postgres/__init__.py diff --git a/tests/providers/qdrant/__init__.py b/providers/tests/postgres/assets/__init__.py similarity index 100% rename from tests/providers/qdrant/__init__.py rename to providers/tests/postgres/assets/__init__.py diff --git a/tests/providers/postgres/assets/test_postgres.py b/providers/tests/postgres/assets/test_postgres.py similarity index 100% rename from tests/providers/postgres/assets/test_postgres.py rename to providers/tests/postgres/assets/test_postgres.py diff --git a/tests/providers/papermill/operators/__init__.py b/providers/tests/postgres/hooks/__init__.py similarity index 100% rename from tests/providers/papermill/operators/__init__.py rename to providers/tests/postgres/hooks/__init__.py diff --git a/tests/providers/postgres/hooks/test_postgres.py b/providers/tests/postgres/hooks/test_postgres.py similarity index 100% rename from tests/providers/postgres/hooks/test_postgres.py rename to providers/tests/postgres/hooks/test_postgres.py diff --git a/tests/providers/postgres/__init__.py b/providers/tests/postgres/operators/__init__.py similarity index 100% rename from tests/providers/postgres/__init__.py rename to providers/tests/postgres/operators/__init__.py diff --git a/tests/providers/postgres/operators/test_postgres.py b/providers/tests/postgres/operators/test_postgres.py similarity index 100% rename from tests/providers/postgres/operators/test_postgres.py rename to providers/tests/postgres/operators/test_postgres.py diff --git a/tests/providers/postgres/hooks/__init__.py b/providers/tests/presto/__init__.py similarity index 100% rename from tests/providers/postgres/hooks/__init__.py rename to providers/tests/presto/__init__.py diff --git a/tests/providers/postgres/operators/__init__.py b/providers/tests/presto/hooks/__init__.py similarity index 100% rename from tests/providers/postgres/operators/__init__.py rename to providers/tests/presto/hooks/__init__.py diff --git a/tests/providers/presto/hooks/test_presto.py b/providers/tests/presto/hooks/test_presto.py similarity index 100% rename from tests/providers/presto/hooks/test_presto.py rename to providers/tests/presto/hooks/test_presto.py diff --git a/tests/providers/presto/__init__.py b/providers/tests/presto/transfers/__init__.py similarity index 100% rename from tests/providers/presto/__init__.py rename to providers/tests/presto/transfers/__init__.py diff --git a/tests/providers/presto/transfers/test_gcs_to_presto.py b/providers/tests/presto/transfers/test_gcs_to_presto.py similarity index 100% rename from tests/providers/presto/transfers/test_gcs_to_presto.py rename to providers/tests/presto/transfers/test_gcs_to_presto.py diff --git a/tests/providers/qdrant/hooks/__init__.py b/providers/tests/qdrant/__init__.py similarity index 100% rename from tests/providers/qdrant/hooks/__init__.py rename to providers/tests/qdrant/__init__.py diff --git a/tests/providers/qdrant/operators/__init__.py b/providers/tests/qdrant/hooks/__init__.py similarity index 100% rename from tests/providers/qdrant/operators/__init__.py rename to providers/tests/qdrant/hooks/__init__.py diff --git a/tests/providers/qdrant/hooks/test_qdrant.py b/providers/tests/qdrant/hooks/test_qdrant.py similarity index 100% rename from tests/providers/qdrant/hooks/test_qdrant.py rename to providers/tests/qdrant/hooks/test_qdrant.py diff --git a/tests/providers/salesforce/operators/__init__.py b/providers/tests/qdrant/operators/__init__.py similarity index 100% rename from tests/providers/salesforce/operators/__init__.py rename to providers/tests/qdrant/operators/__init__.py diff --git a/tests/providers/qdrant/operators/test_qdrant.py b/providers/tests/qdrant/operators/test_qdrant.py similarity index 100% rename from tests/providers/qdrant/operators/test_qdrant.py rename to providers/tests/qdrant/operators/test_qdrant.py diff --git a/tests/providers/presto/hooks/__init__.py b/providers/tests/redis/__init__.py similarity index 100% rename from tests/providers/presto/hooks/__init__.py rename to providers/tests/redis/__init__.py diff --git a/tests/providers/presto/transfers/__init__.py b/providers/tests/redis/hooks/__init__.py similarity index 100% rename from tests/providers/presto/transfers/__init__.py rename to providers/tests/redis/hooks/__init__.py diff --git a/tests/providers/redis/hooks/test_redis.py b/providers/tests/redis/hooks/test_redis.py similarity index 100% rename from tests/providers/redis/hooks/test_redis.py rename to providers/tests/redis/hooks/test_redis.py diff --git a/tests/providers/redis/__init__.py b/providers/tests/redis/log/__init__.py similarity index 100% rename from tests/providers/redis/__init__.py rename to providers/tests/redis/log/__init__.py diff --git a/tests/providers/redis/log/test_redis_task_handler.py b/providers/tests/redis/log/test_redis_task_handler.py similarity index 98% rename from tests/providers/redis/log/test_redis_task_handler.py rename to providers/tests/redis/log/test_redis_task_handler.py index f4ded2fa586ae..bc7345df9a53b 100644 --- a/tests/providers/redis/log/test_redis_task_handler.py +++ b/providers/tests/redis/log/test_redis_task_handler.py @@ -28,7 +28,8 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/redis/hooks/__init__.py b/providers/tests/redis/operators/__init__.py similarity index 100% rename from tests/providers/redis/hooks/__init__.py rename to providers/tests/redis/operators/__init__.py diff --git a/tests/providers/redis/operators/test_redis_publish.py b/providers/tests/redis/operators/test_redis_publish.py similarity index 100% rename from tests/providers/redis/operators/test_redis_publish.py rename to providers/tests/redis/operators/test_redis_publish.py diff --git a/tests/providers/redis/log/__init__.py b/providers/tests/redis/sensors/__init__.py similarity index 100% rename from tests/providers/redis/log/__init__.py rename to providers/tests/redis/sensors/__init__.py diff --git a/tests/providers/redis/sensors/test_redis_key.py b/providers/tests/redis/sensors/test_redis_key.py similarity index 100% rename from tests/providers/redis/sensors/test_redis_key.py rename to providers/tests/redis/sensors/test_redis_key.py diff --git a/tests/providers/redis/sensors/test_redis_pub_sub.py b/providers/tests/redis/sensors/test_redis_pub_sub.py similarity index 100% rename from tests/providers/redis/sensors/test_redis_pub_sub.py rename to providers/tests/redis/sensors/test_redis_pub_sub.py diff --git a/tests/providers/redis/operators/__init__.py b/providers/tests/salesforce/__init__.py similarity index 100% rename from tests/providers/redis/operators/__init__.py rename to providers/tests/salesforce/__init__.py diff --git a/tests/providers/redis/sensors/__init__.py b/providers/tests/salesforce/hooks/__init__.py similarity index 100% rename from tests/providers/redis/sensors/__init__.py rename to providers/tests/salesforce/hooks/__init__.py diff --git a/tests/providers/salesforce/hooks/test_salesforce.py b/providers/tests/salesforce/hooks/test_salesforce.py similarity index 100% rename from tests/providers/salesforce/hooks/test_salesforce.py rename to providers/tests/salesforce/hooks/test_salesforce.py diff --git a/tests/providers/sendgrid/__init__.py b/providers/tests/salesforce/operators/__init__.py similarity index 100% rename from tests/providers/sendgrid/__init__.py rename to providers/tests/salesforce/operators/__init__.py diff --git a/tests/providers/salesforce/operators/test_bulk.py b/providers/tests/salesforce/operators/test_bulk.py similarity index 100% rename from tests/providers/salesforce/operators/test_bulk.py rename to providers/tests/salesforce/operators/test_bulk.py diff --git a/tests/providers/salesforce/operators/test_salesforce_apex_rest.py b/providers/tests/salesforce/operators/test_salesforce_apex_rest.py similarity index 100% rename from tests/providers/salesforce/operators/test_salesforce_apex_rest.py rename to providers/tests/salesforce/operators/test_salesforce_apex_rest.py diff --git a/tests/providers/salesforce/__init__.py b/providers/tests/samba/__init__.py similarity index 100% rename from tests/providers/salesforce/__init__.py rename to providers/tests/samba/__init__.py diff --git a/tests/providers/salesforce/hooks/__init__.py b/providers/tests/samba/hooks/__init__.py similarity index 100% rename from tests/providers/salesforce/hooks/__init__.py rename to providers/tests/samba/hooks/__init__.py diff --git a/tests/providers/samba/hooks/test_samba.py b/providers/tests/samba/hooks/test_samba.py similarity index 100% rename from tests/providers/samba/hooks/test_samba.py rename to providers/tests/samba/hooks/test_samba.py diff --git a/tests/providers/samba/__init__.py b/providers/tests/samba/transfers/__init__.py similarity index 100% rename from tests/providers/samba/__init__.py rename to providers/tests/samba/transfers/__init__.py diff --git a/tests/providers/samba/transfers/test_gcs_to_samba.py b/providers/tests/samba/transfers/test_gcs_to_samba.py similarity index 100% rename from tests/providers/samba/transfers/test_gcs_to_samba.py rename to providers/tests/samba/transfers/test_gcs_to_samba.py diff --git a/tests/providers/samba/hooks/__init__.py b/providers/tests/segment/__init__.py similarity index 100% rename from tests/providers/samba/hooks/__init__.py rename to providers/tests/segment/__init__.py diff --git a/tests/providers/samba/transfers/__init__.py b/providers/tests/segment/hooks/__init__.py similarity index 100% rename from tests/providers/samba/transfers/__init__.py rename to providers/tests/segment/hooks/__init__.py diff --git a/tests/providers/segment/hooks/test_segment.py b/providers/tests/segment/hooks/test_segment.py similarity index 100% rename from tests/providers/segment/hooks/test_segment.py rename to providers/tests/segment/hooks/test_segment.py diff --git a/tests/providers/segment/__init__.py b/providers/tests/segment/operators/__init__.py similarity index 100% rename from tests/providers/segment/__init__.py rename to providers/tests/segment/operators/__init__.py diff --git a/tests/providers/segment/operators/test_segment_track_event.py b/providers/tests/segment/operators/test_segment_track_event.py similarity index 100% rename from tests/providers/segment/operators/test_segment_track_event.py rename to providers/tests/segment/operators/test_segment_track_event.py diff --git a/tests/providers/sendgrid/utils/__init__.py b/providers/tests/sendgrid/__init__.py similarity index 100% rename from tests/providers/sendgrid/utils/__init__.py rename to providers/tests/sendgrid/__init__.py diff --git a/tests/providers/sftp/__init__.py b/providers/tests/sendgrid/utils/__init__.py similarity index 100% rename from tests/providers/sftp/__init__.py rename to providers/tests/sendgrid/utils/__init__.py diff --git a/tests/providers/sendgrid/utils/test_emailer.py b/providers/tests/sendgrid/utils/test_emailer.py similarity index 100% rename from tests/providers/sendgrid/utils/test_emailer.py rename to providers/tests/sendgrid/utils/test_emailer.py diff --git a/tests/providers/sftp/decorators/__init__.py b/providers/tests/sftp/__init__.py similarity index 100% rename from tests/providers/sftp/decorators/__init__.py rename to providers/tests/sftp/__init__.py diff --git a/tests/providers/sftp/decorators/sensors/__init__.py b/providers/tests/sftp/decorators/__init__.py similarity index 100% rename from tests/providers/sftp/decorators/sensors/__init__.py rename to providers/tests/sftp/decorators/__init__.py diff --git a/tests/providers/sftp/hooks/__init__.py b/providers/tests/sftp/decorators/sensors/__init__.py similarity index 100% rename from tests/providers/sftp/hooks/__init__.py rename to providers/tests/sftp/decorators/sensors/__init__.py diff --git a/tests/providers/sftp/decorators/sensors/test_sftp.py b/providers/tests/sftp/decorators/sensors/test_sftp.py similarity index 100% rename from tests/providers/sftp/decorators/sensors/test_sftp.py rename to providers/tests/sftp/decorators/sensors/test_sftp.py diff --git a/tests/providers/sftp/operators/__init__.py b/providers/tests/sftp/hooks/__init__.py similarity index 100% rename from tests/providers/sftp/operators/__init__.py rename to providers/tests/sftp/hooks/__init__.py diff --git a/tests/providers/sftp/hooks/test_sftp.py b/providers/tests/sftp/hooks/test_sftp.py similarity index 99% rename from tests/providers/sftp/hooks/test_sftp.py rename to providers/tests/sftp/hooks/test_sftp.py index 3305f9cb761a1..7a7a2991a7031 100644 --- a/tests/providers/sftp/hooks/test_sftp.py +++ b/providers/tests/sftp/hooks/test_sftp.py @@ -406,8 +406,9 @@ def test_deprecation_ftp_conn_id(self, mock_get_connection): def test_invalid_ssh_hook(self, mock_get_connection): connection = Connection(conn_id="sftp_default", login="root", host="localhost") mock_get_connection.return_value = connection - with pytest.raises(AirflowException, match="ssh_hook must be an instance of SSHHook"), pytest.warns( - AirflowProviderDeprecationWarning, match=r"Parameter `ssh_hook` is deprecated.*" + with ( + pytest.raises(AirflowException, match="ssh_hook must be an instance of SSHHook"), + pytest.warns(AirflowProviderDeprecationWarning, match=r"Parameter `ssh_hook` is deprecated.*"), ): SFTPHook(ssh_hook="invalid_hook") diff --git a/tests/providers/sftp/sensors/__init__.py b/providers/tests/sftp/operators/__init__.py similarity index 100% rename from tests/providers/sftp/sensors/__init__.py rename to providers/tests/sftp/operators/__init__.py diff --git a/tests/providers/sftp/operators/test_sftp.py b/providers/tests/sftp/operators/test_sftp.py similarity index 97% rename from tests/providers/sftp/operators/test_sftp.py rename to providers/tests/sftp/operators/test_sftp.py index a6835675da26f..e6c90c2c74f5d 100644 --- a/tests/providers/sftp/operators/test_sftp.py +++ b/providers/tests/sftp/operators/test_sftp.py @@ -35,7 +35,8 @@ from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils import timezone from airflow.utils.timezone import datetime -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test @@ -151,8 +152,9 @@ def test_file_transfer_no_intermediate_dir_error_put(self, create_task_instance_ operation=SFTPOperation.PUT, create_intermediate_dirs=False, ) - with pytest.raises(AirflowException) as ctx, pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + with ( + pytest.raises(AirflowException) as ctx, + pytest.warns(AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*"), ): ti2.run() assert "No such file" in str(ctx.value) @@ -292,8 +294,11 @@ def test_file_transfer_no_intermediate_dir_error_get(self, dag_maker, create_rem for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: # This should raise an error with "No such file" as the directory # does not exist. - with pytest.raises(AirflowException) as ctx, pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + with ( + pytest.raises(AirflowException) as ctx, + pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ), ): ti.run() assert "No such file" in str(ctx.value) @@ -376,8 +381,9 @@ def test_arg_checking(self): operation=SFTPOperation.PUT, dag=dag, ) - with contextlib.suppress(Exception), pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + with ( + contextlib.suppress(Exception), + pytest.warns(AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*"), ): task_3.execute(None) assert task_3.sftp_hook.ssh_conn_id == self.hook.ssh_conn_id diff --git a/tests/providers/sftp/triggers/__init__.py b/providers/tests/sftp/sensors/__init__.py similarity index 100% rename from tests/providers/sftp/triggers/__init__.py rename to providers/tests/sftp/sensors/__init__.py diff --git a/tests/providers/sftp/sensors/test_sftp.py b/providers/tests/sftp/sensors/test_sftp.py similarity index 100% rename from tests/providers/sftp/sensors/test_sftp.py rename to providers/tests/sftp/sensors/test_sftp.py diff --git a/tests/providers/slack/notifications/__init__.py b/providers/tests/sftp/triggers/__init__.py similarity index 100% rename from tests/providers/slack/notifications/__init__.py rename to providers/tests/sftp/triggers/__init__.py diff --git a/tests/providers/sftp/triggers/test_sftp.py b/providers/tests/sftp/triggers/test_sftp.py similarity index 100% rename from tests/providers/sftp/triggers/test_sftp.py rename to providers/tests/sftp/triggers/test_sftp.py diff --git a/tests/providers/segment/hooks/__init__.py b/providers/tests/singularity/__init__.py similarity index 100% rename from tests/providers/segment/hooks/__init__.py rename to providers/tests/singularity/__init__.py diff --git a/tests/providers/segment/operators/__init__.py b/providers/tests/singularity/operators/__init__.py similarity index 100% rename from tests/providers/segment/operators/__init__.py rename to providers/tests/singularity/operators/__init__.py diff --git a/tests/providers/singularity/operators/test_singularity.py b/providers/tests/singularity/operators/test_singularity.py similarity index 100% rename from tests/providers/singularity/operators/test_singularity.py rename to providers/tests/singularity/operators/test_singularity.py diff --git a/tests/providers/singularity/__init__.py b/providers/tests/slack/__init__.py similarity index 100% rename from tests/providers/singularity/__init__.py rename to providers/tests/slack/__init__.py diff --git a/tests/providers/singularity/operators/__init__.py b/providers/tests/slack/hooks/__init__.py similarity index 100% rename from tests/providers/singularity/operators/__init__.py rename to providers/tests/slack/hooks/__init__.py diff --git a/tests/providers/slack/hooks/test_slack.py b/providers/tests/slack/hooks/test_slack.py similarity index 100% rename from tests/providers/slack/hooks/test_slack.py rename to providers/tests/slack/hooks/test_slack.py diff --git a/tests/providers/slack/hooks/test_slack_webhook.py b/providers/tests/slack/hooks/test_slack_webhook.py similarity index 100% rename from tests/providers/slack/hooks/test_slack_webhook.py rename to providers/tests/slack/hooks/test_slack_webhook.py diff --git a/tests/providers/slack/utils/__init__.py b/providers/tests/slack/notifications/__init__.py similarity index 100% rename from tests/providers/slack/utils/__init__.py rename to providers/tests/slack/notifications/__init__.py diff --git a/tests/providers/slack/notifications/test_slack.py b/providers/tests/slack/notifications/test_slack.py similarity index 100% rename from tests/providers/slack/notifications/test_slack.py rename to providers/tests/slack/notifications/test_slack.py diff --git a/tests/providers/slack/notifications/test_slack_webhook.py b/providers/tests/slack/notifications/test_slack_webhook.py similarity index 100% rename from tests/providers/slack/notifications/test_slack_webhook.py rename to providers/tests/slack/notifications/test_slack_webhook.py diff --git a/tests/providers/slack/__init__.py b/providers/tests/slack/operators/__init__.py similarity index 100% rename from tests/providers/slack/__init__.py rename to providers/tests/slack/operators/__init__.py diff --git a/tests/providers/slack/operators/test_slack.py b/providers/tests/slack/operators/test_slack.py similarity index 98% rename from tests/providers/slack/operators/test_slack.py rename to providers/tests/slack/operators/test_slack.py index 603cef7835ded..f276ee43159e6 100644 --- a/tests/providers/slack/operators/test_slack.py +++ b/providers/tests/slack/operators/test_slack.py @@ -362,7 +362,8 @@ def test_partial_both_channel_parameters(self, channel, channels, dag_maker, ses with set_current_task_instance_session(session=session): warning_match = r"Argument `channel` is deprecated.*use `channels` instead" for ti in tis: - with pytest.warns(AirflowProviderDeprecationWarning, match=warning_match), pytest.raises( - ValueError, match="Cannot set both arguments" + with ( + pytest.warns(AirflowProviderDeprecationWarning, match=warning_match), + pytest.raises(ValueError, match="Cannot set both arguments"), ): ti.render_templates() diff --git a/tests/providers/slack/operators/test_slack_webhook.py b/providers/tests/slack/operators/test_slack_webhook.py similarity index 100% rename from tests/providers/slack/operators/test_slack_webhook.py rename to providers/tests/slack/operators/test_slack_webhook.py diff --git a/tests/providers/slack/hooks/__init__.py b/providers/tests/slack/transfers/__init__.py similarity index 100% rename from tests/providers/slack/hooks/__init__.py rename to providers/tests/slack/transfers/__init__.py diff --git a/tests/providers/slack/transfers/conftest.py b/providers/tests/slack/transfers/conftest.py similarity index 100% rename from tests/providers/slack/transfers/conftest.py rename to providers/tests/slack/transfers/conftest.py diff --git a/tests/providers/slack/transfers/test_base_sql_to_slack.py b/providers/tests/slack/transfers/test_base_sql_to_slack.py similarity index 100% rename from tests/providers/slack/transfers/test_base_sql_to_slack.py rename to providers/tests/slack/transfers/test_base_sql_to_slack.py diff --git a/tests/providers/slack/transfers/test_sql_to_slack.py b/providers/tests/slack/transfers/test_sql_to_slack.py similarity index 100% rename from tests/providers/slack/transfers/test_sql_to_slack.py rename to providers/tests/slack/transfers/test_sql_to_slack.py diff --git a/tests/providers/slack/transfers/test_sql_to_slack_webhook.py b/providers/tests/slack/transfers/test_sql_to_slack_webhook.py similarity index 96% rename from tests/providers/slack/transfers/test_sql_to_slack_webhook.py rename to providers/tests/slack/transfers/test_sql_to_slack_webhook.py index 2f6ef63bc687f..c56f895b7bc78 100644 --- a/tests/providers/slack/transfers/test_sql_to_slack_webhook.py +++ b/providers/tests/slack/transfers/test_sql_to_slack_webhook.py @@ -164,8 +164,9 @@ def test_conflicting_conn_id(self): "slack_message": "message: {{ ds }}, {{ xxxx }}", "sql": "sql {{ ds }}", } - with pytest.raises(ValueError, match="Conflicting Connection ids provided"), pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `slack_conn_id` is deprecated" + with ( + pytest.raises(ValueError, match="Conflicting Connection ids provided"), + pytest.warns(AirflowProviderDeprecationWarning, match="Parameter `slack_conn_id` is deprecated"), ): self._construct_operator(**operator_args, slack_webhook_conn_id="foo", slack_conn_id="bar") @@ -312,7 +313,8 @@ def test_partial_ambiguous_slack_connections(self, dag_maker, session): with set_current_task_instance_session(session=session): warning_match = r"Parameter `slack_conn_id` is deprecated" for ti in tis: - with pytest.warns(AirflowProviderDeprecationWarning, match=warning_match), pytest.raises( - ValueError, match="Conflicting Connection ids provided" + with ( + pytest.warns(AirflowProviderDeprecationWarning, match=warning_match), + pytest.raises(ValueError, match="Conflicting Connection ids provided"), ): ti.render_templates() diff --git a/tests/providers/smtp/notifications/__init__.py b/providers/tests/slack/utils/__init__.py similarity index 100% rename from tests/providers/smtp/notifications/__init__.py rename to providers/tests/slack/utils/__init__.py diff --git a/tests/providers/slack/utils/test_utils.py b/providers/tests/slack/utils/test_utils.py similarity index 100% rename from tests/providers/slack/utils/test_utils.py rename to providers/tests/slack/utils/test_utils.py diff --git a/tests/providers/slack/operators/__init__.py b/providers/tests/smtp/__init__.py similarity index 100% rename from tests/providers/slack/operators/__init__.py rename to providers/tests/smtp/__init__.py diff --git a/tests/providers/slack/transfers/__init__.py b/providers/tests/smtp/hooks/__init__.py similarity index 100% rename from tests/providers/slack/transfers/__init__.py rename to providers/tests/smtp/hooks/__init__.py diff --git a/tests/providers/smtp/hooks/test_smtp.py b/providers/tests/smtp/hooks/test_smtp.py similarity index 99% rename from tests/providers/smtp/hooks/test_smtp.py rename to providers/tests/smtp/hooks/test_smtp.py index 505fc303051ef..04a20e0ca6f0a 100644 --- a/tests/providers/smtp/hooks/test_smtp.py +++ b/providers/tests/smtp/hooks/test_smtp.py @@ -30,7 +30,8 @@ from airflow.providers.smtp.hooks.smtp import SmtpHook from airflow.utils import db from airflow.utils.session import create_session -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/snowflake/__init__.py b/providers/tests/smtp/notifications/__init__.py similarity index 100% rename from tests/providers/snowflake/__init__.py rename to providers/tests/smtp/notifications/__init__.py diff --git a/tests/providers/smtp/notifications/test_smtp.py b/providers/tests/smtp/notifications/test_smtp.py similarity index 96% rename from tests/providers/smtp/notifications/test_smtp.py rename to providers/tests/smtp/notifications/test_smtp.py index 39b51e8e02ce7..75e1ebf530fae 100644 --- a/tests/providers/smtp/notifications/test_smtp.py +++ b/providers/tests/smtp/notifications/test_smtp.py @@ -31,8 +31,9 @@ send_smtp_notification, ) from airflow.utils import timezone -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test @@ -182,9 +183,10 @@ def test_notifier_with_nondefault_conf_vars(self, mock_smtphook_hook, create_tas ti = create_task_instance(dag_id="dag", task_id="op", execution_date=timezone.datetime(2018, 1, 1)) context = {"dag": ti.dag_run.dag, "ti": ti} - with tempfile.NamedTemporaryFile(mode="wt", suffix=".txt") as f_subject, tempfile.NamedTemporaryFile( - mode="wt", suffix=".txt" - ) as f_content: + with ( + tempfile.NamedTemporaryFile(mode="wt", suffix=".txt") as f_subject, + tempfile.NamedTemporaryFile(mode="wt", suffix=".txt") as f_content, + ): f_subject.write("Task {{ ti.task_id }} failed") f_subject.flush() diff --git a/tests/providers/smtp/__init__.py b/providers/tests/smtp/operators/__init__.py similarity index 100% rename from tests/providers/smtp/__init__.py rename to providers/tests/smtp/operators/__init__.py diff --git a/tests/providers/smtp/operators/test_smtp.py b/providers/tests/smtp/operators/test_smtp.py similarity index 100% rename from tests/providers/smtp/operators/test_smtp.py rename to providers/tests/smtp/operators/test_smtp.py diff --git a/tests/providers/snowflake/decorators/__init__.py b/providers/tests/snowflake/__init__.py similarity index 100% rename from tests/providers/snowflake/decorators/__init__.py rename to providers/tests/snowflake/__init__.py diff --git a/tests/providers/snowflake/hooks/__init__.py b/providers/tests/snowflake/decorators/__init__.py similarity index 100% rename from tests/providers/snowflake/hooks/__init__.py rename to providers/tests/snowflake/decorators/__init__.py diff --git a/tests/providers/snowflake/decorators/test_snowpark.py b/providers/tests/snowflake/decorators/test_snowpark.py similarity index 100% rename from tests/providers/snowflake/decorators/test_snowpark.py rename to providers/tests/snowflake/decorators/test_snowpark.py diff --git a/tests/providers/snowflake/operators/__init__.py b/providers/tests/snowflake/hooks/__init__.py similarity index 100% rename from tests/providers/snowflake/operators/__init__.py rename to providers/tests/snowflake/hooks/__init__.py diff --git a/tests/providers/snowflake/hooks/test_snowflake.py b/providers/tests/snowflake/hooks/test_snowflake.py similarity index 93% rename from tests/providers/snowflake/hooks/test_snowflake.py rename to providers/tests/snowflake/hooks/test_snowflake.py index 9ef0c4d2a5717..b7c9382654be0 100644 --- a/tests/providers/snowflake/hooks/test_snowflake.py +++ b/providers/tests/snowflake/hooks/test_snowflake.py @@ -394,9 +394,10 @@ def test_get_conn_params_should_support_private_auth_with_unencrypted_key( with mock.patch.dict("os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()): assert "private_key" in SnowflakeHook(snowflake_conn_id="test_conn")._get_conn_params connection_kwargs["password"] = _PASSWORD - with mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() - ), pytest.raises(TypeError, match="Password was given but private key is not encrypted."): + with ( + mock.patch.dict("os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()), + pytest.raises(TypeError, match="Password was given but private key is not encrypted."), + ): SnowflakeHook(snowflake_conn_id="test_conn")._get_conn_params def test_get_conn_params_should_fail_on_invalid_key(self): @@ -412,9 +413,10 @@ def test_get_conn_params_should_fail_on_invalid_key(self): "private_key_file": "/dev/urandom", }, } - with mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() - ), pytest.raises(ValueError, match="The private_key_file path points to an empty or invalid file."): + with ( + mock.patch.dict("os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()), + pytest.raises(ValueError, match="The private_key_file path points to an empty or invalid file."), + ): SnowflakeHook(snowflake_conn_id="test_conn").get_conn() def test_should_add_partner_info(self): @@ -428,18 +430,24 @@ def test_should_add_partner_info(self): ) def test_get_conn_should_call_connect(self): - with mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**BASE_CONNECTION_KWARGS).get_uri() - ), mock.patch("airflow.providers.snowflake.hooks.snowflake.connector") as mock_connector: + with ( + mock.patch.dict( + "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**BASE_CONNECTION_KWARGS).get_uri() + ), + mock.patch("airflow.providers.snowflake.hooks.snowflake.connector") as mock_connector, + ): hook = SnowflakeHook(snowflake_conn_id="test_conn") conn = hook.get_conn() mock_connector.connect.assert_called_once_with(**hook._get_conn_params) assert mock_connector.connect.return_value == conn def test_get_sqlalchemy_engine_should_support_pass_auth(self): - with mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**BASE_CONNECTION_KWARGS).get_uri() - ), mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine: + with ( + mock.patch.dict( + "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**BASE_CONNECTION_KWARGS).get_uri() + ), + mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine, + ): hook = SnowflakeHook(snowflake_conn_id="test_conn") conn = hook.get_sqlalchemy_engine() mock_create_engine.assert_called_once_with( @@ -452,9 +460,10 @@ def test_get_sqlalchemy_engine_should_support_insecure_mode(self): connection_kwargs = deepcopy(BASE_CONNECTION_KWARGS) connection_kwargs["extra"]["extra__snowflake__insecure_mode"] = "True" - with mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() - ), mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine: + with ( + mock.patch.dict("os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()), + mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine, + ): hook = SnowflakeHook(snowflake_conn_id="test_conn") conn = hook.get_sqlalchemy_engine() mock_create_engine.assert_called_once_with( @@ -468,9 +477,10 @@ def test_get_sqlalchemy_engine_should_support_session_parameters(self): connection_kwargs = deepcopy(BASE_CONNECTION_KWARGS) connection_kwargs["extra"]["session_parameters"] = {"TEST_PARAM": "AA", "TEST_PARAM_B": 123} - with mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() - ), mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine: + with ( + mock.patch.dict("os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()), + mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine, + ): hook = SnowflakeHook(snowflake_conn_id="test_conn") conn = hook.get_sqlalchemy_engine() mock_create_engine.assert_called_once_with( @@ -485,9 +495,10 @@ def test_get_sqlalchemy_engine_should_support_private_key_auth(self, non_encrypt connection_kwargs["password"] = "" connection_kwargs["extra"]["private_key_file"] = str(non_encrypted_temporary_private_key) - with mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() - ), mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine: + with ( + mock.patch.dict("os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()), + mock.patch("airflow.providers.snowflake.hooks.snowflake.create_engine") as mock_create_engine, + ): hook = SnowflakeHook(snowflake_conn_id="test_conn") conn = hook.get_sqlalchemy_engine() assert "private_key" in mock_create_engine.call_args.kwargs["connect_args"] diff --git a/tests/providers/snowflake/hooks/test_snowflake_sql_api.py b/providers/tests/snowflake/hooks/test_snowflake_sql_api.py similarity index 97% rename from tests/providers/snowflake/hooks/test_snowflake_sql_api.py rename to providers/tests/snowflake/hooks/test_snowflake_sql_api.py index df3f06db2f302..1247e3e031820 100644 --- a/tests/providers/snowflake/hooks/test_snowflake_sql_api.py +++ b/providers/tests/snowflake/hooks/test_snowflake_sql_api.py @@ -442,12 +442,15 @@ def test_get_private_key_raise_exception(self, encrypted_temporary_private_key: }, } hook = SnowflakeSqlApiHook(snowflake_conn_id="test_conn") - with unittest.mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() - ), pytest.raises( - AirflowException, - match="The private_key_file and private_key_content extra fields are mutually " - "exclusive. Please remove one.", + with ( + unittest.mock.patch.dict( + "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() + ), + pytest.raises( + AirflowException, + match="The private_key_file and private_key_content extra fields are mutually " + "exclusive. Please remove one.", + ), ): hook.get_private_key() @@ -504,9 +507,12 @@ def test_get_private_key_should_support_private_auth_with_unencrypted_key( hook.get_private_key() assert hook.private_key is not None connection_kwargs["password"] = _PASSWORD - with unittest.mock.patch.dict( - "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() - ), pytest.raises(TypeError, match="Password was given but private key is not encrypted."): + with ( + unittest.mock.patch.dict( + "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() + ), + pytest.raises(TypeError, match="Password was given but private key is not encrypted."), + ): SnowflakeSqlApiHook(snowflake_conn_id="test_conn").get_private_key() @pytest.mark.parametrize( diff --git a/tests/providers/snowflake/hooks/test_sql.py b/providers/tests/snowflake/hooks/test_sql.py similarity index 100% rename from tests/providers/snowflake/hooks/test_sql.py rename to providers/tests/snowflake/hooks/test_sql.py diff --git a/tests/providers/snowflake/transfers/__init__.py b/providers/tests/snowflake/operators/__init__.py similarity index 100% rename from tests/providers/snowflake/transfers/__init__.py rename to providers/tests/snowflake/operators/__init__.py diff --git a/tests/providers/snowflake/operators/test_snowflake.py b/providers/tests/snowflake/operators/test_snowflake.py similarity index 100% rename from tests/providers/snowflake/operators/test_snowflake.py rename to providers/tests/snowflake/operators/test_snowflake.py diff --git a/tests/providers/snowflake/operators/test_snowflake_sql.py b/providers/tests/snowflake/operators/test_snowflake_sql.py similarity index 100% rename from tests/providers/snowflake/operators/test_snowflake_sql.py rename to providers/tests/snowflake/operators/test_snowflake_sql.py diff --git a/tests/providers/snowflake/operators/test_snowpark.py b/providers/tests/snowflake/operators/test_snowpark.py similarity index 100% rename from tests/providers/snowflake/operators/test_snowpark.py rename to providers/tests/snowflake/operators/test_snowpark.py diff --git a/tests/providers/snowflake/triggers/__init__.py b/providers/tests/snowflake/transfers/__init__.py similarity index 100% rename from tests/providers/snowflake/triggers/__init__.py rename to providers/tests/snowflake/transfers/__init__.py diff --git a/tests/providers/snowflake/transfers/test_copy_into_snowflake.py b/providers/tests/snowflake/transfers/test_copy_into_snowflake.py similarity index 100% rename from tests/providers/snowflake/transfers/test_copy_into_snowflake.py rename to providers/tests/snowflake/transfers/test_copy_into_snowflake.py diff --git a/tests/providers/snowflake/utils/__init__.py b/providers/tests/snowflake/triggers/__init__.py similarity index 100% rename from tests/providers/snowflake/utils/__init__.py rename to providers/tests/snowflake/triggers/__init__.py diff --git a/tests/providers/snowflake/triggers/test_snowflake.py b/providers/tests/snowflake/triggers/test_snowflake.py similarity index 100% rename from tests/providers/snowflake/triggers/test_snowflake.py rename to providers/tests/snowflake/triggers/test_snowflake.py diff --git a/tests/providers/standard/__init__.py b/providers/tests/snowflake/utils/__init__.py similarity index 100% rename from tests/providers/standard/__init__.py rename to providers/tests/snowflake/utils/__init__.py diff --git a/tests/providers/snowflake/utils/test_common.py b/providers/tests/snowflake/utils/test_common.py similarity index 100% rename from tests/providers/snowflake/utils/test_common.py rename to providers/tests/snowflake/utils/test_common.py diff --git a/tests/providers/snowflake/utils/test_openlineage.py b/providers/tests/snowflake/utils/test_openlineage.py similarity index 100% rename from tests/providers/snowflake/utils/test_openlineage.py rename to providers/tests/snowflake/utils/test_openlineage.py diff --git a/tests/providers/snowflake/utils/test_snowpark.py b/providers/tests/snowflake/utils/test_snowpark.py similarity index 100% rename from tests/providers/snowflake/utils/test_snowpark.py rename to providers/tests/snowflake/utils/test_snowpark.py diff --git a/tests/providers/snowflake/utils/test_sql_api_generate_jwt.py b/providers/tests/snowflake/utils/test_sql_api_generate_jwt.py similarity index 100% rename from tests/providers/snowflake/utils/test_sql_api_generate_jwt.py rename to providers/tests/snowflake/utils/test_sql_api_generate_jwt.py diff --git a/tests/providers/smtp/hooks/__init__.py b/providers/tests/sqlite/__init__.py similarity index 100% rename from tests/providers/smtp/hooks/__init__.py rename to providers/tests/sqlite/__init__.py diff --git a/tests/providers/smtp/operators/__init__.py b/providers/tests/sqlite/hooks/__init__.py similarity index 100% rename from tests/providers/smtp/operators/__init__.py rename to providers/tests/sqlite/hooks/__init__.py diff --git a/tests/providers/sqlite/hooks/test_sqlite.py b/providers/tests/sqlite/hooks/test_sqlite.py similarity index 100% rename from tests/providers/sqlite/hooks/test_sqlite.py rename to providers/tests/sqlite/hooks/test_sqlite.py diff --git a/tests/providers/sqlite/__init__.py b/providers/tests/sqlite/operators/__init__.py similarity index 100% rename from tests/providers/sqlite/__init__.py rename to providers/tests/sqlite/operators/__init__.py diff --git a/tests/providers/sqlite/operators/test_sqlite.py b/providers/tests/sqlite/operators/test_sqlite.py similarity index 100% rename from tests/providers/sqlite/operators/test_sqlite.py rename to providers/tests/sqlite/operators/test_sqlite.py diff --git a/tests/providers/sqlite/hooks/__init__.py b/providers/tests/ssh/__init__.py similarity index 100% rename from tests/providers/sqlite/hooks/__init__.py rename to providers/tests/ssh/__init__.py diff --git a/tests/providers/sqlite/operators/__init__.py b/providers/tests/ssh/hooks/__init__.py similarity index 100% rename from tests/providers/sqlite/operators/__init__.py rename to providers/tests/ssh/hooks/__init__.py diff --git a/tests/providers/ssh/hooks/test_ssh.py b/providers/tests/ssh/hooks/test_ssh.py similarity index 99% rename from tests/providers/ssh/hooks/test_ssh.py rename to providers/tests/ssh/hooks/test_ssh.py index 71661e5b4ec6d..6b3e5dcdae116 100644 --- a/tests/providers/ssh/hooks/test_ssh.py +++ b/providers/tests/ssh/hooks/test_ssh.py @@ -534,8 +534,9 @@ def test_tunnel(self): args=["python", "-c", HELLO_SERVER_CMD], stdout=subprocess.PIPE, ) - with subprocess.Popen(**subprocess_kwargs) as server_handle, hook.get_tunnel( - local_port=2135, remote_port=2134 + with ( + subprocess.Popen(**subprocess_kwargs) as server_handle, + hook.get_tunnel(local_port=2135, remote_port=2134), ): server_output = server_handle.stdout.read(5) assert b"ready" == server_output diff --git a/tests/providers/ssh/__init__.py b/providers/tests/ssh/operators/__init__.py similarity index 100% rename from tests/providers/ssh/__init__.py rename to providers/tests/ssh/operators/__init__.py diff --git a/tests/providers/ssh/operators/test_ssh.py b/providers/tests/ssh/operators/test_ssh.py similarity index 99% rename from tests/providers/ssh/operators/test_ssh.py rename to providers/tests/ssh/operators/test_ssh.py index 9b4ecbbb20dbf..b304a24882bb9 100644 --- a/tests/providers/ssh/operators/test_ssh.py +++ b/providers/tests/ssh/operators/test_ssh.py @@ -31,7 +31,8 @@ from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.timezone import datetime from airflow.utils.types import NOTSET -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/standard/operators/__init__.py b/providers/tests/standard/__init__.py similarity index 100% rename from tests/providers/standard/operators/__init__.py rename to providers/tests/standard/__init__.py diff --git a/tests/providers/standard/sensors/__init__.py b/providers/tests/standard/operators/__init__.py similarity index 100% rename from tests/providers/standard/sensors/__init__.py rename to providers/tests/standard/operators/__init__.py diff --git a/tests/providers/standard/operators/test_bash.py b/providers/tests/standard/operators/test_bash.py similarity index 99% rename from tests/providers/standard/operators/test_bash.py rename to providers/tests/standard/operators/test_bash.py index 2c29a0b96dc93..305651ee596ae 100644 --- a/tests/providers/standard/operators/test_bash.py +++ b/providers/tests/standard/operators/test_bash.py @@ -32,7 +32,8 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/standard/operators/test_datetime.py b/providers/tests/standard/operators/test_datetime.py similarity index 99% rename from tests/providers/standard/operators/test_datetime.py rename to providers/tests/standard/operators/test_datetime.py index 530450a963bd3..4fbb0863d35ae 100644 --- a/tests/providers/standard/operators/test_datetime.py +++ b/providers/tests/standard/operators/test_datetime.py @@ -30,7 +30,8 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/standard/operators/test_weekday.py b/providers/tests/standard/operators/test_weekday.py similarity index 99% rename from tests/providers/standard/operators/test_weekday.py rename to providers/tests/standard/operators/test_weekday.py index 7ad7415c366f7..6c20299d15e79 100644 --- a/tests/providers/standard/operators/test_weekday.py +++ b/providers/tests/standard/operators/test_weekday.py @@ -32,7 +32,8 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.weekday import WeekDay -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/providers/tableau/__init__.py b/providers/tests/standard/sensors/__init__.py similarity index 100% rename from tests/providers/tableau/__init__.py rename to providers/tests/standard/sensors/__init__.py diff --git a/tests/providers/standard/sensors/test_bash.py b/providers/tests/standard/sensors/test_bash.py similarity index 100% rename from tests/providers/standard/sensors/test_bash.py rename to providers/tests/standard/sensors/test_bash.py diff --git a/tests/providers/standard/sensors/test_date_time.py b/providers/tests/standard/sensors/test_date_time.py similarity index 100% rename from tests/providers/standard/sensors/test_date_time.py rename to providers/tests/standard/sensors/test_date_time.py diff --git a/tests/providers/standard/sensors/test_time.py b/providers/tests/standard/sensors/test_time.py similarity index 100% rename from tests/providers/standard/sensors/test_time.py rename to providers/tests/standard/sensors/test_time.py diff --git a/tests/providers/standard/sensors/test_time_delta.py b/providers/tests/standard/sensors/test_time_delta.py similarity index 100% rename from tests/providers/standard/sensors/test_time_delta.py rename to providers/tests/standard/sensors/test_time_delta.py diff --git a/tests/providers/standard/sensors/test_weekday.py b/providers/tests/standard/sensors/test_weekday.py similarity index 99% rename from tests/providers/standard/sensors/test_weekday.py rename to providers/tests/standard/sensors/test_weekday.py index d4a0f04c2290e..ef0091a9fdd3a 100644 --- a/tests/providers/standard/sensors/test_weekday.py +++ b/providers/tests/standard/sensors/test_weekday.py @@ -27,7 +27,8 @@ from airflow.providers.standard.sensors.weekday import DayOfWeekSensor from airflow.utils.timezone import datetime from airflow.utils.weekday import WeekDay -from tests.test_utils import db + +from dev.tests_common.test_utils import db pytestmark = pytest.mark.db_test diff --git a/tests/providers/tableau/operators/__init__.py b/providers/tests/system/__init__.py similarity index 100% rename from tests/providers/tableau/operators/__init__.py rename to providers/tests/system/__init__.py diff --git a/tests/providers/tableau/sensors/__init__.py b/providers/tests/system/airbyte/__init__.py similarity index 100% rename from tests/providers/tableau/sensors/__init__.py rename to providers/tests/system/airbyte/__init__.py diff --git a/tests/system/providers/airbyte/example_airbyte_trigger_job.py b/providers/tests/system/airbyte/example_airbyte_trigger_job.py similarity index 96% rename from tests/system/providers/airbyte/example_airbyte_trigger_job.py rename to providers/tests/system/airbyte/example_airbyte_trigger_job.py index 4ff2b5e0d48cd..1dd3f01d72618 100644 --- a/tests/system/providers/airbyte/example_airbyte_trigger_job.py +++ b/providers/tests/system/airbyte/example_airbyte_trigger_job.py @@ -61,7 +61,7 @@ # Task dependency created via `XComArgs`: # async_source_destination >> airbyte_sensor -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/teradata/__init__.py b/providers/tests/system/alibaba/__init__.py similarity index 100% rename from tests/providers/teradata/__init__.py rename to providers/tests/system/alibaba/__init__.py diff --git a/tests/system/providers/alibaba/example_adb_spark_batch.py b/providers/tests/system/alibaba/example_adb_spark_batch.py similarity index 93% rename from tests/system/providers/alibaba/example_adb_spark_batch.py rename to providers/tests/system/alibaba/example_adb_spark_batch.py index 9f23693066aec..35e03a088871d 100644 --- a/tests/system/providers/alibaba/example_adb_spark_batch.py +++ b/providers/tests/system/alibaba/example_adb_spark_batch.py @@ -50,14 +50,14 @@ spark_pi >> spark_lr - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() # [END howto_operator_adb_spark_batch] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/alibaba/example_adb_spark_sql.py b/providers/tests/system/alibaba/example_adb_spark_sql.py similarity index 93% rename from tests/system/providers/alibaba/example_adb_spark_sql.py rename to providers/tests/system/alibaba/example_adb_spark_sql.py index fcfe4b896ccba..ac29330d451db 100644 --- a/tests/system/providers/alibaba/example_adb_spark_sql.py +++ b/providers/tests/system/alibaba/example_adb_spark_sql.py @@ -42,14 +42,14 @@ show_databases >> show_tables - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() # [END howto_operator_adb_spark_sql] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/alibaba/example_oss_bucket.py b/providers/tests/system/alibaba/example_oss_bucket.py similarity index 93% rename from tests/system/providers/alibaba/example_oss_bucket.py rename to providers/tests/system/alibaba/example_oss_bucket.py index 1e39d3eb45033..4870a3b577977 100644 --- a/tests/system/providers/alibaba/example_oss_bucket.py +++ b/providers/tests/system/alibaba/example_oss_bucket.py @@ -41,14 +41,14 @@ create_bucket >> delete_bucket - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() # [END howto_operator_oss_bucket] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/alibaba/example_oss_object.py b/providers/tests/system/alibaba/example_oss_object.py similarity index 94% rename from tests/system/providers/alibaba/example_oss_object.py rename to providers/tests/system/alibaba/example_oss_object.py index 5b73fb1ba7a6a..7305c05bf8384 100644 --- a/tests/system/providers/alibaba/example_oss_object.py +++ b/providers/tests/system/alibaba/example_oss_object.py @@ -65,14 +65,14 @@ create_object >> download_object >> delete_object >> delete_batch_object - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/CONTRIBUTING.md b/providers/tests/system/amazon/CONTRIBUTING.md similarity index 99% rename from tests/system/providers/amazon/CONTRIBUTING.md rename to providers/tests/system/amazon/CONTRIBUTING.md index b9aaaa66915c2..f12062aa7fb59 100644 --- a/tests/system/providers/amazon/CONTRIBUTING.md +++ b/providers/tests/system/amazon/CONTRIBUTING.md @@ -212,7 +212,7 @@ chain( task2, # task2 has trigger rule "all done" defined ) -from tests.system.utils.watcher import watcher +from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG diff --git a/tests/system/providers/amazon/README.md b/providers/tests/system/amazon/README.md similarity index 100% rename from tests/system/providers/amazon/README.md rename to providers/tests/system/amazon/README.md diff --git a/tests/providers/ssh/hooks/__init__.py b/providers/tests/system/amazon/__init__.py similarity index 100% rename from tests/providers/ssh/hooks/__init__.py rename to providers/tests/system/amazon/__init__.py diff --git a/tests/providers/ssh/operators/__init__.py b/providers/tests/system/amazon/aws/__init__.py similarity index 100% rename from tests/providers/ssh/operators/__init__.py rename to providers/tests/system/amazon/aws/__init__.py diff --git a/tests/system/providers/amazon/aws/example_appflow.py b/providers/tests/system/amazon/aws/example_appflow.py similarity index 94% rename from tests/system/providers/amazon/aws/example_appflow.py rename to providers/tests/system/amazon/aws/example_appflow.py index 5ba38533b0211..db9dd9a428947 100644 --- a/tests/system/providers/amazon/aws/example_appflow.py +++ b/providers/tests/system/amazon/aws/example_appflow.py @@ -28,7 +28,8 @@ AppflowRunFullOperator, ) from airflow.providers.standard.operators.bash import BashOperator -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -110,13 +111,13 @@ should_be_skipped, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_appflow_run.py b/providers/tests/system/amazon/aws/example_appflow_run.py similarity index 97% rename from tests/system/providers/amazon/aws/example_appflow_run.py rename to providers/tests/system/amazon/aws/example_appflow_run.py index d73a426127808..d1151d1dd5c8b 100644 --- a/tests/system/providers/amazon/aws/example_appflow_run.py +++ b/providers/tests/system/amazon/aws/example_appflow_run.py @@ -33,7 +33,8 @@ S3DeleteBucketOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -192,13 +193,13 @@ def delete_flow(flow_name: str): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_athena.py b/providers/tests/system/amazon/aws/example_athena.py similarity index 96% rename from tests/system/providers/amazon/aws/example_athena.py rename to providers/tests/system/amazon/aws/example_athena.py index 92c56c24ee7fc..2ee1c11ab7881 100644 --- a/tests/system/providers/amazon/aws/example_athena.py +++ b/providers/tests/system/amazon/aws/example_athena.py @@ -32,7 +32,8 @@ ) from airflow.providers.amazon.aws.sensors.athena import AthenaSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -173,14 +174,14 @@ def read_results_from_s3(bucket_name, query_execution_id): delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_azure_blob_to_s3.py b/providers/tests/system/amazon/aws/example_azure_blob_to_s3.py similarity index 93% rename from tests/system/providers/amazon/aws/example_azure_blob_to_s3.py rename to providers/tests/system/amazon/aws/example_azure_blob_to_s3.py index 0109b8813063d..78a55a6a7f7dc 100644 --- a/tests/system/providers/amazon/aws/example_azure_blob_to_s3.py +++ b/providers/tests/system/amazon/aws/example_azure_blob_to_s3.py @@ -23,7 +23,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.azure_blob_to_s3 import AzureBlobStorageToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -71,13 +72,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_batch.py b/providers/tests/system/amazon/aws/example_batch.py similarity index 97% rename from tests/system/providers/amazon/aws/example_batch.py rename to providers/tests/system/amazon/aws/example_batch.py index b33078407a296..da035050a77b4 100644 --- a/tests/system/providers/amazon/aws/example_batch.py +++ b/providers/tests/system/amazon/aws/example_batch.py @@ -32,7 +32,8 @@ BatchSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ( + +from providers.tests.system.amazon.aws.utils import ( ENV_ID_KEY, SystemTestContextBuilder, prune_logs, @@ -282,13 +283,13 @@ def delete_job_queue(job_queue_name): log_cleanup, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_bedrock.py b/providers/tests/system/amazon/aws/example_bedrock.py similarity index 97% rename from tests/system/providers/amazon/aws/example_bedrock.py rename to providers/tests/system/amazon/aws/example_bedrock.py index 3c015496f992e..8e67ee4d3880b 100644 --- a/tests/system/providers/amazon/aws/example_bedrock.py +++ b/providers/tests/system/amazon/aws/example_bedrock.py @@ -43,7 +43,8 @@ ) from airflow.utils.edgemodifier import Label from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder # Externally fetched variables: ROLE_ARN_KEY = "ROLE_ARN" @@ -217,14 +218,14 @@ def run_or_skip(): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py b/providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py similarity index 99% rename from tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py rename to providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py index a1d1211da4c46..0d9eb278460f8 100644 --- a/tests/system/providers/amazon/aws/example_bedrock_retrieve_and_generate.py +++ b/providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py @@ -60,7 +60,8 @@ from airflow.providers.amazon.aws.utils import get_botocore_version from airflow.utils.edgemodifier import Label from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder ####################################################################### # NOTE: @@ -593,14 +594,14 @@ def delete_opensearch_policies(collection_name: str): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_cloudformation.py b/providers/tests/system/amazon/aws/example_cloudformation.py similarity index 95% rename from tests/system/providers/amazon/aws/example_cloudformation.py rename to providers/tests/system/amazon/aws/example_cloudformation.py index a376addd3489b..38827eebbdb5f 100644 --- a/tests/system/providers/amazon/aws/example_cloudformation.py +++ b/providers/tests/system/amazon/aws/example_cloudformation.py @@ -30,7 +30,8 @@ CloudFormationDeleteStackSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -107,14 +108,14 @@ wait_for_stack_delete, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_comprehend.py b/providers/tests/system/amazon/aws/example_comprehend.py similarity index 95% rename from tests/system/providers/amazon/aws/example_comprehend.py rename to providers/tests/system/amazon/aws/example_comprehend.py index 58e34329b67f7..d8a1f9af88b6c 100644 --- a/tests/system/providers/amazon/aws/example_comprehend.py +++ b/providers/tests/system/amazon/aws/example_comprehend.py @@ -32,7 +32,8 @@ ComprehendStartPiiEntitiesDetectionJobCompletedSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder ROLE_ARN_KEY = "ROLE_ARN" sys_test_context_task = SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build() @@ -125,13 +126,13 @@ def pii_entities_detection_job_workflow(): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_comprehend_document_classifier.py b/providers/tests/system/amazon/aws/example_comprehend_document_classifier.py similarity index 97% rename from tests/system/providers/amazon/aws/example_comprehend_document_classifier.py rename to providers/tests/system/amazon/aws/example_comprehend_document_classifier.py index 08750da98141c..f5308265a83ca 100644 --- a/tests/system/providers/amazon/aws/example_comprehend_document_classifier.py +++ b/providers/tests/system/amazon/aws/example_comprehend_document_classifier.py @@ -37,7 +37,8 @@ ) from airflow.providers.amazon.aws.transfers.http_to_s3 import HttpToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder ROLE_ARN_KEY = "ROLE_ARN" sys_test_context_task = SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build() @@ -230,13 +231,13 @@ def delete_connection(conn_id): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_datasync.py b/providers/tests/system/amazon/aws/example_datasync.py similarity index 97% rename from tests/system/providers/amazon/aws/example_datasync.py rename to providers/tests/system/amazon/aws/example_datasync.py index 8f57d1c656068..a8363fe81b36e 100644 --- a/tests/system/providers/amazon/aws/example_datasync.py +++ b/providers/tests/system/amazon/aws/example_datasync.py @@ -26,7 +26,8 @@ from airflow.providers.amazon.aws.operators.datasync import DataSyncOperator from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_datasync" @@ -234,13 +235,13 @@ def delete_locations(locations): delete_s3_bucket_destination, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_dms.py b/providers/tests/system/amazon/aws/example_dms.py similarity index 98% rename from tests/system/providers/amazon/aws/example_dms.py rename to providers/tests/system/amazon/aws/example_dms.py index 32506a0268bfb..b2e99daf6290f 100644 --- a/tests/system/providers/amazon/aws/example_dms.py +++ b/providers/tests/system/amazon/aws/example_dms.py @@ -46,8 +46,9 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.sensors.dms import DmsTaskBaseSensor, DmsTaskCompletedSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder -from tests.system.providers.amazon.aws.utils.ec2 import get_default_vpc_id + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder +from providers.tests.system.amazon.aws.utils.ec2 import get_default_vpc_id DAG_ID = "example_dms" ROLE_ARN_KEY = "ROLE_ARN" @@ -428,13 +429,13 @@ def delete_security_group(security_group_id: str, security_group_name: str): delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_dynamodb.py b/providers/tests/system/amazon/aws/example_dynamodb.py similarity index 95% rename from tests/system/providers/amazon/aws/example_dynamodb.py rename to providers/tests/system/amazon/aws/example_dynamodb.py index 6c3d770c9ed99..3d709c36b02ad 100644 --- a/tests/system/providers/amazon/aws/example_dynamodb.py +++ b/providers/tests/system/amazon/aws/example_dynamodb.py @@ -25,7 +25,8 @@ from airflow.models.dag import DAG from airflow.providers.amazon.aws.sensors.dynamodb import DynamoDBValueSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder # TODO: FIXME The argument types here seems somewhat tricky to fix # mypy: disable-error-code="arg-type" @@ -116,14 +117,14 @@ def delete_table(table_name: str): delete_table, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_dynamodb_to_s3.py b/providers/tests/system/amazon/aws/example_dynamodb_to_s3.py similarity index 97% rename from tests/system/providers/amazon/aws/example_dynamodb_to_s3.py rename to providers/tests/system/amazon/aws/example_dynamodb_to_s3.py index e22bc2080d5ec..86a5d76c9587f 100644 --- a/tests/system/providers/amazon/aws/example_dynamodb_to_s3.py +++ b/providers/tests/system/amazon/aws/example_dynamodb_to_s3.py @@ -31,7 +31,8 @@ from airflow.providers.amazon.aws.transfers.dynamodb_to_s3 import DynamoDBToS3Operator from airflow.utils.edgemodifier import Label from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder log = logging.getLogger(__name__) @@ -255,13 +256,13 @@ def skip_incremental_export(start_time: datetime, end_time: datetime): delete_table, delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_ec2.py b/providers/tests/system/amazon/aws/example_ec2.py similarity index 97% rename from tests/system/providers/amazon/aws/example_ec2.py rename to providers/tests/system/amazon/aws/example_ec2.py index 10b9c62338b90..e55adab775ffd 100644 --- a/tests/system/providers/amazon/aws/example_ec2.py +++ b/providers/tests/system/amazon/aws/example_ec2.py @@ -34,7 +34,8 @@ ) from airflow.providers.amazon.aws.sensors.ec2 import EC2InstanceStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_ec2" @@ -201,13 +202,13 @@ def parse_response(instance_ids: list): delete_key_pair(key_name), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_ecs.py b/providers/tests/system/amazon/aws/example_ecs.py similarity index 97% rename from tests/system/providers/amazon/aws/example_ecs.py rename to providers/tests/system/amazon/aws/example_ecs.py index 98617dcbcb6d4..e1aad99b5e7cb 100644 --- a/tests/system/providers/amazon/aws/example_ecs.py +++ b/providers/tests/system/amazon/aws/example_ecs.py @@ -36,7 +36,8 @@ EcsTaskDefinitionStateSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_ecs" @@ -211,14 +212,14 @@ def clean_logs(group_name: str): clean_logs(log_group_name), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_ecs_fargate.py b/providers/tests/system/amazon/aws/example_ecs_fargate.py similarity index 96% rename from tests/system/providers/amazon/aws/example_ecs_fargate.py rename to providers/tests/system/amazon/aws/example_ecs_fargate.py index 6a299c788ca19..ca592234aa77f 100644 --- a/tests/system/providers/amazon/aws/example_ecs_fargate.py +++ b/providers/tests/system/amazon/aws/example_ecs_fargate.py @@ -27,7 +27,8 @@ from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator from airflow.providers.amazon.aws.sensors.ecs import EcsTaskStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_ecs_fargate" @@ -155,13 +156,13 @@ def delete_cluster(cluster_name: str) -> None: delete_cluster(cluster_name), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_eks_templated.py b/providers/tests/system/amazon/aws/example_eks_templated.py similarity index 96% rename from tests/system/providers/amazon/aws/example_eks_templated.py rename to providers/tests/system/amazon/aws/example_eks_templated.py index 7ad8bc77c2ece..3b1ba5059725a 100644 --- a/tests/system/providers/amazon/aws/example_eks_templated.py +++ b/providers/tests/system/amazon/aws/example_eks_templated.py @@ -29,7 +29,8 @@ EksPodOperator, ) from airflow.providers.amazon.aws.sensors.eks import EksClusterStateSensor, EksNodegroupStateSensor -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -147,13 +148,13 @@ await_delete_cluster, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py b/providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py similarity index 95% rename from tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py rename to providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py index ae67a26588bdc..27ee2be0d728a 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py +++ b/providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py @@ -28,8 +28,9 @@ ) from airflow.providers.amazon.aws.sensors.eks import EksClusterStateSensor, EksFargateProfileStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder -from tests.system.providers.amazon.aws.utils.k8s import get_describe_pod_operator + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder +from providers.tests.system.amazon.aws.utils.k8s import get_describe_pod_operator DAG_ID = "example_eks_with_fargate_in_one_step" @@ -140,14 +141,14 @@ await_delete_cluster, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py b/providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py similarity index 95% rename from tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py rename to providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py index 9cce50b9bac46..34cbed1645fa5 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py +++ b/providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py @@ -30,8 +30,9 @@ ) from airflow.providers.amazon.aws.sensors.eks import EksClusterStateSensor, EksFargateProfileStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder -from tests.system.providers.amazon.aws.utils.k8s import get_describe_pod_operator + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder +from providers.tests.system.amazon.aws.utils.k8s import get_describe_pod_operator DAG_ID = "example_eks_with_fargate_profile" @@ -174,13 +175,13 @@ await_delete_cluster, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py b/providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py similarity index 95% rename from tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py rename to providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py index 9bfce2cc1696f..11f843688524c 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py +++ b/providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py @@ -31,8 +31,9 @@ ) from airflow.providers.amazon.aws.sensors.eks import EksClusterStateSensor, EksNodegroupStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder -from tests.system.providers.amazon.aws.utils.k8s import get_describe_pod_operator + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder +from providers.tests.system.amazon.aws.utils.k8s import get_describe_pod_operator DAG_ID = "example_eks_with_nodegroup_in_one_step" @@ -154,14 +155,14 @@ def delete_launch_template(template_name: str): delete_launch_template(launch_template_name), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_eks_with_nodegroups.py b/providers/tests/system/amazon/aws/example_eks_with_nodegroups.py similarity index 96% rename from tests/system/providers/amazon/aws/example_eks_with_nodegroups.py rename to providers/tests/system/amazon/aws/example_eks_with_nodegroups.py index a6681ba67f738..8356a8ee4375c 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_nodegroups.py +++ b/providers/tests/system/amazon/aws/example_eks_with_nodegroups.py @@ -33,8 +33,9 @@ ) from airflow.providers.amazon.aws.sensors.eks import EksClusterStateSensor, EksNodegroupStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder -from tests.system.providers.amazon.aws.utils.k8s import get_describe_pod_operator + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder +from providers.tests.system.amazon.aws.utils.k8s import get_describe_pod_operator DAG_ID = "example_eks_with_nodegroups" @@ -197,14 +198,14 @@ def delete_launch_template(template_name: str): delete_launch_template(launch_template_name), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_emr.py b/providers/tests/system/amazon/aws/example_emr.py similarity index 97% rename from tests/system/providers/amazon/aws/example_emr.py rename to providers/tests/system/amazon/aws/example_emr.py index 7d62180d07d9c..44c28630eb69e 100644 --- a/tests/system/providers/amazon/aws/example_emr.py +++ b/providers/tests/system/amazon/aws/example_emr.py @@ -36,7 +36,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.sensors.emr import EmrJobFlowSensor, EmrStepSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_emr" CONFIG_NAME = "EMR Runtime Role Security Configuration" @@ -226,14 +227,14 @@ def get_step_id(step_ids: list): ] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_emr_eks.py b/providers/tests/system/amazon/aws/example_emr_eks.py similarity index 98% rename from tests/system/providers/amazon/aws/example_emr_eks.py rename to providers/tests/system/amazon/aws/example_emr_eks.py index 8b182d42440bf..d95645fd306a6 100644 --- a/tests/system/providers/amazon/aws/example_emr_eks.py +++ b/providers/tests/system/amazon/aws/example_emr_eks.py @@ -36,7 +36,8 @@ from airflow.providers.amazon.aws.sensors.eks import EksClusterStateSensor, EksNodegroupStateSensor from airflow.providers.amazon.aws.sensors.emr import EmrContainerSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_emr_eks" @@ -319,13 +320,13 @@ def delete_virtual_cluster(virtual_cluster_id): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_emr_notebook_execution.py b/providers/tests/system/amazon/aws/example_emr_notebook_execution.py similarity index 95% rename from tests/system/providers/amazon/aws/example_emr_notebook_execution.py rename to providers/tests/system/amazon/aws/example_emr_notebook_execution.py index 86fc3055a1624..48d2b14c96573 100644 --- a/tests/system/providers/amazon/aws/example_emr_notebook_execution.py +++ b/providers/tests/system/amazon/aws/example_emr_notebook_execution.py @@ -27,7 +27,8 @@ EmrStopNotebookExecutionOperator, ) from airflow.providers.amazon.aws.sensors.emr import EmrNotebookExecutionSensor -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_emr_notebook" # Externally fetched variables: @@ -111,13 +112,13 @@ wait_for_execution_finish, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_emr_serverless.py b/providers/tests/system/amazon/aws/example_emr_serverless.py similarity index 96% rename from tests/system/providers/amazon/aws/example_emr_serverless.py rename to providers/tests/system/amazon/aws/example_emr_serverless.py index 1bca31de5f39e..4901fa6736059 100644 --- a/tests/system/providers/amazon/aws/example_emr_serverless.py +++ b/providers/tests/system/amazon/aws/example_emr_serverless.py @@ -32,7 +32,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.sensors.emr import EmrServerlessApplicationSensor, EmrServerlessJobSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_emr_serverless" @@ -154,13 +155,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_eventbridge.py b/providers/tests/system/amazon/aws/example_eventbridge.py similarity index 95% rename from tests/system/providers/amazon/aws/example_eventbridge.py rename to providers/tests/system/amazon/aws/example_eventbridge.py index 9b3bb922e1669..5470bd0ca70cc 100644 --- a/tests/system/providers/amazon/aws/example_eventbridge.py +++ b/providers/tests/system/amazon/aws/example_eventbridge.py @@ -26,7 +26,8 @@ EventBridgePutEventsOperator, EventBridgePutRuleOperator, ) -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_eventbridge" ENTRIES = [ @@ -80,7 +81,7 @@ chain(test_context, put_events, put_rule, enable_rule, disable_rule) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_ftp_to_s3.py b/providers/tests/system/amazon/aws/example_ftp_to_s3.py similarity index 92% rename from tests/system/providers/amazon/aws/example_ftp_to_s3.py rename to providers/tests/system/amazon/aws/example_ftp_to_s3.py index ef62bdc06a733..98a37e197f8fe 100644 --- a/tests/system/providers/amazon/aws/example_ftp_to_s3.py +++ b/providers/tests/system/amazon/aws/example_ftp_to_s3.py @@ -23,7 +23,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.ftp_to_s3 import FTPToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -71,13 +72,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_gcs_to_s3.py b/providers/tests/system/amazon/aws/example_gcs_to_s3.py similarity index 95% rename from tests/system/providers/amazon/aws/example_gcs_to_s3.py rename to providers/tests/system/amazon/aws/example_gcs_to_s3.py index 198f9729feb3f..97ed8dfda3a98 100644 --- a/tests/system/providers/amazon/aws/example_gcs_to_s3.py +++ b/providers/tests/system/amazon/aws/example_gcs_to_s3.py @@ -32,7 +32,8 @@ GCSDeleteBucketOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder # Externally fetched variables: GCP_PROJECT_ID = "GCP_PROJECT_ID" @@ -114,13 +115,13 @@ def upload_gcs_file(bucket_name: str, object_name: str, user_project: str): delete_gcs_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_glacier_to_gcs.py b/providers/tests/system/amazon/aws/example_glacier_to_gcs.py similarity index 95% rename from tests/system/providers/amazon/aws/example_glacier_to_gcs.py rename to providers/tests/system/amazon/aws/example_glacier_to_gcs.py index e57029f33aec0..28c061ae057c9 100644 --- a/tests/system/providers/amazon/aws/example_glacier_to_gcs.py +++ b/providers/tests/system/amazon/aws/example_glacier_to_gcs.py @@ -30,7 +30,8 @@ from airflow.providers.amazon.aws.sensors.glacier import GlacierJobOperationSensor from airflow.providers.amazon.aws.transfers.glacier_to_gcs import GlacierToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -107,13 +108,13 @@ def delete_vault(vault_name): delete_vault(vault_name), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_glue.py b/providers/tests/system/amazon/aws/example_glue.py similarity index 97% rename from tests/system/providers/amazon/aws/example_glue.py rename to providers/tests/system/amazon/aws/example_glue.py index c16aaf8677c17..5314d26eff80c 100644 --- a/tests/system/providers/amazon/aws/example_glue.py +++ b/providers/tests/system/amazon/aws/example_glue.py @@ -35,7 +35,8 @@ from airflow.providers.amazon.aws.sensors.glue_catalog_partition import GlueCatalogPartitionSensor from airflow.providers.amazon.aws.sensors.glue_crawler import GlueCrawlerSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs if TYPE_CHECKING: from botocore.client import BaseClient @@ -216,13 +217,13 @@ def glue_cleanup(crawler_name: str, job_name: str, db_name: str) -> None: log_cleanup, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_glue_data_quality.py b/providers/tests/system/amazon/aws/example_glue_data_quality.py similarity index 97% rename from tests/system/providers/amazon/aws/example_glue_data_quality.py rename to providers/tests/system/amazon/aws/example_glue_data_quality.py index e9b8f418e2f6e..4d05d6e22b436 100644 --- a/tests/system/providers/amazon/aws/example_glue_data_quality.py +++ b/providers/tests/system/amazon/aws/example_glue_data_quality.py @@ -34,7 +34,8 @@ ) from airflow.providers.amazon.aws.sensors.glue import GlueDataQualityRuleSetEvaluationRunSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder ROLE_ARN_KEY = "ROLE_ARN" sys_test_context_task = SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build() @@ -197,13 +198,13 @@ def delete_ruleset(ruleset_name): delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_glue_data_quality_with_recommendation.py b/providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py similarity index 97% rename from tests/system/providers/amazon/aws/example_glue_data_quality_with_recommendation.py rename to providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py index 231750e97134b..fdb3cb8e63ad8 100644 --- a/tests/system/providers/amazon/aws/example_glue_data_quality_with_recommendation.py +++ b/providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py @@ -37,7 +37,8 @@ GlueDataQualityRuleSetEvaluationRunSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder ROLE_ARN_KEY = "ROLE_ARN" sys_test_context_task = SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build() @@ -196,13 +197,13 @@ def delete_ruleset(ruleset_name): delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_glue_databrew.py b/providers/tests/system/amazon/aws/example_glue_databrew.py similarity index 95% rename from tests/system/providers/amazon/aws/example_glue_databrew.py rename to providers/tests/system/amazon/aws/example_glue_databrew.py index 251c7611b3be3..3218b465c290e 100644 --- a/tests/system/providers/amazon/aws/example_glue_databrew.py +++ b/providers/tests/system/amazon/aws/example_glue_databrew.py @@ -31,7 +31,8 @@ S3DeleteBucketOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder DAG_ID = "example_glue_databrew" @@ -154,14 +155,14 @@ def delete_job(job_name: str): delete_output_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py b/providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py similarity index 94% rename from tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py rename to providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py index b736308324f6e..7ea7ae9d072f7 100644 --- a/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py +++ b/providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py @@ -29,7 +29,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.google_api_to_s3 import GoogleApiToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -82,13 +83,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_google_api_youtube_to_s3.py b/providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py similarity index 97% rename from tests/system/providers/amazon/aws/example_google_api_youtube_to_s3.py rename to providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py index f6195f78018da..ae4f721567cec 100644 --- a/tests/system/providers/amazon/aws/example_google_api_youtube_to_s3.py +++ b/providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py @@ -61,7 +61,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.google_api_to_s3 import GoogleApiToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_google_api_youtube_to_s3" @@ -196,13 +197,13 @@ def transform_video_ids(**kwargs): delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_hive_to_dynamodb.py b/providers/tests/system/amazon/aws/example_hive_to_dynamodb.py similarity index 96% rename from tests/system/providers/amazon/aws/example_hive_to_dynamodb.py rename to providers/tests/system/amazon/aws/example_hive_to_dynamodb.py index 6b35b9dfca8e4..5582c13b855f5 100644 --- a/tests/system/providers/amazon/aws/example_hive_to_dynamodb.py +++ b/providers/tests/system/amazon/aws/example_hive_to_dynamodb.py @@ -32,7 +32,8 @@ from airflow.providers.amazon.aws.transfers.hive_to_dynamodb import HiveToDynamoDBOperator from airflow.utils import db from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder DAG_ID = "example_hive_to_dynamodb" @@ -149,13 +150,13 @@ def configure_hive_connection(connection_id, hostname): delete_dynamodb_table(dynamodb_table_name), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_http_to_s3.py b/providers/tests/system/amazon/aws/example_http_to_s3.py similarity index 94% rename from tests/system/providers/amazon/aws/example_http_to_s3.py rename to providers/tests/system/amazon/aws/example_http_to_s3.py index d6424f9802155..cfbc131ee5689 100644 --- a/tests/system/providers/amazon/aws/example_http_to_s3.py +++ b/providers/tests/system/amazon/aws/example_http_to_s3.py @@ -27,7 +27,8 @@ from airflow.providers.amazon.aws.transfers.http_to_s3 import HttpToS3Operator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -118,13 +119,13 @@ def create_connection(conn_id_name: str): delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_imap_attachment_to_s3.py b/providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py similarity index 94% rename from tests/system/providers/amazon/aws/example_imap_attachment_to_s3.py rename to providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py index 7b7fc1bb387d2..b43023115cf4d 100644 --- a/tests/system/providers/amazon/aws/example_imap_attachment_to_s3.py +++ b/providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py @@ -28,7 +28,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.imap_attachment_to_s3 import ImapAttachmentToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder DAG_ID = "example_imap_attachment_to_s3" @@ -89,13 +90,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_kinesis_analytics.py b/providers/tests/system/amazon/aws/example_kinesis_analytics.py similarity index 97% rename from tests/system/providers/amazon/aws/example_kinesis_analytics.py rename to providers/tests/system/amazon/aws/example_kinesis_analytics.py index 007a35407437d..600710cc43b4b 100644 --- a/tests/system/providers/amazon/aws/example_kinesis_analytics.py +++ b/providers/tests/system/amazon/aws/example_kinesis_analytics.py @@ -43,7 +43,8 @@ ) from airflow.providers.amazon.aws.transfers.http_to_s3 import HttpToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder ROLE_ARN_KEY = "ROLE_ARN" sys_test_context_task = SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build() @@ -262,13 +263,13 @@ def delete_kinesis_stream(stream: str, region: str): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_lambda.py b/providers/tests/system/amazon/aws/example_lambda.py similarity index 95% rename from tests/system/providers/amazon/aws/example_lambda.py rename to providers/tests/system/amazon/aws/example_lambda.py index 4ba74a26d5f22..fd346a34589f0 100644 --- a/tests/system/providers/amazon/aws/example_lambda.py +++ b/providers/tests/system/amazon/aws/example_lambda.py @@ -32,7 +32,8 @@ ) from airflow.providers.amazon.aws.sensors.lambda_function import LambdaFunctionStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs DAG_ID = "example_lambda" @@ -128,13 +129,13 @@ def delete_lambda(function_name: str): log_cleanup, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_local_to_s3.py b/providers/tests/system/amazon/aws/example_local_to_s3.py similarity index 93% rename from tests/system/providers/amazon/aws/example_local_to_s3.py rename to providers/tests/system/amazon/aws/example_local_to_s3.py index da979bcd37854..47344c429b958 100644 --- a/tests/system/providers/amazon/aws/example_local_to_s3.py +++ b/providers/tests/system/amazon/aws/example_local_to_s3.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.local_to_s3 import LocalFilesystemToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -90,14 +91,14 @@ def delete_temp_file(): delete_temp_file(), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_mongo_to_s3.py b/providers/tests/system/amazon/aws/example_mongo_to_s3.py similarity index 93% rename from tests/system/providers/amazon/aws/example_mongo_to_s3.py rename to providers/tests/system/amazon/aws/example_mongo_to_s3.py index 33bcd171c0d82..3e5bc318e5c17 100644 --- a/tests/system/providers/amazon/aws/example_mongo_to_s3.py +++ b/providers/tests/system/amazon/aws/example_mongo_to_s3.py @@ -22,7 +22,8 @@ from airflow.providers.amazon.aws.transfers.mongo_to_s3 import MongoToS3Operator from airflow.utils.dates import datetime from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder DAG_ID = "example_mongo_to_s3" @@ -82,13 +83,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_neptune.py b/providers/tests/system/amazon/aws/example_neptune.py similarity index 93% rename from tests/system/providers/amazon/aws/example_neptune.py rename to providers/tests/system/amazon/aws/example_neptune.py index 81276c2dcd86a..4685840b42594 100644 --- a/tests/system/providers/amazon/aws/example_neptune.py +++ b/providers/tests/system/amazon/aws/example_neptune.py @@ -26,7 +26,8 @@ NeptuneStartDbClusterOperator, NeptuneStopDbClusterOperator, ) -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder DAG_ID = "example_neptune" @@ -76,13 +77,13 @@ def delete_cluster(cluster_id): # TEST TEARDOWN delete_cluster(cluster_id), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_quicksight.py b/providers/tests/system/amazon/aws/example_quicksight.py similarity index 97% rename from tests/system/providers/amazon/aws/example_quicksight.py rename to providers/tests/system/amazon/aws/example_quicksight.py index ccf4746122f65..ec5cd38003148 100644 --- a/tests/system/providers/amazon/aws/example_quicksight.py +++ b/providers/tests/system/amazon/aws/example_quicksight.py @@ -33,7 +33,8 @@ ) from airflow.providers.amazon.aws.sensors.quicksight import QuickSightSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder """ Prerequisites: @@ -217,13 +218,13 @@ def delete_ingestion(aws_account_id: str, dataset_name: str, ingestion_name: str delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_rds_event.py b/providers/tests/system/amazon/aws/example_rds_event.py similarity index 95% rename from tests/system/providers/amazon/aws/example_rds_event.py rename to providers/tests/system/amazon/aws/example_rds_event.py index 0d397f43066e4..79d70c9d394e2 100644 --- a/tests/system/providers/amazon/aws/example_rds_event.py +++ b/providers/tests/system/amazon/aws/example_rds_event.py @@ -31,7 +31,8 @@ RdsDeleteEventSubscriptionOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_rds_event" @@ -118,14 +119,14 @@ def delete_sns_topic(topic_arn) -> None: delete_sns_topic(sns_topic), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_rds_export.py b/providers/tests/system/amazon/aws/example_rds_export.py similarity index 96% rename from tests/system/providers/amazon/aws/example_rds_export.py rename to providers/tests/system/amazon/aws/example_rds_export.py index 385a1871e7e27..34638f030f506 100644 --- a/tests/system/providers/amazon/aws/example_rds_export.py +++ b/providers/tests/system/amazon/aws/example_rds_export.py @@ -33,7 +33,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.sensors.rds import RdsExportTaskExistenceSensor, RdsSnapshotExistenceSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_rds_export" @@ -177,13 +178,13 @@ def get_snapshot_arn(snapshot_name: str) -> str: delete_db_instance, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_rds_instance.py b/providers/tests/system/amazon/aws/example_rds_instance.py similarity index 95% rename from tests/system/providers/amazon/aws/example_rds_instance.py rename to providers/tests/system/amazon/aws/example_rds_instance.py index f917fb128486c..ce49f73c1f4ae 100644 --- a/tests/system/providers/amazon/aws/example_rds_instance.py +++ b/providers/tests/system/amazon/aws/example_rds_instance.py @@ -28,7 +28,8 @@ ) from airflow.providers.amazon.aws.sensors.rds import RdsDbSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -110,14 +111,14 @@ delete_db_instance, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_rds_snapshot.py b/providers/tests/system/amazon/aws/example_rds_snapshot.py similarity index 96% rename from tests/system/providers/amazon/aws/example_rds_snapshot.py rename to providers/tests/system/amazon/aws/example_rds_snapshot.py index 5585f339feec2..c58c1db54d539 100644 --- a/tests/system/providers/amazon/aws/example_rds_snapshot.py +++ b/providers/tests/system/amazon/aws/example_rds_snapshot.py @@ -29,7 +29,8 @@ ) from airflow.providers.amazon.aws.sensors.rds import RdsSnapshotExistenceSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_rds_snapshot" @@ -137,14 +138,14 @@ delete_db_instance, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_redshift.py b/providers/tests/system/amazon/aws/example_redshift.py similarity index 97% rename from tests/system/providers/amazon/aws/example_redshift.py rename to providers/tests/system/amazon/aws/example_redshift.py index 67b822d41ef55..986bce5a07b52 100644 --- a/tests/system/providers/amazon/aws/example_redshift.py +++ b/providers/tests/system/amazon/aws/example_redshift.py @@ -33,7 +33,8 @@ from airflow.providers.amazon.aws.operators.redshift_data import RedshiftDataOperator from airflow.providers.amazon.aws.sensors.redshift_cluster import RedshiftClusterSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_redshift" @@ -247,13 +248,13 @@ delete_cluster_snapshot, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_redshift_s3_transfers.py b/providers/tests/system/amazon/aws/example_redshift_s3_transfers.py similarity index 98% rename from tests/system/providers/amazon/aws/example_redshift_s3_transfers.py rename to providers/tests/system/amazon/aws/example_redshift_s3_transfers.py index 9fb989ec53697..2a553eae8cb60 100644 --- a/tests/system/providers/amazon/aws/example_redshift_s3_transfers.py +++ b/providers/tests/system/amazon/aws/example_redshift_s3_transfers.py @@ -35,7 +35,8 @@ from airflow.providers.amazon.aws.transfers.redshift_to_s3 import RedshiftToS3Operator from airflow.providers.amazon.aws.transfers.s3_to_redshift import S3ToRedshiftOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_redshift_to_s3" @@ -317,13 +318,13 @@ def _insert_data(table_name: str) -> str: delete_cluster, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_s3.py b/providers/tests/system/amazon/aws/example_s3.py similarity index 98% rename from tests/system/providers/amazon/aws/example_s3.py rename to providers/tests/system/amazon/aws/example_s3.py index 06f60b4ac99cf..d016076022874 100644 --- a/tests/system/providers/amazon/aws/example_s3.py +++ b/providers/tests/system/amazon/aws/example_s3.py @@ -36,7 +36,8 @@ ) from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor, S3KeysUnchangedSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_s3" @@ -322,13 +323,13 @@ def check_fn(files: list, **kwargs) -> bool: delete_bucket, delete_bucket_2, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_s3_to_dynamodb.py b/providers/tests/system/amazon/aws/example_s3_to_dynamodb.py similarity index 96% rename from tests/system/providers/amazon/aws/example_s3_to_dynamodb.py rename to providers/tests/system/amazon/aws/example_s3_to_dynamodb.py index b415ffad7bdc1..efd762cfd93c1 100644 --- a/tests/system/providers/amazon/aws/example_s3_to_dynamodb.py +++ b/providers/tests/system/amazon/aws/example_s3_to_dynamodb.py @@ -31,7 +31,8 @@ ) from airflow.providers.amazon.aws.transfers.s3_to_dynamodb import S3ToDynamoDBOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder log = logging.getLogger(__name__) @@ -180,13 +181,13 @@ def delete_dynamodb_table(table_name: str): delete_new_table, delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_s3_to_ftp.py b/providers/tests/system/amazon/aws/example_s3_to_ftp.py similarity index 92% rename from tests/system/providers/amazon/aws/example_s3_to_ftp.py rename to providers/tests/system/amazon/aws/example_s3_to_ftp.py index bf909ddc8a219..0ffb4722ba867 100644 --- a/tests/system/providers/amazon/aws/example_s3_to_ftp.py +++ b/providers/tests/system/amazon/aws/example_s3_to_ftp.py @@ -23,7 +23,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.s3_to_ftp import S3ToFTPOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -70,13 +71,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_s3_to_sftp.py b/providers/tests/system/amazon/aws/example_s3_to_sftp.py similarity index 92% rename from tests/system/providers/amazon/aws/example_s3_to_sftp.py rename to providers/tests/system/amazon/aws/example_s3_to_sftp.py index 78825d7151468..5270d3ec7d4b9 100644 --- a/tests/system/providers/amazon/aws/example_s3_to_sftp.py +++ b/providers/tests/system/amazon/aws/example_s3_to_sftp.py @@ -23,7 +23,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.s3_to_sftp import S3ToSFTPOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -70,13 +71,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_s3_to_sql.py b/providers/tests/system/amazon/aws/example_s3_to_sql.py similarity index 97% rename from tests/system/providers/amazon/aws/example_s3_to_sql.py rename to providers/tests/system/amazon/aws/example_s3_to_sql.py index 5e5e2ba010e89..e7e8a5e9543cf 100644 --- a/tests/system/providers/amazon/aws/example_s3_to_sql.py +++ b/providers/tests/system/amazon/aws/example_s3_to_sql.py @@ -39,8 +39,9 @@ from airflow.providers.amazon.aws.transfers.s3_to_sql import S3ToSqlOperator from airflow.providers.common.sql.operators.sql import SQLTableCheckOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder -from tests.system.utils.watcher import watcher + +from dev.tests_common.test_utils.watcher import watcher +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder # Externally fetched variables: SECURITY_GROUP_KEY = "SECURITY_GROUP" @@ -253,7 +254,7 @@ def parse_csv_to_generator(filepath): list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sagemaker.py b/providers/tests/system/amazon/aws/example_sagemaker.py similarity index 99% rename from tests/system/providers/amazon/aws/example_sagemaker.py rename to providers/tests/system/amazon/aws/example_sagemaker.py index 15b756494eac3..acb05400b951e 100644 --- a/tests/system/providers/amazon/aws/example_sagemaker.py +++ b/providers/tests/system/amazon/aws/example_sagemaker.py @@ -52,7 +52,8 @@ SageMakerTuningSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs logger = logging.getLogger(__name__) @@ -645,13 +646,13 @@ def delete_docker_image(image_name): log_cleanup, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sagemaker_endpoint.py b/providers/tests/system/amazon/aws/example_sagemaker_endpoint.py similarity index 97% rename from tests/system/providers/amazon/aws/example_sagemaker_endpoint.py rename to providers/tests/system/amazon/aws/example_sagemaker_endpoint.py index 226b690cbf409..fecaa92264b6e 100644 --- a/tests/system/providers/amazon/aws/example_sagemaker_endpoint.py +++ b/providers/tests/system/amazon/aws/example_sagemaker_endpoint.py @@ -38,7 +38,8 @@ ) from airflow.providers.amazon.aws.sensors.sagemaker import SageMakerEndpointSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder, prune_logs DAG_ID = "example_sagemaker_endpoint" @@ -58,7 +59,7 @@ } # For an example of how to obtain the following train and test data, please see -# https://github.com/apache/airflow/blob/main/tests/system/providers/amazon/aws/example_sagemaker.py +# https://github.com/apache/airflow/blob/main/providers/tests/system/amazon/aws/example_sagemaker.py TRAIN_DATA = "0,4.9,2.5,4.5,1.7\n1,7.0,3.2,4.7,1.4\n0,7.3,2.9,6.3,1.8\n2,5.1,3.5,1.4,0.2\n" SAMPLE_TEST_DATA = "6.4,3.2,4.5,1.5" @@ -290,13 +291,13 @@ def set_up(env_id, role_arn, ti=None): archive_logs(f"/aws/sagemaker/Endpoints/{test_setup['endpoint_name']}"), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sagemaker_notebook.py b/providers/tests/system/amazon/aws/example_sagemaker_notebook.py similarity index 94% rename from tests/system/providers/amazon/aws/example_sagemaker_notebook.py rename to providers/tests/system/amazon/aws/example_sagemaker_notebook.py index b61bfa822bb04..76d35dc2b1479 100644 --- a/tests/system/providers/amazon/aws/example_sagemaker_notebook.py +++ b/providers/tests/system/amazon/aws/example_sagemaker_notebook.py @@ -26,7 +26,8 @@ SageMakerStartNoteBookOperator, SageMakerStopNotebookOperator, ) -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_sagemaker_notebook" @@ -96,13 +97,13 @@ delete_instance, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sagemaker_pipeline.py b/providers/tests/system/amazon/aws/example_sagemaker_pipeline.py similarity index 97% rename from tests/system/providers/amazon/aws/example_sagemaker_pipeline.py rename to providers/tests/system/amazon/aws/example_sagemaker_pipeline.py index 3f32511702401..614fb5c13cc9e 100644 --- a/tests/system/providers/amazon/aws/example_sagemaker_pipeline.py +++ b/providers/tests/system/amazon/aws/example_sagemaker_pipeline.py @@ -31,8 +31,9 @@ SageMakerPipelineSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.example_sagemaker import delete_experiments -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.example_sagemaker import delete_experiments +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_sagemaker_pipeline" @@ -116,13 +117,13 @@ def delete_pipeline(name: str): delete_experiments([pipeline_name]), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_salesforce_to_s3.py b/providers/tests/system/amazon/aws/example_salesforce_to_s3.py similarity index 93% rename from tests/system/providers/amazon/aws/example_salesforce_to_s3.py rename to providers/tests/system/amazon/aws/example_salesforce_to_s3.py index 112be85c5ad29..7ee15f7030beb 100644 --- a/tests/system/providers/amazon/aws/example_salesforce_to_s3.py +++ b/providers/tests/system/amazon/aws/example_salesforce_to_s3.py @@ -28,7 +28,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.salesforce_to_s3 import SalesforceToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -77,13 +78,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sftp_to_s3.py b/providers/tests/system/amazon/aws/example_sftp_to_s3.py similarity index 92% rename from tests/system/providers/amazon/aws/example_sftp_to_s3.py rename to providers/tests/system/amazon/aws/example_sftp_to_s3.py index c5cae1ebfa68c..85dac988cb82e 100644 --- a/tests/system/providers/amazon/aws/example_sftp_to_s3.py +++ b/providers/tests/system/amazon/aws/example_sftp_to_s3.py @@ -23,7 +23,8 @@ from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.sftp_to_s3 import SFTPToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -70,13 +71,13 @@ delete_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sns.py b/providers/tests/system/amazon/aws/example_sns.py similarity index 92% rename from tests/system/providers/amazon/aws/example_sns.py rename to providers/tests/system/amazon/aws/example_sns.py index 4cfb2b89153c8..d00227536da0b 100644 --- a/tests/system/providers/amazon/aws/example_sns.py +++ b/providers/tests/system/amazon/aws/example_sns.py @@ -25,7 +25,8 @@ from airflow.models.dag import DAG from airflow.providers.amazon.aws.operators.sns import SnsPublishOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -74,14 +75,14 @@ def delete_topic(topic_arn) -> None: delete_topic(create_sns_topic), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sql_to_s3.py b/providers/tests/system/amazon/aws/example_sql_to_s3.py similarity index 96% rename from tests/system/providers/amazon/aws/example_sql_to_s3.py rename to providers/tests/system/amazon/aws/example_sql_to_s3.py index d52ca3630b9d5..dd333073a0431 100644 --- a/tests/system/providers/amazon/aws/example_sql_to_s3.py +++ b/providers/tests/system/amazon/aws/example_sql_to_s3.py @@ -35,7 +35,8 @@ from airflow.providers.amazon.aws.sensors.redshift_cluster import RedshiftClusterSensor from airflow.providers.amazon.aws.transfers.sql_to_s3 import SqlToS3Operator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_sql_to_s3" @@ -198,14 +199,14 @@ def create_connection(conn_id_name: str, cluster_id: str): delete_cluster, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_sqs.py b/providers/tests/system/amazon/aws/example_sqs.py similarity index 94% rename from tests/system/providers/amazon/aws/example_sqs.py rename to providers/tests/system/amazon/aws/example_sqs.py index 0a14d5c446408..2aba6a580c4d7 100644 --- a/tests/system/providers/amazon/aws/example_sqs.py +++ b/providers/tests/system/amazon/aws/example_sqs.py @@ -25,7 +25,8 @@ from airflow.providers.amazon.aws.operators.sqs import SqsPublishOperator from airflow.providers.amazon.aws.sensors.sqs import SqsSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -99,14 +100,14 @@ def delete_queue(queue_url): delete_queue(sqs_queue), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/example_step_functions.py b/providers/tests/system/amazon/aws/example_step_functions.py similarity index 95% rename from tests/system/providers/amazon/aws/example_step_functions.py rename to providers/tests/system/amazon/aws/example_step_functions.py index 1c0e90979d694..beeb12528c7ac 100644 --- a/tests/system/providers/amazon/aws/example_step_functions.py +++ b/providers/tests/system/amazon/aws/example_step_functions.py @@ -28,7 +28,8 @@ StepFunctionStartExecutionOperator, ) from airflow.providers.amazon.aws.sensors.step_function import StepFunctionExecutionSensor -from tests.system.providers.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder DAG_ID = "example_step_functions" @@ -111,13 +112,13 @@ def delete_state_machine(state_machine_arn): delete_state_machine(state_machine_arn), ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/trino/assets/__init__.py b/providers/tests/system/amazon/aws/tests/__init__.py similarity index 100% rename from tests/providers/trino/assets/__init__.py rename to providers/tests/system/amazon/aws/tests/__init__.py diff --git a/tests/system/providers/amazon/aws/tests/test_aws_auth_manager.py b/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py similarity index 91% rename from tests/system/providers/amazon/aws/tests/test_aws_auth_manager.py rename to providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py index dac7398a1ba8f..774aec21b2b6b 100644 --- a/tests/system/providers/amazon/aws/tests/test_aws_auth_manager.py +++ b/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py @@ -23,9 +23,10 @@ import pytest from airflow.www import app as application -from tests.system.providers.amazon.aws.utils import set_env_id -from tests.test_utils.config import conf_vars -from tests.test_utils.www import check_content_in_response + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.www import check_content_in_response +from providers.tests.system.amazon.aws.utils import set_env_id SAML_METADATA_URL = "/saml/metadata" SAML_METADATA_PARSED = { @@ -124,11 +125,14 @@ def base_app(region_name, avp_policy_store_id): ("aws_auth_manager", "avp_policy_store_id"): avp_policy_store_id, } ): - with patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" - ) as mock_parser, patch( - "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" - ) as mock_init_saml_auth: + with ( + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.OneLogin_Saml2_IdPMetadataParser" + ) as mock_parser, + patch( + "airflow.providers.amazon.aws.auth_manager.views.auth.AwsAuthManagerAuthenticationViews._init_saml_auth" + ) as mock_init_saml_auth, + ): mock_parser.parse_remote.return_value = SAML_METADATA_PARSED yield mock_init_saml_auth diff --git a/tests/system/providers/amazon/aws/utils/__init__.py b/providers/tests/system/amazon/aws/utils/__init__.py similarity index 99% rename from tests/system/providers/amazon/aws/utils/__init__.py rename to providers/tests/system/amazon/aws/utils/__init__.py index 8b4114fc90ad0..1b6b5cc47c1dc 100644 --- a/tests/system/providers/amazon/aws/utils/__init__.py +++ b/providers/tests/system/amazon/aws/utils/__init__.py @@ -48,7 +48,7 @@ INVALID_ENV_ID_MSG: str = ( "To maximize compatibility, the SYSTEM_TESTS_ENV_ID must be an alphanumeric string " - "which starts with a letter. Please see `tests/system/providers/amazon/README.md`." + "which starts with a letter. Please see `providers/tests/system/amazon/README.md`." ) LOWERCASE_ENV_ID_MSG: str = ( "The provided Environment ID contains uppercase letters and " diff --git a/tests/system/providers/amazon/aws/utils/ec2.py b/providers/tests/system/amazon/aws/utils/ec2.py similarity index 100% rename from tests/system/providers/amazon/aws/utils/ec2.py rename to providers/tests/system/amazon/aws/utils/ec2.py diff --git a/tests/system/providers/amazon/aws/utils/k8s.py b/providers/tests/system/amazon/aws/utils/k8s.py similarity index 100% rename from tests/system/providers/amazon/aws/utils/k8s.py rename to providers/tests/system/amazon/aws/utils/k8s.py diff --git a/tests/providers/trino/operators/__init__.py b/providers/tests/system/apache/__init__.py similarity index 100% rename from tests/providers/trino/operators/__init__.py rename to providers/tests/system/apache/__init__.py diff --git a/tests/providers/tableau/hooks/__init__.py b/providers/tests/system/apache/beam/__init__.py similarity index 100% rename from tests/providers/tableau/hooks/__init__.py rename to providers/tests/system/apache/beam/__init__.py diff --git a/tests/system/providers/apache/beam/example_beam.py b/providers/tests/system/apache/beam/example_beam.py similarity index 95% rename from tests/system/providers/apache/beam/example_beam.py rename to providers/tests/system/apache/beam/example_beam.py index b0c62bbc00baa..4166caa3477a4 100644 --- a/tests/system/providers/apache/beam/example_beam.py +++ b/providers/tests/system/apache/beam/example_beam.py @@ -24,7 +24,8 @@ from airflow import models from airflow.providers.apache.beam.operators.beam import BeamRunJavaPipelineOperator from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( GCS_INPUT, GCS_JAR_DIRECT_RUNNER_BUCKET_NAME, GCS_JAR_DIRECT_RUNNER_OBJECT_NAME, @@ -60,7 +61,7 @@ # [END howto_operator_start_java_direct_runner_pipeline] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_beam_java_flink.py b/providers/tests/system/apache/beam/example_beam_java_flink.py similarity index 94% rename from tests/system/providers/apache/beam/example_beam_java_flink.py rename to providers/tests/system/apache/beam/example_beam_java_flink.py index 42cfad2bdc8c6..a68637d8ef8d9 100644 --- a/tests/system/providers/apache/beam/example_beam_java_flink.py +++ b/providers/tests/system/apache/beam/example_beam_java_flink.py @@ -24,7 +24,8 @@ from airflow import models from airflow.providers.apache.beam.operators.beam import BeamRunJavaPipelineOperator from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( GCS_INPUT, GCS_JAR_FLINK_RUNNER_BUCKET_NAME, GCS_JAR_FLINK_RUNNER_OBJECT_NAME, @@ -59,7 +60,7 @@ jar_to_local_flink_runner >> start_java_pipeline_flink_runner -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_beam_java_spark.py b/providers/tests/system/apache/beam/example_beam_java_spark.py similarity index 94% rename from tests/system/providers/apache/beam/example_beam_java_spark.py rename to providers/tests/system/apache/beam/example_beam_java_spark.py index 29259dd63a253..eb3fea6d16dab 100644 --- a/tests/system/providers/apache/beam/example_beam_java_spark.py +++ b/providers/tests/system/apache/beam/example_beam_java_spark.py @@ -24,7 +24,8 @@ from airflow import models from airflow.providers.apache.beam.operators.beam import BeamRunJavaPipelineOperator from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( GCS_INPUT, GCS_JAR_SPARK_RUNNER_BUCKET_NAME, GCS_JAR_SPARK_RUNNER_OBJECT_NAME, @@ -59,7 +60,7 @@ jar_to_local_spark_runner >> start_java_pipeline_spark_runner -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_go.py b/providers/tests/system/apache/beam/example_go.py similarity index 96% rename from tests/system/providers/apache/beam/example_go.py rename to providers/tests/system/apache/beam/example_go.py index f5fc70ff1e917..792eee088bf6d 100644 --- a/tests/system/providers/apache/beam/example_go.py +++ b/providers/tests/system/apache/beam/example_go.py @@ -24,7 +24,8 @@ from airflow import models from airflow.providers.apache.beam.operators.beam import BeamRunGoPipelineOperator from airflow.providers.google.cloud.operators.dataflow import DataflowConfiguration -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( DEFAULT_ARGS, GCP_PROJECT_ID, GCS_GO, @@ -102,7 +103,7 @@ ) -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_go_dataflow.py b/providers/tests/system/apache/beam/example_go_dataflow.py similarity index 95% rename from tests/system/providers/apache/beam/example_go_dataflow.py rename to providers/tests/system/apache/beam/example_go_dataflow.py index 4cc81d9263042..35a63cb56e1e7 100644 --- a/tests/system/providers/apache/beam/example_go_dataflow.py +++ b/providers/tests/system/apache/beam/example_go_dataflow.py @@ -26,7 +26,8 @@ from airflow.providers.google.cloud.hooks.dataflow import DataflowJobStatus from airflow.providers.google.cloud.operators.dataflow import DataflowConfiguration from airflow.providers.google.cloud.sensors.dataflow import DataflowJobStatusSensor -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( DEFAULT_ARGS, GCP_PROJECT_ID, GCS_GO_DATAFLOW_ASYNC, @@ -75,7 +76,7 @@ # [END howto_operator_start_go_dataflow_runner_pipeline_async_gcs_file] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_java_dataflow.py b/providers/tests/system/apache/beam/example_java_dataflow.py similarity index 95% rename from tests/system/providers/apache/beam/example_java_dataflow.py rename to providers/tests/system/apache/beam/example_java_dataflow.py index 0a941eb32a5a3..334142dfd315c 100644 --- a/tests/system/providers/apache/beam/example_java_dataflow.py +++ b/providers/tests/system/apache/beam/example_java_dataflow.py @@ -24,7 +24,8 @@ from airflow import models from airflow.providers.apache.beam.operators.beam import BeamRunJavaPipelineOperator from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( GCS_JAR_DATAFLOW_RUNNER_BUCKET_NAME, GCS_JAR_DATAFLOW_RUNNER_OBJECT_NAME, GCS_OUTPUT, @@ -65,7 +66,7 @@ # [END howto_operator_start_java_dataflow_runner_pipeline] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_python.py b/providers/tests/system/apache/beam/example_python.py similarity index 97% rename from tests/system/providers/apache/beam/example_python.py rename to providers/tests/system/apache/beam/example_python.py index 12907df08f5ac..d685cb33d2dad 100644 --- a/tests/system/providers/apache/beam/example_python.py +++ b/providers/tests/system/apache/beam/example_python.py @@ -24,7 +24,8 @@ from airflow import models from airflow.providers.apache.beam.operators.beam import BeamRunPythonPipelineOperator from airflow.providers.google.cloud.operators.dataflow import DataflowConfiguration -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( DEFAULT_ARGS, GCP_PROJECT_ID, GCS_OUTPUT, @@ -119,7 +120,7 @@ ) -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_python_async.py b/providers/tests/system/apache/beam/example_python_async.py similarity index 97% rename from tests/system/providers/apache/beam/example_python_async.py rename to providers/tests/system/apache/beam/example_python_async.py index ab2e9a8063ea4..8465278b6bf76 100644 --- a/tests/system/providers/apache/beam/example_python_async.py +++ b/providers/tests/system/apache/beam/example_python_async.py @@ -24,7 +24,8 @@ from airflow import models from airflow.providers.apache.beam.operators.beam import BeamRunPythonPipelineOperator from airflow.providers.google.cloud.operators.dataflow import DataflowConfiguration -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( DEFAULT_ARGS, GCP_PROJECT_ID, GCS_OUTPUT, @@ -128,7 +129,7 @@ ) -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/example_python_dataflow.py b/providers/tests/system/apache/beam/example_python_dataflow.py similarity index 96% rename from tests/system/providers/apache/beam/example_python_dataflow.py rename to providers/tests/system/apache/beam/example_python_dataflow.py index 9b849e3900d9c..36d6b3b8562e3 100644 --- a/tests/system/providers/apache/beam/example_python_dataflow.py +++ b/providers/tests/system/apache/beam/example_python_dataflow.py @@ -26,7 +26,8 @@ from airflow.providers.google.cloud.hooks.dataflow import DataflowJobStatus from airflow.providers.google.cloud.operators.dataflow import DataflowConfiguration from airflow.providers.google.cloud.sensors.dataflow import DataflowJobStatusSensor -from tests.system.providers.apache.beam.utils import ( + +from providers.tests.system.apache.beam.utils import ( DEFAULT_ARGS, GCP_PROJECT_ID, GCS_OUTPUT, @@ -78,7 +79,7 @@ # [END howto_operator_start_python_dataflow_runner_pipeline_async_gcs_file] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/beam/utils.py b/providers/tests/system/apache/beam/utils.py similarity index 100% rename from tests/system/providers/apache/beam/utils.py rename to providers/tests/system/apache/beam/utils.py diff --git a/tests/providers/weaviate/__init__.py b/providers/tests/system/apache/cassandra/__init__.py similarity index 100% rename from tests/providers/weaviate/__init__.py rename to providers/tests/system/apache/cassandra/__init__.py diff --git a/tests/system/providers/apache/cassandra/example_cassandra_dag.py b/providers/tests/system/apache/cassandra/example_cassandra_dag.py similarity index 96% rename from tests/system/providers/apache/cassandra/example_cassandra_dag.py rename to providers/tests/system/apache/cassandra/example_cassandra_dag.py index 79e9918b446bf..0b25fa2975dbd 100644 --- a/tests/system/providers/apache/cassandra/example_cassandra_dag.py +++ b/providers/tests/system/apache/cassandra/example_cassandra_dag.py @@ -50,7 +50,7 @@ record_sensor = CassandraRecordSensor(task_id="cassandra_record_sensor", keys={"p1": "v1", "p2": "v2"}) # [END howto_operator_cassandra_sensors] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/weaviate/hooks/__init__.py b/providers/tests/system/apache/drill/__init__.py similarity index 100% rename from tests/providers/weaviate/hooks/__init__.py rename to providers/tests/system/apache/drill/__init__.py diff --git a/tests/system/providers/apache/drill/example_drill_dag.py b/providers/tests/system/apache/drill/example_drill_dag.py similarity index 95% rename from tests/system/providers/apache/drill/example_drill_dag.py rename to providers/tests/system/apache/drill/example_drill_dag.py index 62f332f318a8c..fdaf376d5a162 100644 --- a/tests/system/providers/apache/drill/example_drill_dag.py +++ b/providers/tests/system/apache/drill/example_drill_dag.py @@ -47,7 +47,7 @@ ) # [END howto_operator_drill] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/weaviate/operators/__init__.py b/providers/tests/system/apache/druid/__init__.py similarity index 100% rename from tests/providers/weaviate/operators/__init__.py rename to providers/tests/system/apache/druid/__init__.py diff --git a/tests/system/providers/apache/druid/example_druid_dag.py b/providers/tests/system/apache/druid/example_druid_dag.py similarity index 96% rename from tests/system/providers/apache/druid/example_druid_dag.py rename to providers/tests/system/apache/druid/example_druid_dag.py index 1426a12456bae..5e1bfc446bb09 100644 --- a/tests/system/providers/apache/druid/example_druid_dag.py +++ b/providers/tests/system/apache/druid/example_druid_dag.py @@ -55,7 +55,7 @@ """ # [END howto_operator_druid_submit] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/telegram/__init__.py b/providers/tests/system/apache/hive/__init__.py similarity index 100% rename from tests/providers/telegram/__init__.py rename to providers/tests/system/apache/hive/__init__.py diff --git a/tests/system/providers/apache/hive/example_twitter_README.md b/providers/tests/system/apache/hive/example_twitter_README.md similarity index 100% rename from tests/system/providers/apache/hive/example_twitter_README.md rename to providers/tests/system/apache/hive/example_twitter_README.md diff --git a/tests/system/providers/apache/hive/example_twitter_dag.py b/providers/tests/system/apache/hive/example_twitter_dag.py similarity index 97% rename from tests/system/providers/apache/hive/example_twitter_dag.py rename to providers/tests/system/apache/hive/example_twitter_dag.py index 4ceb119ba551c..129a8ebf9f77f 100644 --- a/tests/system/providers/apache/hive/example_twitter_dag.py +++ b/providers/tests/system/apache/hive/example_twitter_dag.py @@ -154,13 +154,13 @@ def transfer_to_db(): analyze >> load_to_hdfs >> load_to_hive >> hive_to_mysql - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/telegram/hooks/__init__.py b/providers/tests/system/apache/iceberg/__init__.py similarity index 100% rename from tests/providers/telegram/hooks/__init__.py rename to providers/tests/system/apache/iceberg/__init__.py diff --git a/tests/system/providers/apache/iceberg/example_iceberg.py b/providers/tests/system/apache/iceberg/example_iceberg.py similarity index 95% rename from tests/system/providers/apache/iceberg/example_iceberg.py rename to providers/tests/system/apache/iceberg/example_iceberg.py index 41e751624b5c0..a4708805b6788 100644 --- a/tests/system/providers/apache/iceberg/example_iceberg.py +++ b/providers/tests/system/apache/iceberg/example_iceberg.py @@ -49,7 +49,7 @@ ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/yandex/__init__.py b/providers/tests/system/apache/kafka/__init__.py similarity index 100% rename from tests/providers/yandex/__init__.py rename to providers/tests/system/apache/kafka/__init__.py diff --git a/tests/system/providers/apache/kafka/example_dag_event_listener.py b/providers/tests/system/apache/kafka/example_dag_event_listener.py similarity index 97% rename from tests/system/providers/apache/kafka/example_dag_event_listener.py rename to providers/tests/system/apache/kafka/example_dag_event_listener.py index 24d8177ce8cab..eb6866735e830 100644 --- a/tests/system/providers/apache/kafka/example_dag_event_listener.py +++ b/providers/tests/system/apache/kafka/example_dag_event_listener.py @@ -120,7 +120,7 @@ def wait_for_event(message, **context): t0 >> t1 -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/kafka/example_dag_hello_kafka.py b/providers/tests/system/apache/kafka/example_dag_hello_kafka.py similarity index 98% rename from tests/system/providers/apache/kafka/example_dag_hello_kafka.py rename to providers/tests/system/apache/kafka/example_dag_hello_kafka.py index fc7078c3dc5a3..5e70d7324a1a0 100644 --- a/tests/system/providers/apache/kafka/example_dag_hello_kafka.py +++ b/providers/tests/system/apache/kafka/example_dag_hello_kafka.py @@ -240,7 +240,7 @@ def hello_kafka(): t0 >> t3 >> [t4, t4b] >> t5 >> t6 -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/yandex/hooks/__init__.py b/providers/tests/system/apache/kylin/__init__.py similarity index 100% rename from tests/providers/yandex/hooks/__init__.py rename to providers/tests/system/apache/kylin/__init__.py diff --git a/tests/system/providers/apache/kylin/example_kylin_dag.py b/providers/tests/system/apache/kylin/example_kylin_dag.py similarity index 96% rename from tests/system/providers/apache/kylin/example_kylin_dag.py rename to providers/tests/system/apache/kylin/example_kylin_dag.py index c851ad7c5e3df..1bfec2cd1a7dd 100644 --- a/tests/system/providers/apache/kylin/example_kylin_dag.py +++ b/providers/tests/system/apache/kylin/example_kylin_dag.py @@ -109,13 +109,13 @@ def gen_build_time(): # gen_build_time >> refresh_task1 # gen_build_time >> merge_task # gen_build_time >> build_task3 - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/yandex/links/__init__.py b/providers/tests/system/apache/livy/__init__.py similarity index 100% rename from tests/providers/yandex/links/__init__.py rename to providers/tests/system/apache/livy/__init__.py diff --git a/tests/system/providers/apache/livy/example_livy.py b/providers/tests/system/apache/livy/example_livy.py similarity index 95% rename from tests/system/providers/apache/livy/example_livy.py rename to providers/tests/system/apache/livy/example_livy.py index 507bbaefede43..09f8f7e244e02 100644 --- a/tests/system/providers/apache/livy/example_livy.py +++ b/providers/tests/system/apache/livy/example_livy.py @@ -73,13 +73,13 @@ livy_java_task_deferrable >> livy_python_task_deferrable # [END create_livy_deferrable] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/yandex/operators/__init__.py b/providers/tests/system/apache/pig/__init__.py similarity index 100% rename from tests/providers/yandex/operators/__init__.py rename to providers/tests/system/apache/pig/__init__.py diff --git a/tests/system/providers/apache/pig/example_pig.py b/providers/tests/system/apache/pig/example_pig.py similarity index 95% rename from tests/system/providers/apache/pig/example_pig.py rename to providers/tests/system/apache/pig/example_pig.py index 938987f32e78f..bb556a85111d1 100644 --- a/tests/system/providers/apache/pig/example_pig.py +++ b/providers/tests/system/apache/pig/example_pig.py @@ -44,7 +44,7 @@ # [END create_pig] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/yandex/secrets/__init__.py b/providers/tests/system/apache/pinot/__init__.py similarity index 100% rename from tests/providers/yandex/secrets/__init__.py rename to providers/tests/system/apache/pinot/__init__.py diff --git a/tests/system/providers/apache/pinot/example_pinot_dag.py b/providers/tests/system/apache/pinot/example_pinot_dag.py similarity index 96% rename from tests/system/providers/apache/pinot/example_pinot_dag.py rename to providers/tests/system/apache/pinot/example_pinot_dag.py index eecd6a4c2abed..d95022f366910 100644 --- a/tests/system/providers/apache/pinot/example_pinot_dag.py +++ b/providers/tests/system/apache/pinot/example_pinot_dag.py @@ -52,7 +52,7 @@ def pinot_dbi_api(): pinot_admin() pinot_dbi_api() -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/yandex/utils/__init__.py b/providers/tests/system/apache/spark/__init__.py similarity index 100% rename from tests/providers/yandex/utils/__init__.py rename to providers/tests/system/apache/spark/__init__.py diff --git a/tests/system/providers/apache/spark/example_pyspark.py b/providers/tests/system/apache/spark/example_pyspark.py similarity index 96% rename from tests/system/providers/apache/spark/example_pyspark.py rename to providers/tests/system/apache/spark/example_pyspark.py index cc18911a38c72..0ca14a76f48f9 100644 --- a/tests/system/providers/apache/spark/example_pyspark.py +++ b/providers/tests/system/apache/spark/example_pyspark.py @@ -70,7 +70,7 @@ def print_df(df: pd.DataFrame): dag = example_pyspark() # type: ignore -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/spark/example_spark_dag.py b/providers/tests/system/apache/spark/example_spark_dag.py similarity index 97% rename from tests/system/providers/apache/spark/example_spark_dag.py rename to providers/tests/system/apache/spark/example_spark_dag.py index 00bb415585c0b..b68fc2cf761a9 100644 --- a/tests/system/providers/apache/spark/example_spark_dag.py +++ b/providers/tests/system/apache/spark/example_spark_dag.py @@ -75,7 +75,7 @@ ) # [END howto_operator_spark_sql] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/ydb/__init__.py b/providers/tests/system/asana/__init__.py similarity index 100% rename from tests/providers/ydb/__init__.py rename to providers/tests/system/asana/__init__.py diff --git a/tests/system/providers/asana/example_asana.py b/providers/tests/system/asana/example_asana.py similarity index 97% rename from tests/system/providers/asana/example_asana.py rename to providers/tests/system/asana/example_asana.py index 79a3636a1cb77..ca68b6cc3dd2e 100644 --- a/tests/system/providers/asana/example_asana.py +++ b/providers/tests/system/asana/example_asana.py @@ -100,13 +100,13 @@ create >> find >> update >> delete # [END asana_example_dag] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/ydb/hooks/__init__.py b/providers/tests/system/cncf/__init__.py similarity index 100% rename from tests/providers/ydb/hooks/__init__.py rename to providers/tests/system/cncf/__init__.py diff --git a/tests/providers/ydb/operators/__init__.py b/providers/tests/system/cncf/kubernetes/__init__.py similarity index 100% rename from tests/providers/ydb/operators/__init__.py rename to providers/tests/system/cncf/kubernetes/__init__.py diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes.py b/providers/tests/system/cncf/kubernetes/example_kubernetes.py similarity index 97% rename from tests/system/providers/cncf/kubernetes/example_kubernetes.py rename to providers/tests/system/cncf/kubernetes/example_kubernetes.py index 3756d0c4e21d8..bed3b9da746b9 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes.py @@ -167,13 +167,13 @@ write_xcom >> pod_task_xcom_result # [END howto_operator_k8s_write_xcom] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py similarity index 98% rename from tests/system/providers/cncf/kubernetes/example_kubernetes_async.py rename to providers/tests/system/cncf/kubernetes/example_kubernetes_async.py index cb3d25a33fcbc..43eb0a1f919cf 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py @@ -198,13 +198,13 @@ write_xcom_async >> pod_task_xcom_result_async # [END howto_operator_k8s_write_xcom_async] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_decorator.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py similarity index 97% rename from tests/system/providers/cncf/kubernetes/example_kubernetes_decorator.py rename to providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py index 20fd7d5f74fa7..088a3fdae2ede 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_decorator.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py @@ -63,7 +63,7 @@ def print_pattern(): # [END howto_operator_kubernetes] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_job.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_job.py similarity index 95% rename from tests/system/providers/cncf/kubernetes/example_kubernetes_job.py rename to providers/tests/system/cncf/kubernetes/example_kubernetes_job.py index bf017120674bc..1a2432c135898 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_job.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_job.py @@ -94,13 +94,13 @@ k8s_job >> update_job >> delete_job_task k8s_job_def >> delete_job_task_def - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_resource.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py similarity index 94% rename from tests/system/providers/cncf/kubernetes/example_kubernetes_resource.py rename to providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py index 2133d6e7a79af..1c24a533a85f0 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_resource.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py @@ -72,13 +72,13 @@ t1 >> t2 - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cncf/kubernetes/example_spark_kubernetes.py b/providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py similarity index 95% rename from tests/system/providers/cncf/kubernetes/example_spark_kubernetes.py rename to providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py index 4584d653e4fa1..072cffeb90f04 100644 --- a/tests/system/providers/cncf/kubernetes/example_spark_kubernetes.py +++ b/providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py @@ -76,13 +76,13 @@ t1 >> t2 # [END SparkKubernetesOperator_DAG] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml b/providers/tests/system/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml similarity index 100% rename from tests/system/providers/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml rename to providers/tests/system/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml diff --git a/tests/system/providers/cncf/kubernetes/spark_job_template.yaml b/providers/tests/system/cncf/kubernetes/spark_job_template.yaml similarity index 100% rename from tests/system/providers/cncf/kubernetes/spark_job_template.yaml rename to providers/tests/system/cncf/kubernetes/spark_job_template.yaml diff --git a/tests/providers/ydb/utils/__init__.py b/providers/tests/system/cohere/__init__.py similarity index 100% rename from tests/providers/ydb/utils/__init__.py rename to providers/tests/system/cohere/__init__.py diff --git a/tests/system/providers/cohere/example_cohere_embedding_operator.py b/providers/tests/system/cohere/example_cohere_embedding_operator.py similarity index 96% rename from tests/system/providers/cohere/example_cohere_embedding_operator.py rename to providers/tests/system/cohere/example_cohere_embedding_operator.py index ec97ee91e57cb..9686724f2cea9 100644 --- a/tests/system/providers/cohere/example_cohere_embedding_operator.py +++ b/providers/tests/system/cohere/example_cohere_embedding_operator.py @@ -34,7 +34,7 @@ # [END howto_operator_cohere_embedding] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/zendesk/__init__.py b/providers/tests/system/common/__init__.py similarity index 100% rename from tests/providers/zendesk/__init__.py rename to providers/tests/system/common/__init__.py diff --git a/tests/providers/zendesk/hooks/__init__.py b/providers/tests/system/common/io/__init__.py similarity index 100% rename from tests/providers/zendesk/hooks/__init__.py rename to providers/tests/system/common/io/__init__.py diff --git a/tests/system/providers/common/io/example_file_transfer_local_to_s3.py b/providers/tests/system/common/io/example_file_transfer_local_to_s3.py similarity index 94% rename from tests/system/providers/common/io/example_file_transfer_local_to_s3.py rename to providers/tests/system/common/io/example_file_transfer_local_to_s3.py index 13c495e620972..0b90a75f2e6e2 100644 --- a/tests/system/providers/common/io/example_file_transfer_local_to_s3.py +++ b/providers/tests/system/common/io/example_file_transfer_local_to_s3.py @@ -76,12 +76,12 @@ def remove_bucket(): temp_file >> transfer >> remove_bucket() >> delete_temp_file(temp_file_path) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/airbyte/__init__.py b/providers/tests/system/common/sql/__init__.py similarity index 100% rename from tests/system/providers/airbyte/__init__.py rename to providers/tests/system/common/sql/__init__.py diff --git a/tests/system/providers/common/sql/example_sql_column_table_check.py b/providers/tests/system/common/sql/example_sql_column_table_check.py similarity index 97% rename from tests/system/providers/common/sql/example_sql_column_table_check.py rename to providers/tests/system/common/sql/example_sql_column_table_check.py index 2790d092a2b47..c48dc60bfc639 100644 --- a/tests/system/providers/common/sql/example_sql_column_table_check.py +++ b/providers/tests/system/common/sql/example_sql_column_table_check.py @@ -79,7 +79,7 @@ column_check >> row_count_check -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/common/sql/example_sql_execute_query.py b/providers/tests/system/common/sql/example_sql_execute_query.py similarity index 96% rename from tests/system/providers/common/sql/example_sql_execute_query.py rename to providers/tests/system/common/sql/example_sql_execute_query.py index 694ca3f2ef99c..8dced19cef719 100644 --- a/tests/system/providers/common/sql/example_sql_execute_query.py +++ b/providers/tests/system/common/sql/example_sql_execute_query.py @@ -56,7 +56,7 @@ # [END howto_operator_sql_execute_query] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/alibaba/__init__.py b/providers/tests/system/databricks/__init__.py similarity index 100% rename from tests/system/providers/alibaba/__init__.py rename to providers/tests/system/databricks/__init__.py diff --git a/tests/system/providers/databricks/example_databricks.py b/providers/tests/system/databricks/example_databricks.py similarity index 98% rename from tests/system/providers/databricks/example_databricks.py rename to providers/tests/system/databricks/example_databricks.py index 82c5d313421a9..00d9969bd99b3 100644 --- a/tests/system/providers/databricks/example_databricks.py +++ b/providers/tests/system/databricks/example_databricks.py @@ -238,13 +238,13 @@ ) # [END howto_operator_databricks_task_sql] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/databricks/example_databricks_repos.py b/providers/tests/system/databricks/example_databricks_repos.py similarity index 95% rename from tests/system/providers/databricks/example_databricks_repos.py rename to providers/tests/system/databricks/example_databricks_repos.py index dd583c67b9c90..d07226e76ed01 100644 --- a/tests/system/providers/databricks/example_databricks_repos.py +++ b/providers/tests/system/databricks/example_databricks_repos.py @@ -78,13 +78,13 @@ (create_repo >> update_repo >> notebook_task >> delete_repo) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/databricks/example_databricks_sensors.py b/providers/tests/system/databricks/example_databricks_sensors.py similarity index 96% rename from tests/system/providers/databricks/example_databricks_sensors.py rename to providers/tests/system/databricks/example_databricks_sensors.py index fd572a6bd9055..ea233a79395bc 100644 --- a/tests/system/providers/databricks/example_databricks_sensors.py +++ b/providers/tests/system/databricks/example_databricks_sensors.py @@ -88,7 +88,7 @@ # runs, else all the subsequent DAG tasks and the DAG are marked as failed. (sql_sensor >> partition_sensor) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This example does not need a watcher in order to properly mark success/failure # since it is a single task, but it is given here as an example for users to @@ -96,7 +96,7 @@ # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/databricks/example_databricks_sql.py b/providers/tests/system/databricks/example_databricks_sql.py similarity index 97% rename from tests/system/providers/databricks/example_databricks_sql.py rename to providers/tests/system/databricks/example_databricks_sql.py index 3f7ed0858c82e..f08ce3cd56b65 100644 --- a/tests/system/providers/databricks/example_databricks_sql.py +++ b/providers/tests/system/databricks/example_databricks_sql.py @@ -113,13 +113,13 @@ (create >> create_file >> import_csv >> select >> select_into_file) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/databricks/example_databricks_workflow.py b/providers/tests/system/databricks/example_databricks_workflow.py similarity index 97% rename from tests/system/providers/databricks/example_databricks_workflow.py rename to providers/tests/system/databricks/example_databricks_workflow.py index 1cfd81e9c7ec0..3a6c829520d89 100644 --- a/tests/system/providers/databricks/example_databricks_workflow.py +++ b/providers/tests/system/databricks/example_databricks_workflow.py @@ -143,13 +143,13 @@ notebook_1 >> notebook_2 >> task_operator_nb_1 >> sql_query # [END howto_databricks_workflow_notebook] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/amazon/aws/tests/__init__.py b/providers/tests/system/dbt/__init__.py similarity index 100% rename from tests/system/providers/amazon/aws/tests/__init__.py rename to providers/tests/system/dbt/__init__.py diff --git a/tests/system/providers/apache/cassandra/__init__.py b/providers/tests/system/dbt/cloud/__init__.py similarity index 100% rename from tests/system/providers/apache/cassandra/__init__.py rename to providers/tests/system/dbt/cloud/__init__.py diff --git a/tests/system/providers/dbt/cloud/example_dbt_cloud.py b/providers/tests/system/dbt/cloud/example_dbt_cloud.py similarity index 94% rename from tests/system/providers/dbt/cloud/example_dbt_cloud.py rename to providers/tests/system/dbt/cloud/example_dbt_cloud.py index 897abb293b5de..17f8a59efea02 100644 --- a/tests/system/providers/dbt/cloud/example_dbt_cloud.py +++ b/providers/tests/system/dbt/cloud/example_dbt_cloud.py @@ -27,7 +27,8 @@ ) from airflow.providers.dbt.cloud.sensors.dbt import DbtCloudJobRunSensor from airflow.utils.edgemodifier import Label -from tests.system.utils import get_test_env_id + +from dev.tests_common.test_utils.system_tests import get_test_env_id ENV_ID = get_test_env_id() DAG_ID = "example_dbt_cloud" @@ -91,13 +92,13 @@ # trigger_job_run2 >> job_run_sensor # trigger_job_run2 >> job_run_sensor_deferred - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/drill/__init__.py b/providers/tests/system/dingding/__init__.py similarity index 100% rename from tests/system/providers/apache/drill/__init__.py rename to providers/tests/system/dingding/__init__.py diff --git a/tests/system/providers/dingding/example_dingding.py b/providers/tests/system/dingding/example_dingding.py similarity index 98% rename from tests/system/providers/dingding/example_dingding.py rename to providers/tests/system/dingding/example_dingding.py index 240a732805d3e..4ecb41c68cc91 100644 --- a/tests/system/providers/dingding/example_dingding.py +++ b/providers/tests/system/dingding/example_dingding.py @@ -199,13 +199,13 @@ def failure_callback(context): >> msg_failure_callback ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/druid/__init__.py b/providers/tests/system/docker/__init__.py similarity index 100% rename from tests/system/providers/apache/druid/__init__.py rename to providers/tests/system/docker/__init__.py diff --git a/tests/system/providers/docker/example_docker.py b/providers/tests/system/docker/example_docker.py similarity index 96% rename from tests/system/providers/docker/example_docker.py rename to providers/tests/system/docker/example_docker.py index 18f7d2f0ea0c6..108813236ac62 100644 --- a/tests/system/providers/docker/example_docker.py +++ b/providers/tests/system/docker/example_docker.py @@ -57,7 +57,7 @@ t1 >> [t2, t3] >> t4 ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/docker/example_docker_copy_data.py b/providers/tests/system/docker/example_docker_copy_data.py similarity index 97% rename from tests/system/providers/docker/example_docker_copy_data.py rename to providers/tests/system/docker/example_docker_copy_data.py index 4e4e8466e501f..c270408390cfa 100644 --- a/tests/system/providers/docker/example_docker_copy_data.py +++ b/providers/tests/system/docker/example_docker_copy_data.py @@ -103,7 +103,7 @@ t_is_data_available >> t_move >> t_print ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/docker/example_docker_swarm.py b/providers/tests/system/docker/example_docker_swarm.py similarity index 95% rename from tests/system/providers/docker/example_docker_swarm.py rename to providers/tests/system/docker/example_docker_swarm.py index 6546c7d434aca..3a68dcea81673 100644 --- a/tests/system/providers/docker/example_docker_swarm.py +++ b/providers/tests/system/docker/example_docker_swarm.py @@ -47,7 +47,7 @@ t1 ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/docker/example_taskflow_api_docker_virtualenv.py b/providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py similarity index 97% rename from tests/system/providers/docker/example_taskflow_api_docker_virtualenv.py rename to providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py index 95c502a221073..3c8b528606e02 100644 --- a/tests/system/providers/docker/example_taskflow_api_docker_virtualenv.py +++ b/providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py @@ -116,7 +116,7 @@ def load(total_order_value: float): tutorial_dag = tutorial_taskflow_api_docker_virtualenv() # [END dag_invocation] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/kafka/__init__.py b/providers/tests/system/elasticsearch/__init__.py similarity index 100% rename from tests/system/providers/apache/kafka/__init__.py rename to providers/tests/system/elasticsearch/__init__.py diff --git a/tests/system/providers/elasticsearch/example_elasticsearch_query.py b/providers/tests/system/elasticsearch/example_elasticsearch_query.py similarity index 97% rename from tests/system/providers/elasticsearch/example_elasticsearch_query.py rename to providers/tests/system/elasticsearch/example_elasticsearch_query.py index 31e9440e6cbed..93471d874c3c9 100644 --- a/tests/system/providers/elasticsearch/example_elasticsearch_query.py +++ b/providers/tests/system/elasticsearch/example_elasticsearch_query.py @@ -81,7 +81,7 @@ def use_elasticsearch_hook(): task_id="print_data_from_elasticsearch", python_callable=use_elasticsearch_hook ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/kylin/__init__.py b/providers/tests/system/ftp/__init__.py similarity index 100% rename from tests/system/providers/apache/kylin/__init__.py rename to providers/tests/system/ftp/__init__.py diff --git a/tests/system/providers/ftp/example_ftp.py b/providers/tests/system/ftp/example_ftp.py similarity index 95% rename from tests/system/providers/ftp/example_ftp.py rename to providers/tests/system/ftp/example_ftp.py index bdc9399e46887..8fb2e38806ae4 100644 --- a/tests/system/providers/ftp/example_ftp.py +++ b/providers/tests/system/ftp/example_ftp.py @@ -87,13 +87,13 @@ ftp_put >> ftp_get ftps_put >> ftps_get - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/livy/__init__.py b/providers/tests/system/github/__init__.py similarity index 100% rename from tests/system/providers/apache/livy/__init__.py rename to providers/tests/system/github/__init__.py diff --git a/tests/system/providers/github/example_github.py b/providers/tests/system/github/example_github.py similarity index 97% rename from tests/system/providers/github/example_github.py rename to providers/tests/system/github/example_github.py index 81a458021aa67..70eb8bf902f04 100644 --- a/tests/system/providers/github/example_github.py +++ b/providers/tests/system/github/example_github.py @@ -100,7 +100,7 @@ def tag_checker(repo: Any, tag_name: str) -> bool | None: # [END howto_operator_list_tags_github] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/README.md b/providers/tests/system/google/README.md similarity index 100% rename from tests/system/providers/google/README.md rename to providers/tests/system/google/README.md diff --git a/tests/system/providers/google/__init__.py b/providers/tests/system/google/__init__.py similarity index 100% rename from tests/system/providers/google/__init__.py rename to providers/tests/system/google/__init__.py diff --git a/tests/system/providers/apache/pig/__init__.py b/providers/tests/system/google/ads/__init__.py similarity index 100% rename from tests/system/providers/apache/pig/__init__.py rename to providers/tests/system/google/ads/__init__.py diff --git a/tests/system/providers/google/ads/example_ads.py b/providers/tests/system/google/ads/example_ads.py similarity index 95% rename from tests/system/providers/google/ads/example_ads.py rename to providers/tests/system/google/ads/example_ads.py index 0f4e6d3a3f4a7..162fd45334f67 100644 --- a/tests/system/providers/google/ads/example_ads.py +++ b/providers/tests/system/google/ads/example_ads.py @@ -29,7 +29,8 @@ from airflow.providers.google.ads.transfers.ads_to_gcs import GoogleAdsToGcsOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [START howto_google_ads_env_variables] ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -116,14 +117,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/apache/pinot/__init__.py b/providers/tests/system/google/cloud/__init__.py similarity index 100% rename from tests/system/providers/apache/pinot/__init__.py rename to providers/tests/system/google/cloud/__init__.py diff --git a/tests/system/providers/apache/spark/__init__.py b/providers/tests/system/google/cloud/automl/__init__.py similarity index 100% rename from tests/system/providers/apache/spark/__init__.py rename to providers/tests/system/google/cloud/automl/__init__.py diff --git a/tests/system/providers/google/cloud/automl/example_automl_dataset.py b/providers/tests/system/google/cloud/automl/example_automl_dataset.py similarity index 97% rename from tests/system/providers/google/cloud/automl/example_automl_dataset.py rename to providers/tests/system/google/cloud/automl/example_automl_dataset.py index f90caf32df0c4..49de111ffb358 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_dataset.py +++ b/providers/tests/system/google/cloud/automl/example_automl_dataset.py @@ -162,14 +162,14 @@ def upload_updated_csv_file_to_gcs(): >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/automl/example_automl_translation.py b/providers/tests/system/google/cloud/automl/example_automl_translation.py similarity index 97% rename from tests/system/providers/google/cloud/automl/example_automl_translation.py rename to providers/tests/system/google/cloud/automl/example_automl_translation.py index cda70693fb57c..60dbf782f281e 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_translation.py +++ b/providers/tests/system/google/cloud/automl/example_automl_translation.py @@ -188,14 +188,14 @@ def upload_csv_file_to_gcs(): >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/automl/example_automl_video_classification.py b/providers/tests/system/google/cloud/automl/example_automl_video_classification.py similarity index 97% rename from tests/system/providers/google/cloud/automl/example_automl_video_classification.py rename to providers/tests/system/google/cloud/automl/example_automl_video_classification.py index 21e4c4623768d..36853c3826037 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_video_classification.py +++ b/providers/tests/system/google/cloud/automl/example_automl_video_classification.py @@ -158,13 +158,13 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/automl/example_automl_video_tracking.py b/providers/tests/system/google/cloud/automl/example_automl_video_tracking.py similarity index 97% rename from tests/system/providers/google/cloud/automl/example_automl_video_tracking.py rename to providers/tests/system/google/cloud/automl/example_automl_video_tracking.py index 36133b3674a89..65718f10b7697 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_video_tracking.py +++ b/providers/tests/system/google/cloud/automl/example_automl_video_tracking.py @@ -158,13 +158,13 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/automl/example_automl_vision_classification.py b/providers/tests/system/google/cloud/automl/example_automl_vision_classification.py similarity index 97% rename from tests/system/providers/google/cloud/automl/example_automl_vision_classification.py rename to providers/tests/system/google/cloud/automl/example_automl_vision_classification.py index 2e21f252d8185..40e68d3291ce8 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_vision_classification.py +++ b/providers/tests/system/google/cloud/automl/example_automl_vision_classification.py @@ -132,13 +132,13 @@ >> delete_image_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py b/providers/tests/system/google/cloud/automl/example_automl_vision_object_detection.py similarity index 97% rename from tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py rename to providers/tests/system/google/cloud/automl/example_automl_vision_object_detection.py index 2f92e81d9aaf9..334e154cea4d1 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py +++ b/providers/tests/system/google/cloud/automl/example_automl_vision_object_detection.py @@ -134,13 +134,13 @@ >> delete_image_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/asana/__init__.py b/providers/tests/system/google/cloud/automl/resources/__init__.py similarity index 100% rename from tests/system/providers/asana/__init__.py rename to providers/tests/system/google/cloud/automl/resources/__init__.py diff --git a/tests/system/providers/cncf/__init__.py b/providers/tests/system/google/cloud/azure/__init__.py similarity index 100% rename from tests/system/providers/cncf/__init__.py rename to providers/tests/system/google/cloud/azure/__init__.py diff --git a/tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py b/providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py similarity index 94% rename from tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py rename to providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py index 265b9e0a4ab86..837cb99e5ffc1 100644 --- a/tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py +++ b/providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py @@ -59,13 +59,13 @@ (wait_for_blob >> transfer_files_to_gcs) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/azure/example_azure_fileshare_to_gcs.py b/providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py similarity index 93% rename from tests/system/providers/google/cloud/azure/example_azure_fileshare_to_gcs.py rename to providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py index 80eaadd6c6d6d..a897b6dc7d3d1 100644 --- a/tests/system/providers/google/cloud/azure/example_azure_fileshare_to_gcs.py +++ b/providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py @@ -23,7 +23,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs import AzureFileShareToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -80,14 +81,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cncf/kubernetes/__init__.py b/providers/tests/system/google/cloud/bigquery/__init__.py similarity index 100% rename from tests/system/providers/cncf/kubernetes/__init__.py rename to providers/tests/system/google/cloud/bigquery/__init__.py diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py similarity index 95% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py index 004f996975be1..87e13a2c25165 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py @@ -85,14 +85,14 @@ >> delete_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py similarity index 97% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py index 3f75d43935131..b387563069a94 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py @@ -44,7 +44,8 @@ from airflow.providers.google.cloud.sensors.bigquery_dts import BigQueryDataTransferServiceTransferRunSensor from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -179,14 +180,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py similarity index 96% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py index a394c882beded..5e487f5a93e2a 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py @@ -97,14 +97,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_operations_location.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations_location.py similarity index 95% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_operations_location.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_operations_location.py index ac5b4dd967b30..2b026070a06b4 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_operations_location.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations_location.py @@ -77,14 +77,14 @@ >> delete_dataset_with_location ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py similarity index 97% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py index ab7a4b3757b9b..cd31ea05bcdcb 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py @@ -39,7 +39,8 @@ ) from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -246,7 +247,7 @@ execute_insert_query >> [check_count, check_value, check_interval] >> delete_dataset execute_insert_query >> [column_check, table_check] >> delete_dataset - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG @@ -256,7 +257,7 @@ globals()[DAG_ID] = dag for dag in DAGS_LIST: - from tests.system.utils import get_test_run + from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py similarity index 98% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py index a007e1cd639c0..012eba080d1a4 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py @@ -259,14 +259,14 @@ insert_query_job >> execute_long_running_query >> check_value >> check_interval [check_count, check_interval, bigquery_execute_multi_query, get_data_result] >> delete_dataset - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py similarity index 97% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py index 57cde5e9aa824..aa8d36f5ed893 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py @@ -165,14 +165,14 @@ >> delete_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py similarity index 97% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py index 3539da742a2ce..9c1432a0a8da5 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_tables.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py @@ -41,7 +41,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "bigquery_tables" @@ -223,14 +224,14 @@ >> delete_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_bigquery.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py similarity index 96% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_to_bigquery.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py index 67ad5da80256f..cb1fa63498da9 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_bigquery.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py @@ -100,14 +100,14 @@ >> delete_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_gcs.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py similarity index 94% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_to_gcs.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py index 8edeb1fb9f9d0..21acb27f161db 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_gcs.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py @@ -33,7 +33,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.bigquery_to_gcs import BigQueryToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "bigquery_to_gcs" @@ -97,14 +98,14 @@ >> [delete_bucket, delete_dataset] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_gcs_async.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs_async.py similarity index 94% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_to_gcs_async.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs_async.py index c2ba4bf287d1d..fdf084601c770 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_gcs_async.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs_async.py @@ -33,7 +33,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.bigquery_to_gcs import BigQueryToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "bigquery_to_gcs_async" @@ -96,14 +97,14 @@ >> [delete_bucket, delete_dataset] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py similarity index 98% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py index e9b3269ecfb6c..51e713560bc23 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py @@ -326,13 +326,13 @@ def delete_connection(connection_id: str) -> None: >> delete_persistent_disk ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mysql.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py similarity index 95% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_to_mysql.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py index cacd94a42a6b5..a3b726098773d 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mysql.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py @@ -88,13 +88,13 @@ >> delete_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py similarity index 98% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py index 4a3b0386da0f4..ca4e3148d0a18 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py @@ -362,13 +362,13 @@ def delete_connection(connection_id: str) -> None: >> delete_persistent_disk ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_transfer.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_transfer.py similarity index 95% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_transfer.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_transfer.py index e88858ef8c0b3..73d55710c9db9 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_transfer.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_transfer.py @@ -34,7 +34,8 @@ from airflow.providers.google.cloud.transfers.bigquery_to_bigquery import BigQueryToBigQueryOperator from airflow.providers.google.cloud.transfers.bigquery_to_gcs import BigQueryToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -116,13 +117,13 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_value_check.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_value_check.py similarity index 95% rename from tests/system/providers/google/cloud/bigquery/example_bigquery_value_check.py rename to providers/tests/system/google/cloud/bigquery/example_bigquery_value_check.py index 4a14ac6a810b3..3872ca092321f 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_value_check.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_value_check.py @@ -131,8 +131,8 @@ >> delete_dataset ) - from tests.system.utils import get_test_run - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.system_tests import get_test_run + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG @@ -142,7 +142,7 @@ test_run = get_test_run(dag) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/cohere/__init__.py b/providers/tests/system/google/cloud/bigquery/resources/__init__.py similarity index 100% rename from tests/system/providers/cohere/__init__.py rename to providers/tests/system/google/cloud/bigquery/resources/__init__.py diff --git a/tests/system/providers/google/cloud/bigquery/resources/example_bigquery_query.sql b/providers/tests/system/google/cloud/bigquery/resources/example_bigquery_query.sql similarity index 100% rename from tests/system/providers/google/cloud/bigquery/resources/example_bigquery_query.sql rename to providers/tests/system/google/cloud/bigquery/resources/example_bigquery_query.sql diff --git a/tests/system/providers/google/cloud/bigquery/resources/update_table_schema.json b/providers/tests/system/google/cloud/bigquery/resources/update_table_schema.json similarity index 100% rename from tests/system/providers/google/cloud/bigquery/resources/update_table_schema.json rename to providers/tests/system/google/cloud/bigquery/resources/update_table_schema.json diff --git a/tests/system/providers/google/cloud/bigquery/resources/us-states.csv b/providers/tests/system/google/cloud/bigquery/resources/us-states.csv similarity index 100% rename from tests/system/providers/google/cloud/bigquery/resources/us-states.csv rename to providers/tests/system/google/cloud/bigquery/resources/us-states.csv diff --git a/tests/system/providers/common/__init__.py b/providers/tests/system/google/cloud/bigtable/__init__.py similarity index 100% rename from tests/system/providers/common/__init__.py rename to providers/tests/system/google/cloud/bigtable/__init__.py diff --git a/tests/system/providers/google/cloud/bigtable/example_bigtable.py b/providers/tests/system/google/cloud/bigtable/example_bigtable.py similarity index 97% rename from tests/system/providers/google/cloud/bigtable/example_bigtable.py rename to providers/tests/system/google/cloud/bigtable/example_bigtable.py index 1c690f2ddc721..77abc49fae258 100644 --- a/tests/system/providers/google/cloud/bigtable/example_bigtable.py +++ b/providers/tests/system/google/cloud/bigtable/example_bigtable.py @@ -61,7 +61,8 @@ ) from airflow.providers.google.cloud.sensors.bigtable import BigtableTableReplicationCompletedSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -224,14 +225,14 @@ def update_clusters_and_instance(): >> [delete_instance_task, delete_instance_task2] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/common/io/__init__.py b/providers/tests/system/google/cloud/cloud_batch/__init__.py similarity index 100% rename from tests/system/providers/common/io/__init__.py rename to providers/tests/system/google/cloud/cloud_batch/__init__.py diff --git a/tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py b/providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py similarity index 96% rename from tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py rename to providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py index 58d852a286bc8..84dc031a5268f 100644 --- a/tests/system/providers/google/cloud/cloud_batch/example_cloud_batch.py +++ b/providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py @@ -35,7 +35,8 @@ CloudBatchSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -186,13 +187,13 @@ def _create_job(): ([submit1, submit2] >> list_tasks >> assert_tasks >> list_jobs >> get_name >> [delete_job1, delete_job2]) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/common/sql/__init__.py b/providers/tests/system/google/cloud/cloud_build/__init__.py similarity index 100% rename from tests/system/providers/common/sql/__init__.py rename to providers/tests/system/google/cloud/cloud_build/__init__.py diff --git a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py b/providers/tests/system/google/cloud/cloud_build/example_cloud_build.py similarity index 97% rename from tests/system/providers/google/cloud/cloud_build/example_cloud_build.py rename to providers/tests/system/google/cloud/cloud_build/example_cloud_build.py index cb31a3b4d091d..bf2391413a1c3 100644 --- a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +++ b/providers/tests/system/google/cloud/cloud_build/example_cloud_build.py @@ -39,7 +39,8 @@ CloudBuildRetryBuildOperator, ) from airflow.providers.standard.operators.bash import BashOperator -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -51,7 +52,7 @@ # If you'd like to run this system test locally, please # 1. Create Cloud Source Repository # 2. Push into a master branch the following file: -# tests/system/providers/google/cloud/cloud_build/resources/example_cloud_build.yaml +# providers/tests/system/google/cloud/cloud_build/resources/example_cloud_build.yaml GCP_SOURCE_REPOSITORY_NAME = "test-cloud-build-repository" CURRENT_FOLDER = Path(__file__).parent @@ -266,14 +267,14 @@ def no_wait_cancel_retry_get_deferrable(): ] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py b/providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py similarity index 96% rename from tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py rename to providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py index 6c2c64ba6283a..f6873da9c5680 100644 --- a/tests/system/providers/google/cloud/cloud_build/example_cloud_build_trigger.py +++ b/providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py @@ -41,7 +41,8 @@ CloudBuildUpdateBuildTriggerOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -52,7 +53,7 @@ # If you'd like to run this system test locally, please # 1. Create Cloud Source Repository # 2. Push into a master branch the following file: -# tests/system/providers/google/cloud/cloud_build/resources/example_cloud_build.yaml +# providers/tests/system/google/cloud/cloud_build/resources/example_cloud_build.yaml GCP_SOURCE_REPOSITORY_NAME = "test-cloud-build-repository" TRIGGER_NAME = f"cloud-build-trigger-{ENV_ID}".replace("_", "-") @@ -181,14 +182,14 @@ def get_project_number(): >> list_build_triggers ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/databricks/__init__.py b/providers/tests/system/google/cloud/cloud_build/resources/__init__.py similarity index 100% rename from tests/system/providers/databricks/__init__.py rename to providers/tests/system/google/cloud/cloud_build/resources/__init__.py diff --git a/tests/system/providers/google/cloud/cloud_build/resources/example_cloud_build.yaml b/providers/tests/system/google/cloud/cloud_build/resources/example_cloud_build.yaml similarity index 100% rename from tests/system/providers/google/cloud/cloud_build/resources/example_cloud_build.yaml rename to providers/tests/system/google/cloud/cloud_build/resources/example_cloud_build.yaml diff --git a/tests/system/providers/dbt/cloud/__init__.py b/providers/tests/system/google/cloud/cloud_functions/__init__.py similarity index 100% rename from tests/system/providers/dbt/cloud/__init__.py rename to providers/tests/system/google/cloud/cloud_functions/__init__.py diff --git a/tests/system/providers/google/cloud/cloud_functions/example_functions.py b/providers/tests/system/google/cloud/cloud_functions/example_functions.py similarity index 95% rename from tests/system/providers/google/cloud/cloud_functions/example_functions.py rename to providers/tests/system/google/cloud/cloud_functions/example_functions.py index 84e31bc745178..fd0b8ea5f5a4c 100644 --- a/tests/system/providers/google/cloud/cloud_functions/example_functions.py +++ b/providers/tests/system/google/cloud/cloud_functions/example_functions.py @@ -34,7 +34,8 @@ CloudFunctionDeployFunctionOperator, CloudFunctionInvokeFunctionOperator, ) -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -121,14 +122,14 @@ delete_function, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/dingding/__init__.py b/providers/tests/system/google/cloud/cloud_memorystore/__init__.py similarity index 100% rename from tests/system/providers/dingding/__init__.py rename to providers/tests/system/google/cloud/cloud_memorystore/__init__.py diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py similarity index 98% rename from tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py rename to providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py index 4884122751e09..670a850b04ac4 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +++ b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py @@ -198,14 +198,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py similarity index 97% rename from tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py rename to providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py index c46d966371dac..3d8ebf6f287dc 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +++ b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py @@ -47,7 +47,8 @@ ) from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") ENV_ID_LOWER = ENV_ID.lower() if ENV_ID else "" @@ -265,14 +266,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/docker/__init__.py b/providers/tests/system/google/cloud/cloud_run/__init__.py similarity index 100% rename from tests/system/providers/docker/__init__.py rename to providers/tests/system/google/cloud/cloud_run/__init__.py diff --git a/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py b/providers/tests/system/google/cloud/cloud_run/example_cloud_run.py similarity index 98% rename from tests/system/providers/google/cloud/cloud_run/example_cloud_run.py rename to providers/tests/system/google/cloud/cloud_run/example_cloud_run.py index 23370d26876b6..08145e0336ede 100644 --- a/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +++ b/providers/tests/system/google/cloud/cloud_run/example_cloud_run.py @@ -367,13 +367,13 @@ def _create_job_instance_with_label(): >> (delete_job1, delete_job2, delete_job3) ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_run/example_cloud_run_service.py b/providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py similarity index 95% rename from tests/system/providers/google/cloud/cloud_run/example_cloud_run_service.py rename to providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py index 0c838e6d4e28e..43318c4b14306 100644 --- a/tests/system/providers/google/cloud/cloud_run/example_cloud_run_service.py +++ b/providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py @@ -84,14 +84,14 @@ def _create_service(): >> delete_cloud_run_service ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/elasticsearch/__init__.py b/providers/tests/system/google/cloud/cloud_sql/__init__.py similarity index 100% rename from tests/system/providers/elasticsearch/__init__.py rename to providers/tests/system/google/cloud/cloud_sql/__init__.py diff --git a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py similarity index 98% rename from tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py rename to providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py index 7a60e3db61718..52414b2784b27 100644 --- a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +++ b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py @@ -47,7 +47,8 @@ GCSObjectCreateAclEntryOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -293,14 +294,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query.py b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py similarity index 98% rename from tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query.py rename to providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py index f6d588c3156c8..6cf7e0c08751c 100644 --- a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query.py +++ b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py @@ -43,7 +43,8 @@ ) from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -540,13 +541,13 @@ def delete_connection(connection_id: str) -> None: # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py similarity index 98% rename from tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py rename to providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py index 4808141512ec5..db77f18339af0 100644 --- a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py +++ b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py @@ -48,7 +48,8 @@ ) from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -490,13 +491,13 @@ def delete_secret(ssl_secret_id, db_type: str) -> None: # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/ftp/__init__.py b/providers/tests/system/google/cloud/composer/__init__.py similarity index 100% rename from tests/system/providers/ftp/__init__.py rename to providers/tests/system/google/cloud/composer/__init__.py diff --git a/tests/system/providers/google/cloud/composer/example_cloud_composer.py b/providers/tests/system/google/cloud/composer/example_cloud_composer.py similarity index 98% rename from tests/system/providers/google/cloud/composer/example_cloud_composer.py rename to providers/tests/system/google/cloud/composer/example_cloud_composer.py index 75ee8e90d3aa5..266a7e4a444ad 100644 --- a/tests/system/providers/google/cloud/composer/example_cloud_composer.py +++ b/providers/tests/system/google/cloud/composer/example_cloud_composer.py @@ -214,14 +214,14 @@ [delete_env, defer_delete_env], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/github/__init__.py b/providers/tests/system/google/cloud/compute/__init__.py similarity index 100% rename from tests/system/providers/github/__init__.py rename to providers/tests/system/google/cloud/compute/__init__.py diff --git a/tests/system/providers/google/cloud/compute/example_compute.py b/providers/tests/system/google/cloud/compute/example_compute.py similarity index 97% rename from tests/system/providers/google/cloud/compute/example_compute.py rename to providers/tests/system/google/cloud/compute/example_compute.py index 27e8ae758e70b..8343920890708 100644 --- a/tests/system/providers/google/cloud/compute/example_compute.py +++ b/providers/tests/system/google/cloud/compute/example_compute.py @@ -39,7 +39,8 @@ ComputeEngineStopInstanceOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [START howto_operator_gce_args_common] ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -266,14 +267,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/compute/example_compute_igm.py b/providers/tests/system/google/cloud/compute/example_compute_igm.py similarity index 97% rename from tests/system/providers/google/cloud/compute/example_compute_igm.py rename to providers/tests/system/google/cloud/compute/example_compute_igm.py index 74072b2099446..11357452dfc69 100644 --- a/tests/system/providers/google/cloud/compute/example_compute_igm.py +++ b/providers/tests/system/google/cloud/compute/example_compute_igm.py @@ -38,7 +38,8 @@ ComputeEngineInstanceGroupUpdateManagerTemplateOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -235,14 +236,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/compute/example_compute_ssh.py b/providers/tests/system/google/cloud/compute/example_compute_ssh.py similarity index 96% rename from tests/system/providers/google/cloud/compute/example_compute_ssh.py rename to providers/tests/system/google/cloud/compute/example_compute_ssh.py index e0bece195762f..5b7d1523017ff 100644 --- a/tests/system/providers/google/cloud/compute/example_compute_ssh.py +++ b/providers/tests/system/google/cloud/compute/example_compute_ssh.py @@ -34,7 +34,8 @@ ) from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [START howto_operator_gce_args_common] ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -137,14 +138,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/compute/example_compute_ssh_os_login.py b/providers/tests/system/google/cloud/compute/example_compute_ssh_os_login.py similarity index 96% rename from tests/system/providers/google/cloud/compute/example_compute_ssh_os_login.py rename to providers/tests/system/google/cloud/compute/example_compute_ssh_os_login.py index 99e04fa3d3eb3..63ed278a08c5a 100644 --- a/tests/system/providers/google/cloud/compute/example_compute_ssh_os_login.py +++ b/providers/tests/system/google/cloud/compute/example_compute_ssh_os_login.py @@ -34,7 +34,8 @@ ) from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [START howto_operator_gce_args_common] ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -145,14 +146,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/compute/example_compute_ssh_parallel.py b/providers/tests/system/google/cloud/compute/example_compute_ssh_parallel.py similarity index 96% rename from tests/system/providers/google/cloud/compute/example_compute_ssh_parallel.py rename to providers/tests/system/google/cloud/compute/example_compute_ssh_parallel.py index b5964eed7dd5e..3ab61afc0c259 100644 --- a/tests/system/providers/google/cloud/compute/example_compute_ssh_parallel.py +++ b/providers/tests/system/google/cloud/compute/example_compute_ssh_parallel.py @@ -34,7 +34,8 @@ ) from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [START howto_operator_gce_args_common] ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -138,14 +139,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/ads/__init__.py b/providers/tests/system/google/cloud/data_loss_prevention/__init__.py similarity index 100% rename from tests/system/providers/google/ads/__init__.py rename to providers/tests/system/google/cloud/data_loss_prevention/__init__.py diff --git a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py similarity index 96% rename from tests/system/providers/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py rename to providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py index 7acae202dea86..75b1ad93dd479 100644 --- a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py @@ -38,7 +38,8 @@ CloudDLPUpdateDeidentifyTemplateOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dlp_deidentify_content" ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -156,14 +157,14 @@ >> delete_template ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_info_types.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py similarity index 96% rename from tests/system/providers/google/cloud/data_loss_prevention/example_dlp_info_types.py rename to providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py index 40e5c030d981b..fb2198b16793f 100644 --- a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_info_types.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py @@ -41,7 +41,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dlp_info_types" ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -153,14 +154,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_inspect_template.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py similarity index 95% rename from tests/system/providers/google/cloud/data_loss_prevention/example_dlp_inspect_template.py rename to providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py index fb9ca15e2d722..22b259582f4ac 100644 --- a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_inspect_template.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py @@ -37,7 +37,8 @@ CloudDLPUpdateInspectTemplateOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dlp_inspect_template" ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -114,14 +115,14 @@ >> delete_template ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job.py similarity index 94% rename from tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job.py rename to providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job.py index 2bea0ce57bcea..b7dc2ac1a4053 100644 --- a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job.py @@ -36,7 +36,8 @@ CloudDLPListDLPJobsOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dlp_job" ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -89,14 +90,14 @@ (create_job >> list_jobs >> get_job >> cancel_job >> delete_job) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job_trigger.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py similarity index 94% rename from tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job_trigger.py rename to providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py index c2a1908e87f77..902875f7b5921 100644 --- a/tests/system/providers/google/cloud/data_loss_prevention/example_dlp_job_trigger.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py @@ -34,7 +34,8 @@ CloudDLPUpdateJobTriggerOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dlp_job_trigger" ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -94,14 +95,14 @@ (create_trigger >> list_triggers >> get_trigger >> update_trigger >> delete_trigger) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/__init__.py b/providers/tests/system/google/cloud/data_loss_prevention/resources/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/__init__.py rename to providers/tests/system/google/cloud/data_loss_prevention/resources/__init__.py diff --git a/tests/system/providers/google/cloud/data_loss_prevention/resources/dictionary.txt b/providers/tests/system/google/cloud/data_loss_prevention/resources/dictionary.txt similarity index 100% rename from tests/system/providers/google/cloud/data_loss_prevention/resources/dictionary.txt rename to providers/tests/system/google/cloud/data_loss_prevention/resources/dictionary.txt diff --git a/tests/system/providers/google/cloud/automl/__init__.py b/providers/tests/system/google/cloud/dataflow/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/automl/__init__.py rename to providers/tests/system/google/cloud/dataflow/__init__.py diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_go.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_go.py similarity index 98% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_go.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_go.py index 14c23a015d2de..57e5941e3ed6b 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_go.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_go.py @@ -149,7 +149,7 @@ def check_autoscaling_event(autoscaling_events: list[dict]) -> bool: ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_native_java.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py similarity index 97% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_native_java.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py index 34dad6cdeb293..3629194e6ee9b 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_native_java.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py @@ -148,14 +148,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_native_python.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py similarity index 96% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_native_python.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py index 181c5cbbe0106..229373ed1bf3f 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_native_python.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py @@ -110,14 +110,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py similarity index 98% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py index f93378c6db0b0..31f1cd026f19a 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_native_python_async.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py @@ -176,14 +176,14 @@ def check_autoscaling_event(autoscaling_events: list[dict]) -> bool: >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_pipeline.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py similarity index 97% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_pipeline.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py index 8f41b3ed7fb2a..cfd5e06b029b6 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_pipeline.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py @@ -137,14 +137,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_sensors_deferrable.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py similarity index 98% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_sensors_deferrable.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py index 4225b6c9ff80e..1a6d0aec3923b 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_sensors_deferrable.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py @@ -177,14 +177,14 @@ def check_autoscaling_event(autoscaling_events: list[dict]) -> bool: >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_sql.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py similarity index 95% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_sql.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py index 5b016fec3ec32..e3b0ee711a921 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_sql.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py @@ -34,7 +34,8 @@ ) from airflow.providers.google.cloud.operators.dataflow import DataflowStartSqlJobOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -136,13 +137,13 @@ >> delete_bq_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_streaming_python.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py similarity index 96% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_streaming_python.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py index a858ad90aa939..a15dffb3a3282 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_streaming_python.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py @@ -114,14 +114,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_template.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_template.py similarity index 96% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_template.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_template.py index 79ddfbef438de..86545514607aa 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_template.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_template.py @@ -37,7 +37,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -157,14 +158,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_yaml.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py similarity index 96% rename from tests/system/providers/google/cloud/dataflow/example_dataflow_yaml.py rename to providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py index d5162e8adf1b3..2243ad695bdd4 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_yaml.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py @@ -39,7 +39,8 @@ ) from airflow.providers.google.cloud.operators.dataflow import DataflowStartYamlJobOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -161,13 +162,13 @@ >> delete_bq_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/automl/resources/__init__.py b/providers/tests/system/google/cloud/dataflow/resources/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/automl/resources/__init__.py rename to providers/tests/system/google/cloud/dataflow/resources/__init__.py diff --git a/tests/system/providers/google/cloud/dataflow/resources/input.csv b/providers/tests/system/google/cloud/dataflow/resources/input.csv similarity index 100% rename from tests/system/providers/google/cloud/dataflow/resources/input.csv rename to providers/tests/system/google/cloud/dataflow/resources/input.csv diff --git a/tests/system/providers/google/cloud/dataflow/resources/schema.json b/providers/tests/system/google/cloud/dataflow/resources/schema.json similarity index 100% rename from tests/system/providers/google/cloud/dataflow/resources/schema.json rename to providers/tests/system/google/cloud/dataflow/resources/schema.json diff --git a/tests/system/providers/google/cloud/dataflow/resources/text.txt b/providers/tests/system/google/cloud/dataflow/resources/text.txt similarity index 100% rename from tests/system/providers/google/cloud/dataflow/resources/text.txt rename to providers/tests/system/google/cloud/dataflow/resources/text.txt diff --git a/tests/system/providers/google/cloud/dataflow/resources/wordcount.go b/providers/tests/system/google/cloud/dataflow/resources/wordcount.go similarity index 100% rename from tests/system/providers/google/cloud/dataflow/resources/wordcount.go rename to providers/tests/system/google/cloud/dataflow/resources/wordcount.go diff --git a/tests/system/providers/google/cloud/azure/__init__.py b/providers/tests/system/google/cloud/dataform/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/azure/__init__.py rename to providers/tests/system/google/cloud/dataform/__init__.py diff --git a/tests/system/providers/google/cloud/dataform/example_dataform.py b/providers/tests/system/google/cloud/dataform/example_dataform.py similarity index 98% rename from tests/system/providers/google/cloud/dataform/example_dataform.py rename to providers/tests/system/google/cloud/dataform/example_dataform.py index 3f2ad987f233a..b61247a877008 100644 --- a/tests/system/providers/google/cloud/dataform/example_dataform.py +++ b/providers/tests/system/google/cloud/dataform/example_dataform.py @@ -48,7 +48,8 @@ from airflow.providers.google.cloud.sensors.dataform import DataformWorkflowInvocationStateSensor from airflow.providers.google.cloud.utils.dataform import make_initialization_workspace_flow from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -327,13 +328,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/__init__.py b/providers/tests/system/google/cloud/datafusion/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/bigquery/__init__.py rename to providers/tests/system/google/cloud/datafusion/__init__.py diff --git a/tests/system/providers/google/cloud/datafusion/example_datafusion.py b/providers/tests/system/google/cloud/datafusion/example_datafusion.py similarity index 98% rename from tests/system/providers/google/cloud/datafusion/example_datafusion.py rename to providers/tests/system/google/cloud/datafusion/example_datafusion.py index d206c2260ee24..a61fbbd01da91 100644 --- a/tests/system/providers/google/cloud/datafusion/example_datafusion.py +++ b/providers/tests/system/google/cloud/datafusion/example_datafusion.py @@ -41,7 +41,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.sensors.datafusion import CloudDataFusionPipelineStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [START howto_data_fusion_env_variables] SERVICE_ACCOUNT = os.environ.get("GCP_DATAFUSION_SERVICE_ACCOUNT") @@ -339,13 +340,13 @@ def get_artifacts_versions(ti=None): >> [delete_bucket1, delete_bucket2] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigquery/resources/__init__.py b/providers/tests/system/google/cloud/datapipelines/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/bigquery/resources/__init__.py rename to providers/tests/system/google/cloud/datapipelines/__init__.py diff --git a/tests/system/providers/google/cloud/datapipelines/example_datapipeline.py b/providers/tests/system/google/cloud/datapipelines/example_datapipeline.py similarity index 96% rename from tests/system/providers/google/cloud/datapipelines/example_datapipeline.py rename to providers/tests/system/google/cloud/datapipelines/example_datapipeline.py index f13a52a7c6734..91fb7eed26d2a 100644 --- a/tests/system/providers/google/cloud/datapipelines/example_datapipeline.py +++ b/providers/tests/system/google/cloud/datapipelines/example_datapipeline.py @@ -37,7 +37,8 @@ GCSSynchronizeBucketsOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "datapipeline" ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -137,14 +138,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/bigtable/__init__.py b/providers/tests/system/google/cloud/dataplex/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/bigtable/__init__.py rename to providers/tests/system/google/cloud/dataplex/__init__.py diff --git a/tests/system/providers/google/cloud/dataplex/example_dataplex.py b/providers/tests/system/google/cloud/dataplex/example_dataplex.py similarity index 97% rename from tests/system/providers/google/cloud/dataplex/example_dataplex.py rename to providers/tests/system/google/cloud/dataplex/example_dataplex.py index a0ac55e07fc95..34e9a0fd05c19 100644 --- a/tests/system/providers/google/cloud/dataplex/example_dataplex.py +++ b/providers/tests/system/google/cloud/dataplex/example_dataplex.py @@ -40,7 +40,8 @@ ) from airflow.providers.google.cloud.sensors.dataplex import DataplexTaskStateSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -205,14 +206,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py b/providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py similarity index 98% rename from tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py rename to providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py index 3203a0d0cd3e2..44f4af2435925 100644 --- a/tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py +++ b/providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py @@ -50,7 +50,8 @@ ) from airflow.providers.google.cloud.sensors.dataplex import DataplexDataProfileJobStatusSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -328,14 +329,14 @@ [delete_lake, delete_dataset], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py b/providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py similarity index 98% rename from tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py rename to providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py index 6255f5425f18f..d31d574755c90 100644 --- a/tests/system/providers/google/cloud/dataplex/example_dataplex_dq.py +++ b/providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py @@ -50,7 +50,8 @@ ) from airflow.providers.google.cloud.sensors.dataplex import DataplexDataQualityJobStatusSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -362,14 +363,14 @@ [delete_lake, delete_dataset], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_batch/__init__.py b/providers/tests/system/google/cloud/dataprep/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/cloud_batch/__init__.py rename to providers/tests/system/google/cloud/dataprep/__init__.py diff --git a/tests/system/providers/google/cloud/dataprep/example_dataprep.py b/providers/tests/system/google/cloud/dataprep/example_dataprep.py similarity index 98% rename from tests/system/providers/google/cloud/dataprep/example_dataprep.py rename to providers/tests/system/google/cloud/dataprep/example_dataprep.py index 3573285055192..9f603f43fb1b3 100644 --- a/tests/system/providers/google/cloud/dataprep/example_dataprep.py +++ b/providers/tests/system/google/cloud/dataprep/example_dataprep.py @@ -47,7 +47,8 @@ from airflow.providers.google.cloud.sensors.dataprep import DataprepJobGroupIsFinishedSensor from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataprep" @@ -306,13 +307,13 @@ def delete_connection(connection_id: str) -> None: [delete_bucket_task, delete_connection_task], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_build/__init__.py b/providers/tests/system/google/cloud/dataproc/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/cloud_build/__init__.py rename to providers/tests/system/google/cloud/dataproc/__init__.py diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py similarity index 96% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py index 6771a529d79eb..54481f43ca762 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py @@ -35,7 +35,8 @@ ) from airflow.providers.google.cloud.sensors.dataproc import DataprocBatchSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -175,13 +176,13 @@ >> delete_batch_4 ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py similarity index 93% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py index 557b8ce49060f..79d3766995e3d 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py @@ -34,7 +34,8 @@ DataprocGetBatchOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_batch_deferrable" @@ -90,14 +91,14 @@ >> delete_batch ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py similarity index 96% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py index fb592b0cdb6fe..a8ed6b13de552 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py @@ -35,7 +35,8 @@ ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_batch_ps" @@ -140,14 +141,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py index ef57500639dc5..0eb27762c2f98 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py @@ -34,7 +34,8 @@ DataprocStopClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_create_existing_stopped_cluster" @@ -120,13 +121,13 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py index 3ff91d95a95be..8ae262dbf624c 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py @@ -33,7 +33,8 @@ DataprocUpdateClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_cluster_def" @@ -136,14 +137,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py index 1d94f688996f8..3eacc905bc703 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py @@ -33,7 +33,8 @@ DataprocDiagnoseClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_diagnose_cluster" @@ -114,14 +115,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py index 19e2300e28ead..f990363c48004 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py @@ -39,7 +39,8 @@ GCSSynchronizeBucketsOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_cluster_generation" @@ -134,14 +135,14 @@ >> [delete_cluster, delete_bucket] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py index 7759ddd098f0e..2e4b698573bf2 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py @@ -34,7 +34,8 @@ DataprocStopClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_cluster_start_stop" @@ -110,13 +111,13 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py index ea7725209f808..7ed0061d9947c 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py @@ -33,7 +33,8 @@ DataprocUpdateClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_update" @@ -119,14 +120,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_flink.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_flink.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_flink.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_flink.py index 2eb6d4c4bdf4f..ce1a6fc4451e9 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_flink.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_flink.py @@ -34,7 +34,8 @@ ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_flink" @@ -123,14 +124,14 @@ >> [delete_cluster, delete_bucket] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py similarity index 96% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py index becf273784ab1..bb1044da62414 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py @@ -43,7 +43,8 @@ GKEDeleteClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_gke" @@ -143,13 +144,13 @@ >> delete_gke_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py index 23ddd6eb581b7..c1f5423041fa4 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py @@ -34,7 +34,8 @@ ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_hadoop" @@ -122,14 +123,14 @@ >> [delete_cluster, delete_bucket] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py index 193d790a5282f..6aadceb552f61 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py @@ -33,7 +33,8 @@ DataprocSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_hive" @@ -123,14 +124,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_pig.py similarity index 94% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_pig.py index 344adaeca2e66..762ca2bc73fd2 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_pig.py @@ -33,7 +33,8 @@ DataprocSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_pig" ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") @@ -107,14 +108,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py similarity index 94% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py index 224dfc3db5b1f..f5bdf25732a5e 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py @@ -33,7 +33,8 @@ DataprocSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_presto" @@ -114,14 +115,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py index 44809b283767d..71e7245b7d728 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py @@ -38,7 +38,8 @@ GCSSynchronizeBucketsOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_pyspark" @@ -139,14 +140,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py similarity index 94% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py index ca76810cb43e6..e43a56d230bf4 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py @@ -33,7 +33,8 @@ DataprocSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark" @@ -110,14 +111,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py index 2ba8cc512f73c..475e9912e8562 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py @@ -34,7 +34,8 @@ ) from airflow.providers.google.cloud.sensors.dataproc import DataprocJobSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark_async" @@ -120,14 +121,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py similarity index 94% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py index cee74bb819b9d..5e6d0b773af6f 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py @@ -34,7 +34,8 @@ DataprocSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark_deferrable" @@ -111,14 +112,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py similarity index 94% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py index dc446ad332e12..aef860a0e545a 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py @@ -33,7 +33,8 @@ DataprocSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark_sql" @@ -107,14 +108,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py index 1468edf05eceb..7b416ade38648 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py @@ -38,7 +38,8 @@ GCSSynchronizeBucketsOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_sparkr" @@ -135,14 +136,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py similarity index 94% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py index 4631733cd0b4b..d3f7f2a1a3a4c 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py @@ -33,7 +33,8 @@ DataprocSubmitJobOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_trino" @@ -116,14 +117,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_workflow.py index 334826b56523a..ab465a124d5fd 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow.py @@ -29,7 +29,8 @@ DataprocInstantiateInlineWorkflowTemplateOperator, DataprocInstantiateWorkflowTemplateOperator, ) -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_workflow" @@ -103,14 +104,14 @@ >> instantiate_inline_workflow_template ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py rename to providers/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py index ec765250a3326..e2319a35a124f 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py @@ -29,7 +29,8 @@ DataprocInstantiateInlineWorkflowTemplateOperator, DataprocInstantiateWorkflowTemplateOperator, ) -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_workflow_def" @@ -107,14 +108,14 @@ >> instantiate_inline_workflow_template_async ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_build/resources/__init__.py b/providers/tests/system/google/cloud/dataproc_metastore/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/cloud_build/resources/__init__.py rename to providers/tests/system/google/cloud/dataproc_metastore/__init__.py diff --git a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py similarity index 97% rename from tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py rename to providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py index 0d0c41f52fd9b..ee3b5b70ad519 100644 --- a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +++ b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py @@ -42,7 +42,8 @@ GCSSynchronizeBucketsOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_metastore" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -191,14 +192,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py similarity index 95% rename from tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py rename to providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py index 6a5a7566b3429..12af45d210653 100644 --- a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +++ b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py @@ -35,7 +35,8 @@ DataprocMetastoreRestoreServiceOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_metastore_backup" @@ -125,14 +126,14 @@ ) (create_service >> backup_service >> list_backups >> restore_service >> delete_backup >> delete_service) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py similarity index 97% rename from tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py rename to providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py index 07677e11cbf36..6b06b868c73e2 100644 --- a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py +++ b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py @@ -42,7 +42,8 @@ from airflow.providers.google.cloud.sensors.dataproc_metastore import MetastoreHivePartitionSensor from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "hive_partition_sensor" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -231,14 +232,14 @@ def get_hive_warehouse_bucket(**kwargs): >> [delete_dataproc_cluster, delete_metastore_service, delete_warehouse_bucket] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_functions/__init__.py b/providers/tests/system/google/cloud/datastore/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/cloud_functions/__init__.py rename to providers/tests/system/google/cloud/datastore/__init__.py diff --git a/tests/system/providers/google/cloud/datastore/example_datastore_commit.py b/providers/tests/system/google/cloud/datastore/example_datastore_commit.py similarity index 96% rename from tests/system/providers/google/cloud/datastore/example_datastore_commit.py rename to providers/tests/system/google/cloud/datastore/example_datastore_commit.py index fc7eabfb183a2..430d176ed733c 100644 --- a/tests/system/providers/google/cloud/datastore/example_datastore_commit.py +++ b/providers/tests/system/google/cloud/datastore/example_datastore_commit.py @@ -38,7 +38,8 @@ ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -159,14 +160,14 @@ [delete_bucket, delete_export_operation, delete_import_operation], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/datastore/example_datastore_query.py b/providers/tests/system/google/cloud/datastore/example_datastore_query.py similarity index 93% rename from tests/system/providers/google/cloud/datastore/example_datastore_query.py rename to providers/tests/system/google/cloud/datastore/example_datastore_query.py index 24bb1b85dd983..0b282a1feb34e 100644 --- a/tests/system/providers/google/cloud/datastore/example_datastore_query.py +++ b/providers/tests/system/google/cloud/datastore/example_datastore_query.py @@ -31,7 +31,8 @@ CloudDatastoreBeginTransactionOperator, CloudDatastoreRunQueryOperator, ) -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -79,14 +80,14 @@ allocate_ids >> begin_transaction_query >> run_query - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/datastore/example_datastore_rollback.py b/providers/tests/system/google/cloud/datastore/example_datastore_rollback.py similarity index 91% rename from tests/system/providers/google/cloud/datastore/example_datastore_rollback.py rename to providers/tests/system/google/cloud/datastore/example_datastore_rollback.py index 09661b0ae0c1e..44d0dc1709418 100644 --- a/tests/system/providers/google/cloud/datastore/example_datastore_rollback.py +++ b/providers/tests/system/google/cloud/datastore/example_datastore_rollback.py @@ -30,7 +30,8 @@ CloudDatastoreBeginTransactionOperator, CloudDatastoreRollbackOperator, ) -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -62,14 +63,14 @@ begin_transaction_to_rollback >> rollback_transaction - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_memorystore/__init__.py b/providers/tests/system/google/cloud/gcs/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/cloud_memorystore/__init__.py rename to providers/tests/system/google/cloud/gcs/__init__.py diff --git a/tests/system/providers/google/cloud/gcs/example_calendar_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py similarity index 96% rename from tests/system/providers/google/cloud/gcs/example_calendar_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py index 52023060c7d6d..f44e7315e2d18 100644 --- a/tests/system/providers/google/cloud/gcs/example_calendar_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py @@ -115,13 +115,13 @@ def delete_connection(connection_id: str) -> None: >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_firestore.py b/providers/tests/system/google/cloud/gcs/example_firestore.py similarity index 97% rename from tests/system/providers/google/cloud/gcs/example_firestore.py rename to providers/tests/system/google/cloud/gcs/example_firestore.py index cd829f0ca79c5..89e0e3fe1c6d9 100644 --- a/tests/system/providers/google/cloud/gcs/example_firestore.py +++ b/providers/tests/system/google/cloud/gcs/example_firestore.py @@ -170,14 +170,14 @@ >> [delete_dataset, delete_bucket] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_acl.py b/providers/tests/system/google/cloud/gcs/example_gcs_acl.py similarity index 94% rename from tests/system/providers/google/cloud/gcs/example_gcs_acl.py rename to providers/tests/system/google/cloud/gcs/example_gcs_acl.py index 0b550b22ba07c..7843c0877c1d9 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_acl.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_acl.py @@ -33,7 +33,8 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -107,14 +108,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py b/providers/tests/system/google/cloud/gcs/example_gcs_copy_delete.py similarity index 95% rename from tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py rename to providers/tests/system/google/cloud/gcs/example_gcs_copy_delete.py index aebb1e3e7ed85..6bbec540df9f2 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_copy_delete.py @@ -36,7 +36,8 @@ from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -121,14 +122,14 @@ [delete_bucket_src, delete_bucket_dst], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_sensor.py b/providers/tests/system/google/cloud/gcs/example_gcs_sensor.py similarity index 97% rename from tests/system/providers/google/cloud/gcs/example_gcs_sensor.py rename to providers/tests/system/google/cloud/gcs/example_gcs_sensor.py index 5dc42604ddca0..2d4da2887d4e5 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_sensor.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_sensor.py @@ -36,7 +36,8 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -200,14 +201,14 @@ def mode_setter(self, value): delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py similarity index 93% rename from tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery.py rename to providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py index 94286ade4369c..5d7e393bff29e 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py @@ -31,7 +31,8 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "gcs_to_bigquery_operator" @@ -81,13 +82,13 @@ >> delete_test_dataset ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery_async.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py similarity index 96% rename from tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery_async.py rename to providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py index 27bef2486fd56..1ca531b818008 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_bigquery_async.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py @@ -31,7 +31,8 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -173,13 +174,13 @@ >> delete_test_dataset_delimiter ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py similarity index 97% rename from tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py index 55bec85a50562..ac1d70307a535 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py @@ -40,7 +40,8 @@ from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -274,14 +275,14 @@ def delete_work_dir(create_workdir_result: str) -> None: [delete_bucket_src, delete_bucket_dst, delete_work_dir(create_workdir_task)], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_gdrive.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py similarity index 97% rename from tests/system/providers/google/cloud/gcs/example_gcs_to_gdrive.py rename to providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py index e121e63bac971..bfb200a15a63d 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_gdrive.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py @@ -39,7 +39,8 @@ from airflow.providers.google.suite.transfers.gcs_to_gdrive import GCSToGoogleDriveOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -203,13 +204,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_sheets.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py similarity index 97% rename from tests/system/providers/google/cloud/gcs/example_gcs_to_sheets.py rename to providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py index a25c499ec5071..69947698cd7bb 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_sheets.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py @@ -131,13 +131,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_transform.py b/providers/tests/system/google/cloud/gcs/example_gcs_transform.py similarity index 93% rename from tests/system/providers/google/cloud/gcs/example_gcs_transform.py rename to providers/tests/system/google/cloud/gcs/example_gcs_transform.py index 018076173d28f..0b59119c6bc1e 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_transform.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_transform.py @@ -33,7 +33,8 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -94,14 +95,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_transform_timespan.py b/providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py similarity index 95% rename from tests/system/providers/google/cloud/gcs/example_gcs_transform_timespan.py rename to providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py index eaa3d5e5c0eda..3a5b03695ee40 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_transform_timespan.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py @@ -34,7 +34,8 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -113,14 +114,14 @@ [delete_bucket_src, delete_bucket_dst], ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_upload_download.py b/providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py similarity index 94% rename from tests/system/providers/google/cloud/gcs/example_gcs_upload_download.py rename to providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py index ae34aa639f957..6907a29fb5389 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_upload_download.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py @@ -30,7 +30,8 @@ from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -91,14 +92,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_gdrive_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py similarity index 96% rename from tests/system/providers/google/cloud/gcs/example_gdrive_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py index a5842ac17a530..6ee188bd957ec 100644 --- a/tests/system/providers/google/cloud/gcs/example_gdrive_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py @@ -33,7 +33,8 @@ from airflow.providers.google.suite.transfers.gcs_to_gdrive import GCSToGoogleDriveOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -164,13 +165,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_mssql_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py similarity index 93% rename from tests/system/providers/google/cloud/gcs/example_mssql_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py index 995361813c7ed..bc6ac79f1e0b1 100644 --- a/tests/system/providers/google/cloud/gcs/example_mssql_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py @@ -23,7 +23,8 @@ from airflow.models.dag import DAG from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID try: from airflow.providers.google.cloud.transfers.mssql_to_gcs import MSSQLToGCSOperator @@ -77,14 +78,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py similarity index 98% rename from tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py index a673ab88f722e..cc53337e56934 100644 --- a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py @@ -295,13 +295,13 @@ def delete_connection(connection_id: str) -> None: mysql_to_gcs >> [delete_gcs_bucket, delete_firewall_rule, delete_gce_instance, delete_connection_task] delete_gce_instance >> delete_persistent_disk - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_oracle_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py similarity index 92% rename from tests/system/providers/google/cloud/gcs/example_oracle_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py index 8e2b6fb979ef8..c727eef1f66fb 100644 --- a/tests/system/providers/google/cloud/gcs/example_oracle_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py @@ -23,7 +23,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.oracle_to_gcs import OracleToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -63,13 +64,13 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_s3_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py similarity index 95% rename from tests/system/providers/google/cloud/gcs/example_s3_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py index 8b53e8e531156..487dbbce552e2 100644 --- a/tests/system/providers/google/cloud/gcs/example_s3_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py @@ -27,7 +27,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.s3_to_gcs import S3ToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -109,14 +110,14 @@ def upload_file(): >> delete_gcs_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py similarity index 95% rename from tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py index 2860d8552e101..994e7e8242e1e 100644 --- a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py @@ -31,7 +31,8 @@ from airflow.providers.google.cloud.transfers.sftp_to_gcs import SFTPToGCSOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -118,14 +119,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_sheets.py b/providers/tests/system/google/cloud/gcs/example_sheets.py similarity index 97% rename from tests/system/providers/google/cloud/gcs/example_sheets.py rename to providers/tests/system/google/cloud/gcs/example_sheets.py index 2247819494f91..7d17379819168 100644 --- a/tests/system/providers/google/cloud/gcs/example_sheets.py +++ b/providers/tests/system/google/cloud/gcs/example_sheets.py @@ -145,13 +145,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_sheets_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py similarity index 97% rename from tests/system/providers/google/cloud/gcs/example_sheets_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py index d4890cbbb1a44..08688c332563a 100644 --- a/tests/system/providers/google/cloud/gcs/example_sheets_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py @@ -120,13 +120,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py similarity index 98% rename from tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py rename to providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py index f192837437142..af92e550481f7 100644 --- a/tests/system/providers/google/cloud/gcs/example_trino_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py @@ -221,13 +221,13 @@ def safe_name(s: str) -> str: >> [delete_dataset, delete_bucket] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/cloud_run/__init__.py b/providers/tests/system/google/cloud/gcs/resources/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/cloud_run/__init__.py rename to providers/tests/system/google/cloud/gcs/resources/__init__.py diff --git a/tests/system/providers/google/cloud/gcs/resources/example_upload.txt b/providers/tests/system/google/cloud/gcs/resources/example_upload.txt similarity index 100% rename from tests/system/providers/google/cloud/gcs/resources/example_upload.txt rename to providers/tests/system/google/cloud/gcs/resources/example_upload.txt diff --git a/tests/system/providers/google/cloud/gcs/resources/tmp.tar.gz b/providers/tests/system/google/cloud/gcs/resources/tmp.tar.gz similarity index 100% rename from tests/system/providers/google/cloud/gcs/resources/tmp.tar.gz rename to providers/tests/system/google/cloud/gcs/resources/tmp.tar.gz diff --git a/tests/system/providers/google/cloud/gcs/resources/transform_script.py b/providers/tests/system/google/cloud/gcs/resources/transform_script.py similarity index 100% rename from tests/system/providers/google/cloud/gcs/resources/transform_script.py rename to providers/tests/system/google/cloud/gcs/resources/transform_script.py diff --git a/tests/system/providers/google/cloud/gcs/resources/transform_timespan.py b/providers/tests/system/google/cloud/gcs/resources/transform_timespan.py similarity index 100% rename from tests/system/providers/google/cloud/gcs/resources/transform_timespan.py rename to providers/tests/system/google/cloud/gcs/resources/transform_timespan.py diff --git a/tests/system/providers/google/cloud/gcs/resources/us-states.csv b/providers/tests/system/google/cloud/gcs/resources/us-states.csv similarity index 100% rename from tests/system/providers/google/cloud/gcs/resources/us-states.csv rename to providers/tests/system/google/cloud/gcs/resources/us-states.csv diff --git a/tests/system/providers/google/cloud/cloud_sql/__init__.py b/providers/tests/system/google/cloud/kubernetes_engine/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/cloud_sql/__init__.py rename to providers/tests/system/google/cloud/kubernetes_engine/__init__.py diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py similarity index 95% rename from tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py rename to providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py index 173fddad3a065..e9fe3f6836b94 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py @@ -32,7 +32,8 @@ ) from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "kubernetes_engine" @@ -112,14 +113,14 @@ create_cluster >> [pod_task, pod_task_xcom] >> delete_cluster pod_task_xcom >> pod_task_xcom_result - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py similarity index 95% rename from tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py rename to providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py index e974a628c7a52..f5cb8f570754f 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py @@ -32,7 +32,8 @@ ) from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "kubernetes_engine_async" @@ -115,14 +116,14 @@ create_cluster >> [pod_task, pod_task_xcom_async] >> delete_cluster pod_task_xcom_async >> pod_task_xcom_result - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py similarity index 97% rename from tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py rename to providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py index 2c7790c10220c..a4c56c17e5baf 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py @@ -179,14 +179,14 @@ delete_cluster, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py similarity index 97% rename from tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py rename to providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py index 5858256802a34..06c23432a923a 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py @@ -179,14 +179,14 @@ >> delete_cluster ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py similarity index 96% rename from tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py rename to providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py index 44479099d6a72..784ba994862b7 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py @@ -101,14 +101,14 @@ create_cluster >> create_resource_task >> delete_resource_task >> delete_cluster - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/composer/__init__.py b/providers/tests/system/google/cloud/life_sciences/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/composer/__init__.py rename to providers/tests/system/google/cloud/life_sciences/__init__.py diff --git a/tests/system/providers/google/cloud/life_sciences/example_life_sciences.py b/providers/tests/system/google/cloud/life_sciences/example_life_sciences.py similarity index 95% rename from tests/system/providers/google/cloud/life_sciences/example_life_sciences.py rename to providers/tests/system/google/cloud/life_sciences/example_life_sciences.py index f129dc38ea549..170eeb39ad1aa 100644 --- a/tests/system/providers/google/cloud/life_sciences/example_life_sciences.py +++ b/providers/tests/system/google/cloud/life_sciences/example_life_sciences.py @@ -27,7 +27,8 @@ from airflow.providers.google.cloud.operators.life_sciences import LifeSciencesRunPipelineOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -130,14 +131,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/compute/__init__.py b/providers/tests/system/google/cloud/life_sciences/resources/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/compute/__init__.py rename to providers/tests/system/google/cloud/life_sciences/resources/__init__.py diff --git a/tests/system/providers/google/cloud/life_sciences/resources/file b/providers/tests/system/google/cloud/life_sciences/resources/file similarity index 100% rename from tests/system/providers/google/cloud/life_sciences/resources/file rename to providers/tests/system/google/cloud/life_sciences/resources/file diff --git a/tests/system/providers/google/cloud/data_loss_prevention/__init__.py b/providers/tests/system/google/cloud/ml_engine/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/data_loss_prevention/__init__.py rename to providers/tests/system/google/cloud/ml_engine/__init__.py diff --git a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py b/providers/tests/system/google/cloud/ml_engine/example_mlengine.py similarity index 98% rename from tests/system/providers/google/cloud/ml_engine/example_mlengine.py rename to providers/tests/system/google/cloud/ml_engine/example_mlengine.py index bde2c0bbaf9ee..f0eaf7cb30171 100644 --- a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py +++ b/providers/tests/system/google/cloud/ml_engine/example_mlengine.py @@ -280,13 +280,13 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/data_loss_prevention/resources/__init__.py b/providers/tests/system/google/cloud/natural_language/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/data_loss_prevention/resources/__init__.py rename to providers/tests/system/google/cloud/natural_language/__init__.py diff --git a/tests/system/providers/google/cloud/natural_language/example_natural_language.py b/providers/tests/system/google/cloud/natural_language/example_natural_language.py similarity index 97% rename from tests/system/providers/google/cloud/natural_language/example_natural_language.py rename to providers/tests/system/google/cloud/natural_language/example_natural_language.py index e04fdf4fb601b..cc1aba0f165c7 100644 --- a/tests/system/providers/google/cloud/natural_language/example_natural_language.py +++ b/providers/tests/system/google/cloud/natural_language/example_natural_language.py @@ -119,13 +119,13 @@ analyze_sentiment >> analyze_sentiment_result analyze_classify_text >> analyze_classify_text_result - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/__init__.py b/providers/tests/system/google/cloud/pubsub/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/dataflow/__init__.py rename to providers/tests/system/google/cloud/pubsub/__init__.py diff --git a/tests/system/providers/google/cloud/pubsub/example_pubsub.py b/providers/tests/system/google/cloud/pubsub/example_pubsub.py similarity index 97% rename from tests/system/providers/google/cloud/pubsub/example_pubsub.py rename to providers/tests/system/google/cloud/pubsub/example_pubsub.py index 4ff3091e5fa53..93ab1cde6e0a8 100644 --- a/tests/system/providers/google/cloud/pubsub/example_pubsub.py +++ b/providers/tests/system/google/cloud/pubsub/example_pubsub.py @@ -146,14 +146,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/pubsub/example_pubsub_deferrable.py b/providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py similarity index 96% rename from tests/system/providers/google/cloud/pubsub/example_pubsub_deferrable.py rename to providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py index 22c0d012aea9c..a902ea5617f6d 100644 --- a/tests/system/providers/google/cloud/pubsub/example_pubsub_deferrable.py +++ b/providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py @@ -101,14 +101,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataflow/resources/__init__.py b/providers/tests/system/google/cloud/spanner/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/dataflow/resources/__init__.py rename to providers/tests/system/google/cloud/spanner/__init__.py diff --git a/tests/system/providers/google/cloud/spanner/example_spanner.py b/providers/tests/system/google/cloud/spanner/example_spanner.py similarity index 96% rename from tests/system/providers/google/cloud/spanner/example_spanner.py rename to providers/tests/system/google/cloud/spanner/example_spanner.py index a7bef8f6ecd0f..b11a5cd61f92c 100644 --- a/tests/system/providers/google/cloud/spanner/example_spanner.py +++ b/providers/tests/system/google/cloud/spanner/example_spanner.py @@ -34,7 +34,8 @@ SpannerUpdateDatabaseInstanceOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -160,14 +161,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataform/__init__.py b/providers/tests/system/google/cloud/speech_to_text/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/dataform/__init__.py rename to providers/tests/system/google/cloud/speech_to_text/__init__.py diff --git a/tests/system/providers/google/cloud/speech_to_text/example_speech_to_text.py b/providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py similarity index 94% rename from tests/system/providers/google/cloud/speech_to_text/example_speech_to_text.py rename to providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py index d551144a53416..f2382a6053aa4 100644 --- a/tests/system/providers/google/cloud/speech_to_text/example_speech_to_text.py +++ b/providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py @@ -27,7 +27,8 @@ from airflow.providers.google.cloud.operators.speech_to_text import CloudSpeechToTextRecognizeSpeechOperator from airflow.providers.google.cloud.operators.text_to_speech import CloudTextToSpeechSynthesizeOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -88,14 +89,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/datafusion/__init__.py b/providers/tests/system/google/cloud/sql_to_sheets/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/datafusion/__init__.py rename to providers/tests/system/google/cloud/sql_to_sheets/__init__.py diff --git a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py b/providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py similarity index 98% rename from tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py rename to providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py index 11231c0dfd40c..d34531ba52a0a 100644 --- a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py +++ b/providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py @@ -315,13 +315,13 @@ def delete_connection(connection_id: str) -> None: ] delete_gce_instance >> delete_persistent_disk - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/datapipelines/__init__.py b/providers/tests/system/google/cloud/stackdriver/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/datapipelines/__init__.py rename to providers/tests/system/google/cloud/stackdriver/__init__.py diff --git a/tests/system/providers/google/cloud/stackdriver/example_stackdriver.py b/providers/tests/system/google/cloud/stackdriver/example_stackdriver.py similarity index 97% rename from tests/system/providers/google/cloud/stackdriver/example_stackdriver.py rename to providers/tests/system/google/cloud/stackdriver/example_stackdriver.py index ea8690e70dbee..202c9bcfbdccf 100644 --- a/tests/system/providers/google/cloud/stackdriver/example_stackdriver.py +++ b/providers/tests/system/google/cloud/stackdriver/example_stackdriver.py @@ -40,7 +40,8 @@ StackdriverUpsertNotificationChannelOperator, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -227,13 +228,13 @@ delete_alert_policy_2, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataplex/__init__.py b/providers/tests/system/google/cloud/storage_transfer/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/dataplex/__init__.py rename to providers/tests/system/google/cloud/storage_transfer/__init__.py diff --git a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py similarity index 98% rename from tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py rename to providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py index e42be683ea52d..4d6a010954401 100644 --- a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +++ b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py @@ -62,7 +62,8 @@ CloudDataTransferServiceJobStatusSensor, ) from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -253,14 +254,14 @@ ] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py similarity index 98% rename from tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py rename to providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py index 2920409ce7547..b2cef2f831b5f 100644 --- a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py +++ b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py @@ -195,13 +195,13 @@ >> [delete_transfer, delete_bucket_src, delete_bucket_dst] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py similarity index 96% rename from tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py rename to providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py index 53752d2079133..e670eba755428 100644 --- a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py +++ b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py @@ -106,13 +106,13 @@ >> [delete_bucket_src, delete_bucket_dst] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataprep/__init__.py b/providers/tests/system/google/cloud/storage_transfer/resources/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/dataprep/__init__.py rename to providers/tests/system/google/cloud/storage_transfer/resources/__init__.py diff --git a/tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcp_file b/providers/tests/system/google/cloud/storage_transfer/resources/transfer_service_gcp_file similarity index 100% rename from tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcp_file rename to providers/tests/system/google/cloud/storage_transfer/resources/transfer_service_gcp_file diff --git a/tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcs_to_gcs_file b/providers/tests/system/google/cloud/storage_transfer/resources/transfer_service_gcs_to_gcs_file similarity index 100% rename from tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcs_to_gcs_file rename to providers/tests/system/google/cloud/storage_transfer/resources/transfer_service_gcs_to_gcs_file diff --git a/tests/system/providers/google/cloud/dataproc/__init__.py b/providers/tests/system/google/cloud/tasks/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/dataproc/__init__.py rename to providers/tests/system/google/cloud/tasks/__init__.py diff --git a/tests/system/providers/google/cloud/tasks/example_queue.py b/providers/tests/system/google/cloud/tasks/example_queue.py similarity index 97% rename from tests/system/providers/google/cloud/tasks/example_queue.py rename to providers/tests/system/google/cloud/tasks/example_queue.py index 4c29b584f5bfc..9797a29d04e22 100644 --- a/tests/system/providers/google/cloud/tasks/example_queue.py +++ b/providers/tests/system/google/cloud/tasks/example_queue.py @@ -161,14 +161,14 @@ def generate_random_string(): delete_queue, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/tasks/example_tasks.py b/providers/tests/system/google/cloud/tasks/example_tasks.py similarity index 97% rename from tests/system/providers/google/cloud/tasks/example_tasks.py rename to providers/tests/system/google/cloud/tasks/example_tasks.py index 1a85ac53187b3..0eae95fd1075d 100644 --- a/tests/system/providers/google/cloud/tasks/example_tasks.py +++ b/providers/tests/system/google/cloud/tasks/example_tasks.py @@ -152,14 +152,14 @@ def generate_random_string(): random_string, create_queue, create_task, tasks_get, list_tasks, run_task, delete_task, delete_queue ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/dataproc_metastore/__init__.py b/providers/tests/system/google/cloud/text_to_speech/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/dataproc_metastore/__init__.py rename to providers/tests/system/google/cloud/text_to_speech/__init__.py diff --git a/tests/system/providers/google/cloud/text_to_speech/example_text_to_speech.py b/providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py similarity index 93% rename from tests/system/providers/google/cloud/text_to_speech/example_text_to_speech.py rename to providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py index f5c12a44a4abf..0227049508a74 100644 --- a/tests/system/providers/google/cloud/text_to_speech/example_text_to_speech.py +++ b/providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py @@ -24,7 +24,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.operators.text_to_speech import CloudTextToSpeechSynthesizeOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -77,14 +78,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/datastore/__init__.py b/providers/tests/system/google/cloud/transfers/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/datastore/__init__.py rename to providers/tests/system/google/cloud/transfers/__init__.py diff --git a/tests/system/providers/google/cloud/transfers/example_gcs_to_sftp.py b/providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py similarity index 96% rename from tests/system/providers/google/cloud/transfers/example_gcs_to_sftp.py rename to providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py index 0caed0affa19a..8d3f19416313a 100644 --- a/tests/system/providers/google/cloud/transfers/example_gcs_to_sftp.py +++ b/providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py @@ -31,7 +31,8 @@ from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.providers.sftp.sensors.sftp import SFTPSensor from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -170,14 +171,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/transfers/example_gdrive_to_local.py b/providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py similarity index 96% rename from tests/system/providers/google/cloud/transfers/example_gdrive_to_local.py rename to providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py index d9ac60301b77b..abfcbabce0852 100644 --- a/tests/system/providers/google/cloud/transfers/example_gdrive_to_local.py +++ b/providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py @@ -40,7 +40,8 @@ from airflow.providers.google.suite.transfers.gcs_to_gdrive import GCSToGoogleDriveOperator from airflow.settings import Session, json from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -166,13 +167,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py b/providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py similarity index 98% rename from tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py rename to providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py index 33a289c1ffa1c..91bde027982ec 100644 --- a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py +++ b/providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py @@ -290,13 +290,13 @@ def delete_connection(connection_id: str) -> None: postgres_to_gcs >> [delete_gcs_bucket, delete_firewall_rule, delete_gce_instance, delete_connection_task] delete_gce_instance >> delete_persistent_disk - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/gcs/__init__.py b/providers/tests/system/google/cloud/transfers/resources/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/gcs/__init__.py rename to providers/tests/system/google/cloud/transfers/resources/__init__.py diff --git a/tests/system/providers/google/cloud/transfers/resources/empty.txt b/providers/tests/system/google/cloud/transfers/resources/empty.txt similarity index 100% rename from tests/system/providers/google/cloud/transfers/resources/empty.txt rename to providers/tests/system/google/cloud/transfers/resources/empty.txt diff --git a/tests/system/providers/google/cloud/gcs/resources/__init__.py b/providers/tests/system/google/cloud/translate/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/gcs/resources/__init__.py rename to providers/tests/system/google/cloud/translate/__init__.py diff --git a/tests/system/providers/google/cloud/translate/example_translate.py b/providers/tests/system/google/cloud/translate/example_translate.py similarity index 94% rename from tests/system/providers/google/cloud/translate/example_translate.py rename to providers/tests/system/google/cloud/translate/example_translate.py index b593060f6e5bb..13a3ee061e308 100644 --- a/tests/system/providers/google/cloud/translate/example_translate.py +++ b/providers/tests/system/google/cloud/translate/example_translate.py @@ -56,14 +56,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/kubernetes_engine/__init__.py b/providers/tests/system/google/cloud/translate_speech/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/kubernetes_engine/__init__.py rename to providers/tests/system/google/cloud/translate_speech/__init__.py diff --git a/tests/system/providers/google/cloud/translate_speech/example_translate_speech.py b/providers/tests/system/google/cloud/translate_speech/example_translate_speech.py similarity index 96% rename from tests/system/providers/google/cloud/translate_speech/example_translate_speech.py rename to providers/tests/system/google/cloud/translate_speech/example_translate_speech.py index bb50adb4b62c5..4fe34d0b86f57 100644 --- a/tests/system/providers/google/cloud/translate_speech/example_translate_speech.py +++ b/providers/tests/system/google/cloud/translate_speech/example_translate_speech.py @@ -110,14 +110,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/life_sciences/__init__.py b/providers/tests/system/google/cloud/vertex_ai/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/life_sciences/__init__.py rename to providers/tests/system/google/cloud/vertex_ai/__init__.py diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py similarity index 97% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py index 03634b58f645b..8dd0ecc9c9550 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py @@ -150,13 +150,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py similarity index 97% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py index c26ea94325e76..bf96da992aca1 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py @@ -135,13 +135,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py similarity index 93% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py index 3303f219d4e1c..ca6c65e1a6ddd 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py @@ -52,13 +52,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py similarity index 97% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py index 91260eccdea57..fb32c7475c7f1 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py @@ -140,13 +140,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py similarity index 97% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py index cde6bb183e777..6c3db89382f24 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py @@ -145,13 +145,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py index 38198b5526874..78bb9ffa6bad4 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py @@ -231,13 +231,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py index dc09a8be90ed7..1295d20983eeb 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py @@ -208,13 +208,13 @@ def TABULAR_DATASET(bucket_name): # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py index 8762feb85ba39..3fa6169cb2759 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py @@ -250,13 +250,13 @@ def TABULAR_DATASET(bucket_name): # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py index 49a8d870bc394..fbc40b888216b 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py @@ -211,13 +211,13 @@ def TABULAR_DATASET(bucket_name): # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py index 77b69081a4734..3e1b98a2232d1 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_dataset.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py @@ -269,13 +269,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py index 8fa802b51744d..12b1181a79280 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py @@ -206,13 +206,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py index f9fe332b0af7d..7e696184eef41 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py @@ -155,14 +155,14 @@ ) # [END how_to_cloud_vertex_ai_run_evaluation_operator] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py similarity index 94% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py index 18958cb409e65..affba0b6aede8 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py @@ -56,13 +56,13 @@ ) # [END how_to_cloud_vertex_ai_supervised_fine_tuning_train_operator] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py index 913fff2b4e09b..55254741d9cc2 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py @@ -181,13 +181,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py similarity index 93% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py index 0a1a8d3fb247b..75de719b6aa9e 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py @@ -51,13 +51,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py similarity index 98% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py index b06f8287798df..3ad5537a10c29 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_model_service.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py @@ -345,13 +345,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py similarity index 97% rename from tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py rename to providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py index 7dd4aa84fe41e..5ab29ee999f16 100644 --- a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py @@ -177,13 +177,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/life_sciences/resources/__init__.py b/providers/tests/system/google/cloud/video_intelligence/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/life_sciences/resources/__init__.py rename to providers/tests/system/google/cloud/video_intelligence/__init__.py diff --git a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py b/providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py similarity index 97% rename from tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py rename to providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py index 499db2d6427ba..e630fd1471a8a 100644 --- a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +++ b/providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py @@ -154,14 +154,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/ml_engine/__init__.py b/providers/tests/system/google/cloud/vision/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/ml_engine/__init__.py rename to providers/tests/system/google/cloud/vision/__init__.py diff --git a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py b/providers/tests/system/google/cloud/vision/example_vision_annotate_image.py similarity index 97% rename from tests/system/providers/google/cloud/vision/example_vision_annotate_image.py rename to providers/tests/system/google/cloud/vision/example_vision_annotate_image.py index 2a4d7b75f1332..1e09fb7fe4f9d 100644 --- a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +++ b/providers/tests/system/google/cloud/vision/example_vision_annotate_image.py @@ -41,7 +41,7 @@ # [START howto_operator_vision_enums_import] from google.cloud.vision_v1 import Feature # isort:skip -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [END howto_operator_vision_enums_import] @@ -191,14 +191,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vision/example_vision_autogenerated.py b/providers/tests/system/google/cloud/vision/example_vision_autogenerated.py similarity index 98% rename from tests/system/providers/google/cloud/vision/example_vision_autogenerated.py rename to providers/tests/system/google/cloud/vision/example_vision_autogenerated.py index 907386ceb295c..11fd3cdd54029 100644 --- a/tests/system/providers/google/cloud/vision/example_vision_autogenerated.py +++ b/providers/tests/system/google/cloud/vision/example_vision_autogenerated.py @@ -59,7 +59,7 @@ # [END howto_operator_vision_reference_image_import] # [START howto_operator_vision_enums_import] from google.cloud.vision_v1 import Feature # isort:skip -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [END howto_operator_vision_enums_import] @@ -268,14 +268,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vision/example_vision_explicit.py b/providers/tests/system/google/cloud/vision/example_vision_explicit.py similarity index 98% rename from tests/system/providers/google/cloud/vision/example_vision_explicit.py rename to providers/tests/system/google/cloud/vision/example_vision_explicit.py index 663be91259683..0c71be95bda7a 100644 --- a/tests/system/providers/google/cloud/vision/example_vision_explicit.py +++ b/providers/tests/system/google/cloud/vision/example_vision_explicit.py @@ -54,7 +54,7 @@ # [END howto_operator_vision_product_import_2] # [START howto_operator_vision_reference_image_import_2] from google.cloud.vision_v1.types import ReferenceImage # isort:skip -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID # [END howto_operator_vision_reference_image_import_2] @@ -279,14 +279,14 @@ delete_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/natural_language/__init__.py b/providers/tests/system/google/cloud/workflows/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/natural_language/__init__.py rename to providers/tests/system/google/cloud/workflows/__init__.py diff --git a/tests/system/providers/google/cloud/workflows/example_workflows.py b/providers/tests/system/google/cloud/workflows/example_workflows.py similarity index 98% rename from tests/system/providers/google/cloud/workflows/example_workflows.py rename to providers/tests/system/google/cloud/workflows/example_workflows.py index b010a146b9bd1..6d13484acbb42 100644 --- a/tests/system/providers/google/cloud/workflows/example_workflows.py +++ b/providers/tests/system/google/cloud/workflows/example_workflows.py @@ -227,14 +227,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/conftest.py b/providers/tests/system/google/conftest.py similarity index 100% rename from tests/system/providers/google/conftest.py rename to providers/tests/system/google/conftest.py diff --git a/tests/system/providers/google/cloud/pubsub/__init__.py b/providers/tests/system/google/datacatalog/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/pubsub/__init__.py rename to providers/tests/system/google/datacatalog/__init__.py diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py b/providers/tests/system/google/datacatalog/example_datacatalog_entries.py similarity index 97% rename from tests/system/providers/google/datacatalog/example_datacatalog_entries.py rename to providers/tests/system/google/datacatalog/example_datacatalog_entries.py index 47edfb96368f1..0f84361d4fcc8 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_entries.py @@ -37,7 +37,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -199,14 +200,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py b/providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py similarity index 97% rename from tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py rename to providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py index 781d047c53469..a77a023f32338 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py @@ -39,7 +39,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -222,14 +223,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py b/providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py similarity index 97% rename from tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py rename to providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py index b8dd9170c3c02..af49d3de08fa7 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py @@ -36,7 +36,8 @@ ) from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -182,14 +183,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py b/providers/tests/system/google/datacatalog/example_datacatalog_tags.py similarity index 97% rename from tests/system/providers/google/datacatalog/example_datacatalog_tags.py rename to providers/tests/system/google/datacatalog/example_datacatalog_tags.py index 17397fcea2806..fdbdaa451056a 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_tags.py @@ -40,7 +40,8 @@ from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -232,14 +233,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/spanner/__init__.py b/providers/tests/system/google/firebase/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/spanner/__init__.py rename to providers/tests/system/google/firebase/__init__.py diff --git a/tests/providers/telegram/operators/__init__.py b/providers/tests/system/google/leveldb/__init__.py similarity index 100% rename from tests/providers/telegram/operators/__init__.py rename to providers/tests/system/google/leveldb/__init__.py diff --git a/tests/system/providers/google/leveldb/example_leveldb.py b/providers/tests/system/google/leveldb/example_leveldb.py similarity index 94% rename from tests/system/providers/google/leveldb/example_leveldb.py rename to providers/tests/system/google/leveldb/example_leveldb.py index 8474de830275c..8b11d9e87bebc 100644 --- a/tests/system/providers/google/leveldb/example_leveldb.py +++ b/providers/tests/system/google/leveldb/example_leveldb.py @@ -60,14 +60,14 @@ # [END howto_operator_leveldb_put_key] get_key_leveldb_task >> put_key_leveldb_task - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/teradata/hooks/__init__.py b/providers/tests/system/google/marketing_platform/__init__.py similarity index 100% rename from tests/providers/teradata/hooks/__init__.py rename to providers/tests/system/google/marketing_platform/__init__.py diff --git a/tests/system/providers/google/marketing_platform/example_analytics_admin.py b/providers/tests/system/google/marketing_platform/example_analytics_admin.py similarity index 98% rename from tests/system/providers/google/marketing_platform/example_analytics_admin.py rename to providers/tests/system/google/marketing_platform/example_analytics_admin.py index be1ad9f257f99..16ce6f8190bdc 100644 --- a/tests/system/providers/google/marketing_platform/example_analytics_admin.py +++ b/providers/tests/system/google/marketing_platform/example_analytics_admin.py @@ -219,13 +219,13 @@ def delete_connection(connection_id: str) -> None: # TEST TEARDOWN >> delete_connection_task ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/marketing_platform/example_campaign_manager.py b/providers/tests/system/google/marketing_platform/example_campaign_manager.py similarity index 98% rename from tests/system/providers/google/marketing_platform/example_campaign_manager.py rename to providers/tests/system/google/marketing_platform/example_campaign_manager.py index 932769eb52adc..0c3b26e5e98b5 100644 --- a/tests/system/providers/google/marketing_platform/example_campaign_manager.py +++ b/providers/tests/system/google/marketing_platform/example_campaign_manager.py @@ -55,7 +55,8 @@ ) from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -321,13 +322,13 @@ def delete_connection(connection_id: str) -> None: >> delete_connection(connection_id=CONNECTION_ID) ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/marketing_platform/example_search_ads.py b/providers/tests/system/google/marketing_platform/example_search_ads.py similarity index 97% rename from tests/system/providers/google/marketing_platform/example_search_ads.py rename to providers/tests/system/google/marketing_platform/example_search_ads.py index b8e71de90adc2..5d86a3ad1684d 100644 --- a/tests/system/providers/google/marketing_platform/example_search_ads.py +++ b/providers/tests/system/google/marketing_platform/example_search_ads.py @@ -102,7 +102,7 @@ (query_report >> get_field >> search_fields >> get_custom_column >> list_custom_columns) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/speech_to_text/__init__.py b/providers/tests/system/google/suite/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/speech_to_text/__init__.py rename to providers/tests/system/google/suite/__init__.py diff --git a/tests/system/providers/google/suite/example_local_to_drive.py b/providers/tests/system/google/suite/example_local_to_drive.py similarity index 97% rename from tests/system/providers/google/suite/example_local_to_drive.py rename to providers/tests/system/google/suite/example_local_to_drive.py index 2e79a932d4fb9..ac8ebfc5aabbc 100644 --- a/tests/system/providers/google/suite/example_local_to_drive.py +++ b/providers/tests/system/google/suite/example_local_to_drive.py @@ -141,13 +141,13 @@ def delete_connection(connection_id: str) -> None: >> delete_connection_task ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/sql_to_sheets/__init__.py b/providers/tests/system/google/suite/resources/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/sql_to_sheets/__init__.py rename to providers/tests/system/google/suite/resources/__init__.py diff --git a/tests/system/providers/google/suite/resources/test1 b/providers/tests/system/google/suite/resources/test1 similarity index 100% rename from tests/system/providers/google/suite/resources/test1 rename to providers/tests/system/google/suite/resources/test1 diff --git a/tests/system/providers/google/suite/resources/test2 b/providers/tests/system/google/suite/resources/test2 similarity index 100% rename from tests/system/providers/google/suite/resources/test2 rename to providers/tests/system/google/suite/resources/test2 diff --git a/tests/system/providers/google/cloud/stackdriver/__init__.py b/providers/tests/system/google/workplace/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/stackdriver/__init__.py rename to providers/tests/system/google/workplace/__init__.py diff --git a/tests/system/providers/google/cloud/storage_transfer/__init__.py b/providers/tests/system/http/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/storage_transfer/__init__.py rename to providers/tests/system/http/__init__.py diff --git a/tests/system/providers/http/example_http.py b/providers/tests/system/http/example_http.py similarity index 98% rename from tests/system/providers/http/example_http.py rename to providers/tests/system/http/example_http.py index bf5d08f086c18..98423943607cd 100644 --- a/tests/system/providers/http/example_http.py +++ b/providers/tests/system/http/example_http.py @@ -157,7 +157,7 @@ def get_next_page_cursor(response) -> dict | None: task_get_op_response_filter >> task_put_op >> task_del_op >> task_post_op_formenc task_post_op_formenc >> task_get_paginated -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/storage_transfer/resources/__init__.py b/providers/tests/system/influxdb/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/storage_transfer/resources/__init__.py rename to providers/tests/system/influxdb/__init__.py diff --git a/tests/system/providers/influxdb/example_influxdb.py b/providers/tests/system/influxdb/example_influxdb.py similarity index 94% rename from tests/system/providers/influxdb/example_influxdb.py rename to providers/tests/system/influxdb/example_influxdb.py index 9a71271c02d9c..8e4d486742d5f 100644 --- a/tests/system/providers/influxdb/example_influxdb.py +++ b/providers/tests/system/influxdb/example_influxdb.py @@ -61,13 +61,13 @@ def test_influxdb_hook(): ) as dag: test_influxdb_hook() - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/influxdb/example_influxdb_query.py b/providers/tests/system/influxdb/example_influxdb_query.py similarity index 95% rename from tests/system/providers/influxdb/example_influxdb_query.py rename to providers/tests/system/influxdb/example_influxdb_query.py index 6a0c14781aaba..95940e8c8f366 100644 --- a/tests/system/providers/influxdb/example_influxdb_query.py +++ b/providers/tests/system/influxdb/example_influxdb_query.py @@ -43,7 +43,7 @@ # [END howto_operator_influxdb] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/tasks/__init__.py b/providers/tests/system/jdbc/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/tasks/__init__.py rename to providers/tests/system/jdbc/__init__.py diff --git a/tests/system/providers/jdbc/example_jdbc_queries.py b/providers/tests/system/jdbc/example_jdbc_queries.py similarity index 94% rename from tests/system/providers/jdbc/example_jdbc_queries.py rename to providers/tests/system/jdbc/example_jdbc_queries.py index 0da4b8d4479aa..ce9234a6fb0b7 100644 --- a/tests/system/providers/jdbc/example_jdbc_queries.py +++ b/providers/tests/system/jdbc/example_jdbc_queries.py @@ -59,13 +59,13 @@ delete_data >> insert_data >> run_this_last - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/text_to_speech/__init__.py b/providers/tests/system/jenkins/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/text_to_speech/__init__.py rename to providers/tests/system/jenkins/__init__.py diff --git a/tests/system/providers/jenkins/example_jenkins_job_trigger.py b/providers/tests/system/jenkins/example_jenkins_job_trigger.py similarity index 97% rename from tests/system/providers/jenkins/example_jenkins_job_trigger.py rename to providers/tests/system/jenkins/example_jenkins_job_trigger.py index e7fa2e073be97..6b7fab62a0d36 100644 --- a/tests/system/providers/jenkins/example_jenkins_job_trigger.py +++ b/providers/tests/system/jenkins/example_jenkins_job_trigger.py @@ -72,7 +72,7 @@ def grab_artifact_from_jenkins(url): # job_trigger >> grab_artifact_from_jenkins() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/transfers/__init__.py b/providers/tests/system/microsoft/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/transfers/__init__.py rename to providers/tests/system/microsoft/__init__.py diff --git a/tests/providers/teradata/operators/__init__.py b/providers/tests/system/microsoft/azure/__init__.py similarity index 100% rename from tests/providers/teradata/operators/__init__.py rename to providers/tests/system/microsoft/azure/__init__.py diff --git a/tests/system/providers/microsoft/azure/example_adf_run_pipeline.py b/providers/tests/system/microsoft/azure/example_adf_run_pipeline.py similarity index 96% rename from tests/system/providers/microsoft/azure/example_adf_run_pipeline.py rename to providers/tests/system/microsoft/azure/example_adf_run_pipeline.py index d883f5128737f..a6eae4d71486d 100644 --- a/tests/system/providers/microsoft/azure/example_adf_run_pipeline.py +++ b/providers/tests/system/microsoft/azure/example_adf_run_pipeline.py @@ -108,13 +108,13 @@ # Task dependency created via `XComArgs`: # run_pipeline2 >> pipeline_run_sensor - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_adls_create.py b/providers/tests/system/microsoft/azure/example_adls_create.py similarity index 93% rename from tests/system/providers/microsoft/azure/example_adls_create.py rename to providers/tests/system/microsoft/azure/example_adls_create.py index 726e9eba76ae8..3f76525b6b87e 100644 --- a/tests/system/providers/microsoft/azure/example_adls_create.py +++ b/providers/tests/system/microsoft/azure/example_adls_create.py @@ -46,13 +46,13 @@ upload_data >> delete_file - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_adls_delete.py b/providers/tests/system/microsoft/azure/example_adls_delete.py similarity index 93% rename from tests/system/providers/microsoft/azure/example_adls_delete.py rename to providers/tests/system/microsoft/azure/example_adls_delete.py index 2b1977e938dc2..34abc6a9b2dae 100644 --- a/tests/system/providers/microsoft/azure/example_adls_delete.py +++ b/providers/tests/system/microsoft/azure/example_adls_delete.py @@ -46,13 +46,13 @@ upload_file >> remove_file - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_adls_list.py b/providers/tests/system/microsoft/azure/example_adls_list.py similarity index 92% rename from tests/system/providers/microsoft/azure/example_adls_list.py rename to providers/tests/system/microsoft/azure/example_adls_list.py index a6bd2d7bd6652..594b161aaa9d6 100644 --- a/tests/system/providers/microsoft/azure/example_adls_list.py +++ b/providers/tests/system/microsoft/azure/example_adls_list.py @@ -42,13 +42,13 @@ ) # [END howto_operator_adls_list] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_azure_batch_operator.py b/providers/tests/system/microsoft/azure/example_azure_batch_operator.py similarity index 96% rename from tests/system/providers/microsoft/azure/example_azure_batch_operator.py rename to providers/tests/system/microsoft/azure/example_azure_batch_operator.py index 85977f2a0a6e4..944c737f96a0d 100644 --- a/tests/system/providers/microsoft/azure/example_azure_batch_operator.py +++ b/providers/tests/system/microsoft/azure/example_azure_batch_operator.py @@ -57,7 +57,7 @@ ) # [END howto_azure_batch_operator] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_azure_container_instances.py b/providers/tests/system/microsoft/azure/example_azure_container_instances.py similarity index 97% rename from tests/system/providers/microsoft/azure/example_azure_container_instances.py rename to providers/tests/system/microsoft/azure/example_azure_container_instances.py index 0db79a71837b1..ee9b025193e36 100644 --- a/tests/system/providers/microsoft/azure/example_azure_container_instances.py +++ b/providers/tests/system/microsoft/azure/example_azure_container_instances.py @@ -90,7 +90,7 @@ cpu=1.0, task_id="start_container_with_azure_container_volume", ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_azure_cosmosdb.py b/providers/tests/system/microsoft/azure/example_azure_cosmosdb.py similarity index 94% rename from tests/system/providers/microsoft/azure/example_azure_cosmosdb.py rename to providers/tests/system/microsoft/azure/example_azure_cosmosdb.py index d48d636f28dbf..1d4bb60b05163 100644 --- a/tests/system/providers/microsoft/azure/example_azure_cosmosdb.py +++ b/providers/tests/system/microsoft/azure/example_azure_cosmosdb.py @@ -64,13 +64,13 @@ t1 >> t2 - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_azure_service_bus.py b/providers/tests/system/microsoft/azure/example_azure_service_bus.py similarity index 98% rename from tests/system/providers/microsoft/azure/example_azure_service_bus.py rename to providers/tests/system/microsoft/azure/example_azure_service_bus.py index e0b8558ffec10..bfa77db40dba0 100644 --- a/tests/system/providers/microsoft/azure/example_azure_service_bus.py +++ b/providers/tests/system/microsoft/azure/example_azure_service_bus.py @@ -172,13 +172,13 @@ delete_service_bus_queue, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_azure_synapse.py b/providers/tests/system/microsoft/azure/example_azure_synapse.py similarity index 97% rename from tests/system/providers/microsoft/azure/example_azure_synapse.py rename to providers/tests/system/microsoft/azure/example_azure_synapse.py index 7465b71677deb..c7f7800e57834 100644 --- a/tests/system/providers/microsoft/azure/example_azure_synapse.py +++ b/providers/tests/system/microsoft/azure/example_azure_synapse.py @@ -69,7 +69,7 @@ ) # [END howto_operator_azure_synapse] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_fileshare.py b/providers/tests/system/microsoft/azure/example_fileshare.py similarity index 93% rename from tests/system/providers/microsoft/azure/example_fileshare.py rename to providers/tests/system/microsoft/azure/example_fileshare.py index f306a0188efa9..bfa9819dbef68 100644 --- a/tests/system/providers/microsoft/azure/example_fileshare.py +++ b/providers/tests/system/microsoft/azure/example_fileshare.py @@ -55,13 +55,13 @@ def delete_fileshare(): ) as dag: create_fileshare() >> delete_fileshare() - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_local_to_adls.py b/providers/tests/system/microsoft/azure/example_local_to_adls.py similarity index 93% rename from tests/system/providers/microsoft/azure/example_local_to_adls.py rename to providers/tests/system/microsoft/azure/example_local_to_adls.py index f5a75e7ce414b..d540aaf34d993 100644 --- a/tests/system/providers/microsoft/azure/example_local_to_adls.py +++ b/providers/tests/system/microsoft/azure/example_local_to_adls.py @@ -47,13 +47,13 @@ upload_file >> delete_file - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_local_to_wasb.py b/providers/tests/system/microsoft/azure/example_local_to_wasb.py similarity index 94% rename from tests/system/providers/microsoft/azure/example_local_to_wasb.py rename to providers/tests/system/microsoft/azure/example_local_to_wasb.py index b03c11e6b3673..7e368ca839928 100644 --- a/tests/system/providers/microsoft/azure/example_local_to_wasb.py +++ b/providers/tests/system/microsoft/azure/example_local_to_wasb.py @@ -49,13 +49,13 @@ upload >> delete - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_msfabric.py b/providers/tests/system/microsoft/azure/example_msfabric.py similarity index 94% rename from tests/system/providers/microsoft/azure/example_msfabric.py rename to providers/tests/system/microsoft/azure/example_msfabric.py index 5f8b0657c4019..b1113025286d4 100644 --- a/tests/system/providers/microsoft/azure/example_msfabric.py +++ b/providers/tests/system/microsoft/azure/example_msfabric.py @@ -51,13 +51,13 @@ ) # [END howto_operator_ms_fabric_create_item_schedule] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_msgraph.py b/providers/tests/system/microsoft/azure/example_msgraph.py similarity index 93% rename from tests/system/providers/microsoft/azure/example_msgraph.py rename to providers/tests/system/microsoft/azure/example_msgraph.py index 5ff7ba6f88835..33ee00468523b 100644 --- a/tests/system/providers/microsoft/azure/example_msgraph.py +++ b/providers/tests/system/microsoft/azure/example_msgraph.py @@ -49,13 +49,13 @@ site_task >> site_pages_task - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_powerbi.py b/providers/tests/system/microsoft/azure/example_powerbi.py similarity index 96% rename from tests/system/providers/microsoft/azure/example_powerbi.py rename to providers/tests/system/microsoft/azure/example_powerbi.py index 0a1bfde54a7a9..b5a982a1159a5 100644 --- a/tests/system/providers/microsoft/azure/example_powerbi.py +++ b/providers/tests/system/microsoft/azure/example_powerbi.py @@ -97,13 +97,13 @@ workspaces_task >> workspaces_info_task >> check_workspace_status_task refresh_dataset_task >> refresh_dataset_history_task - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_powerbi_dataset_refresh.py b/providers/tests/system/microsoft/azure/example_powerbi_dataset_refresh.py similarity index 95% rename from tests/system/providers/microsoft/azure/example_powerbi_dataset_refresh.py rename to providers/tests/system/microsoft/azure/example_powerbi_dataset_refresh.py index 52f1f001e9988..c02cec3e57f5e 100644 --- a/tests/system/providers/microsoft/azure/example_powerbi_dataset_refresh.py +++ b/providers/tests/system/microsoft/azure/example_powerbi_dataset_refresh.py @@ -76,13 +76,13 @@ def create_connection(conn_id_name: str): refresh_powerbi_dataset, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_s3_to_wasb.py b/providers/tests/system/microsoft/azure/example_s3_to_wasb.py similarity index 94% rename from tests/system/providers/microsoft/azure/example_s3_to_wasb.py rename to providers/tests/system/microsoft/azure/example_s3_to_wasb.py index 48fd428b9cde4..88ae64324d17e 100644 --- a/tests/system/providers/microsoft/azure/example_s3_to_wasb.py +++ b/providers/tests/system/microsoft/azure/example_s3_to_wasb.py @@ -29,7 +29,8 @@ ) from airflow.providers.microsoft.azure.transfers.s3_to_wasb import S3ToAzureBlobStorageOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +from providers.tests.system.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() @@ -102,13 +103,13 @@ remove_s3_bucket, ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure when "tearDown" task with trigger # rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_sftp_to_wasb.py b/providers/tests/system/microsoft/azure/example_sftp_to_wasb.py similarity index 95% rename from tests/system/providers/microsoft/azure/example_sftp_to_wasb.py rename to providers/tests/system/microsoft/azure/example_sftp_to_wasb.py index d80539cb4a238..42e5968304f08 100644 --- a/tests/system/providers/microsoft/azure/example_sftp_to_wasb.py +++ b/providers/tests/system/microsoft/azure/example_sftp_to_wasb.py @@ -75,13 +75,13 @@ def delete_sftp_file(): transfer_files_to_sftp_step >> transfer_files_to_azure >> delete_blob_file_step >> delete_sftp_file() - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_synapse_run_pipeline.py b/providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py similarity index 93% rename from tests/system/providers/microsoft/azure/example_synapse_run_pipeline.py rename to providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py index 6b69bd8972b4f..de87d6b4c2483 100644 --- a/tests/system/providers/microsoft/azure/example_synapse_run_pipeline.py +++ b/providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py @@ -43,13 +43,13 @@ # [END howto_operator_azure_synapse_run_pipeline] begin >> run_pipeline1 - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/example_wasb_sensors.py b/providers/tests/system/microsoft/azure/example_wasb_sensors.py similarity index 96% rename from tests/system/providers/microsoft/azure/example_wasb_sensors.py rename to providers/tests/system/microsoft/azure/example_wasb_sensors.py index 806a863cbfbc3..56c1dce34d688 100644 --- a/tests/system/providers/microsoft/azure/example_wasb_sensors.py +++ b/providers/tests/system/microsoft/azure/example_wasb_sensors.py @@ -62,7 +62,7 @@ # [END wasb_prefix_sensor] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/transfers/resources/__init__.py b/providers/tests/system/microsoft/mssql/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/transfers/resources/__init__.py rename to providers/tests/system/microsoft/mssql/__init__.py diff --git a/tests/system/providers/microsoft/mssql/create_table.sql b/providers/tests/system/microsoft/mssql/create_table.sql similarity index 100% rename from tests/system/providers/microsoft/mssql/create_table.sql rename to providers/tests/system/microsoft/mssql/create_table.sql diff --git a/tests/system/providers/microsoft/mssql/example_mssql.py b/providers/tests/system/microsoft/mssql/example_mssql.py similarity index 97% rename from tests/system/providers/microsoft/mssql/example_mssql.py rename to providers/tests/system/microsoft/mssql/example_mssql.py index 957a8a26c2980..12e2815c59d01 100644 --- a/tests/system/providers/microsoft/mssql/example_mssql.py +++ b/providers/tests/system/microsoft/mssql/example_mssql.py @@ -146,12 +146,12 @@ def insert_mssql_hook(): ) # [END mssql_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/translate/__init__.py b/providers/tests/system/microsoft/winrm/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/translate/__init__.py rename to providers/tests/system/microsoft/winrm/__init__.py diff --git a/tests/system/providers/microsoft/winrm/example_winrm.py b/providers/tests/system/microsoft/winrm/example_winrm.py similarity index 95% rename from tests/system/providers/microsoft/winrm/example_winrm.py rename to providers/tests/system/microsoft/winrm/example_winrm.py index 60a0149dc235a..1de1cb8c0fd68 100644 --- a/tests/system/providers/microsoft/winrm/example_winrm.py +++ b/providers/tests/system/microsoft/winrm/example_winrm.py @@ -64,13 +64,13 @@ [t1, t2, t3] >> run_this_last - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/translate_speech/__init__.py b/providers/tests/system/mysql/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/translate_speech/__init__.py rename to providers/tests/system/mysql/__init__.py diff --git a/tests/system/providers/mysql/example_mysql.py b/providers/tests/system/mysql/example_mysql.py similarity index 95% rename from tests/system/providers/mysql/example_mysql.py rename to providers/tests/system/mysql/example_mysql.py index a890b7846ec95..6f0f884197f3a 100644 --- a/tests/system/providers/mysql/example_mysql.py +++ b/providers/tests/system/mysql/example_mysql.py @@ -59,7 +59,7 @@ drop_table_mysql_task >> mysql_task -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vertex_ai/__init__.py b/providers/tests/system/neo4j/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/vertex_ai/__init__.py rename to providers/tests/system/neo4j/__init__.py diff --git a/tests/system/providers/neo4j/example_neo4j.py b/providers/tests/system/neo4j/example_neo4j.py similarity index 95% rename from tests/system/providers/neo4j/example_neo4j.py rename to providers/tests/system/neo4j/example_neo4j.py index 0aea16f736dba..80db3fbb2ab1d 100644 --- a/tests/system/providers/neo4j/example_neo4j.py +++ b/providers/tests/system/neo4j/example_neo4j.py @@ -48,7 +48,7 @@ # [END run_query_neo4j_operator] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/video_intelligence/__init__.py b/providers/tests/system/openai/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/video_intelligence/__init__.py rename to providers/tests/system/openai/__init__.py diff --git a/tests/system/providers/openai/example_openai.py b/providers/tests/system/openai/example_openai.py similarity index 97% rename from tests/system/providers/openai/example_openai.py rename to providers/tests/system/openai/example_openai.py index ec8dd316b3ed0..d342207b5d09b 100644 --- a/tests/system/providers/openai/example_openai.py +++ b/providers/tests/system/openai/example_openai.py @@ -104,7 +104,7 @@ def task_to_store_input_text_in_xcom(): example_openai_dag() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/openai/example_trigger_batch_operator.py b/providers/tests/system/openai/example_trigger_batch_operator.py similarity index 97% rename from tests/system/providers/openai/example_trigger_batch_operator.py rename to providers/tests/system/openai/example_trigger_batch_operator.py index 6f01f648ccc7b..3dae1df5d9d0b 100644 --- a/tests/system/providers/openai/example_trigger_batch_operator.py +++ b/providers/tests/system/openai/example_trigger_batch_operator.py @@ -111,7 +111,7 @@ def cleanup_batch_output_file(batch_id, **context): openai_batch_chat_completions() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/vision/__init__.py b/providers/tests/system/opensearch/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/vision/__init__.py rename to providers/tests/system/opensearch/__init__.py diff --git a/tests/system/providers/opensearch/example_opensearch.py b/providers/tests/system/opensearch/example_opensearch.py similarity index 96% rename from tests/system/providers/opensearch/example_opensearch.py rename to providers/tests/system/opensearch/example_opensearch.py index b82af8ebddf16..e339fbe788481 100644 --- a/tests/system/providers/opensearch/example_opensearch.py +++ b/providers/tests/system/opensearch/example_opensearch.py @@ -118,14 +118,14 @@ def load_connections(): chain(create_index, add_document_by_class, add_document_by_args, search_high_level, search_low_level) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/workflows/__init__.py b/providers/tests/system/opsgenie/__init__.py similarity index 100% rename from tests/system/providers/google/cloud/workflows/__init__.py rename to providers/tests/system/opsgenie/__init__.py diff --git a/tests/system/providers/opsgenie/example_opsgenie_alert.py b/providers/tests/system/opsgenie/example_opsgenie_alert.py similarity index 96% rename from tests/system/providers/opsgenie/example_opsgenie_alert.py rename to providers/tests/system/opsgenie/example_opsgenie_alert.py index 30fdabc0d5e95..cf11f824eec8c 100644 --- a/tests/system/providers/opsgenie/example_opsgenie_alert.py +++ b/providers/tests/system/opsgenie/example_opsgenie_alert.py @@ -51,7 +51,7 @@ ) # [END howto_opsgenie_delete_alert_operator] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/opsgenie/example_opsgenie_notifier.py b/providers/tests/system/opsgenie/example_opsgenie_notifier.py similarity index 96% rename from tests/system/providers/opsgenie/example_opsgenie_notifier.py rename to providers/tests/system/opsgenie/example_opsgenie_notifier.py index 10edf8debdaed..97f5945aa9772 100644 --- a/tests/system/providers/opsgenie/example_opsgenie_notifier.py +++ b/providers/tests/system/opsgenie/example_opsgenie_notifier.py @@ -37,7 +37,7 @@ ) # [END howto_notifier_opsgenie] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/providers/teradata/transfers/__init__.py b/providers/tests/system/papermill/__init__.py similarity index 100% rename from tests/providers/teradata/transfers/__init__.py rename to providers/tests/system/papermill/__init__.py diff --git a/tests/system/providers/papermill/conftest.py b/providers/tests/system/papermill/conftest.py similarity index 100% rename from tests/system/providers/papermill/conftest.py rename to providers/tests/system/papermill/conftest.py diff --git a/tests/system/providers/papermill/example_papermill.py b/providers/tests/system/papermill/example_papermill.py similarity index 96% rename from tests/system/providers/papermill/example_papermill.py rename to providers/tests/system/papermill/example_papermill.py index 199fd6f64d289..529647fe4e719 100644 --- a/tests/system/providers/papermill/example_papermill.py +++ b/providers/tests/system/papermill/example_papermill.py @@ -52,7 +52,7 @@ ) # [END howto_operator_papermill] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/papermill/example_papermill_remote_verify.py b/providers/tests/system/papermill/example_papermill_remote_verify.py similarity index 97% rename from tests/system/providers/papermill/example_papermill_remote_verify.py rename to providers/tests/system/papermill/example_papermill_remote_verify.py index b4cf6249be241..f325928b8cd01 100644 --- a/tests/system/providers/papermill/example_papermill_remote_verify.py +++ b/providers/tests/system/papermill/example_papermill_remote_verify.py @@ -75,7 +75,7 @@ def check_notebook(output_notebook, execution_date): ) # [END howto_verify_operator_papermill_remote_kernel] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/papermill/example_papermill_verify.py b/providers/tests/system/papermill/example_papermill_verify.py similarity index 97% rename from tests/system/providers/papermill/example_papermill_verify.py rename to providers/tests/system/papermill/example_papermill_verify.py index af24dbdf8ff77..cfae2cc6ed3e0 100644 --- a/tests/system/providers/papermill/example_papermill_verify.py +++ b/providers/tests/system/papermill/example_papermill_verify.py @@ -73,7 +73,7 @@ def check_notebook(inlets, execution_date): run_this >> check_notebook(inlets=AUTO, execution_date="{{ execution_date }}") # [END howto_verify_operator_papermill] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/papermill/input_notebook.ipynb b/providers/tests/system/papermill/input_notebook.ipynb similarity index 100% rename from tests/system/providers/papermill/input_notebook.ipynb rename to providers/tests/system/papermill/input_notebook.ipynb diff --git a/tests/system/providers/google/datacatalog/__init__.py b/providers/tests/system/pgvector/__init__.py similarity index 100% rename from tests/system/providers/google/datacatalog/__init__.py rename to providers/tests/system/pgvector/__init__.py diff --git a/tests/system/providers/pgvector/example_pgvector.py b/providers/tests/system/pgvector/example_pgvector.py similarity index 97% rename from tests/system/providers/pgvector/example_pgvector.py rename to providers/tests/system/pgvector/example_pgvector.py index 8d1d0f0b3703e..5c16b9da307cb 100644 --- a/tests/system/providers/pgvector/example_pgvector.py +++ b/providers/tests/system/pgvector/example_pgvector.py @@ -76,7 +76,7 @@ def cleanup_postgres_objects(): example_pgvector_dag() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/pgvector/example_pgvector_openai.py b/providers/tests/system/pgvector/example_pgvector_openai.py similarity index 97% rename from tests/system/providers/pgvector/example_pgvector_openai.py rename to providers/tests/system/pgvector/example_pgvector_openai.py index 9b7999436f348..9c8f841e8f933 100644 --- a/tests/system/providers/pgvector/example_pgvector_openai.py +++ b/providers/tests/system/pgvector/example_pgvector_openai.py @@ -90,7 +90,7 @@ def cleanup_postgres_objects(): example_pgvector_dag() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/firebase/__init__.py b/providers/tests/system/pinecone/__init__.py similarity index 100% rename from tests/system/providers/google/firebase/__init__.py rename to providers/tests/system/pinecone/__init__.py diff --git a/tests/system/providers/pinecone/example_create_pod_index.py b/providers/tests/system/pinecone/example_create_pod_index.py similarity index 96% rename from tests/system/providers/pinecone/example_create_pod_index.py rename to providers/tests/system/pinecone/example_create_pod_index.py index a2d7f16c6934e..e0342e2d62008 100644 --- a/tests/system/providers/pinecone/example_create_pod_index.py +++ b/providers/tests/system/pinecone/example_create_pod_index.py @@ -56,7 +56,7 @@ def delete_index(): create_index >> delete_index() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/pinecone/example_create_serverless_index.py b/providers/tests/system/pinecone/example_create_serverless_index.py similarity index 96% rename from tests/system/providers/pinecone/example_create_serverless_index.py rename to providers/tests/system/pinecone/example_create_serverless_index.py index cf1e2c5cee4b5..28078b1b608cd 100644 --- a/tests/system/providers/pinecone/example_create_serverless_index.py +++ b/providers/tests/system/pinecone/example_create_serverless_index.py @@ -55,7 +55,7 @@ def delete_index(): create_index >> delete_index() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/pinecone/example_dag_pinecone.py b/providers/tests/system/pinecone/example_dag_pinecone.py similarity index 95% rename from tests/system/providers/pinecone/example_dag_pinecone.py rename to providers/tests/system/pinecone/example_dag_pinecone.py index 8bf59febeca93..744f6518dceae 100644 --- a/tests/system/providers/pinecone/example_dag_pinecone.py +++ b/providers/tests/system/pinecone/example_dag_pinecone.py @@ -46,7 +46,7 @@ # [END howto_operator_pinecone_ingest] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/pinecone/example_pinecone_cohere.py b/providers/tests/system/pinecone/example_pinecone_cohere.py similarity index 97% rename from tests/system/providers/pinecone/example_pinecone_cohere.py rename to providers/tests/system/pinecone/example_pinecone_cohere.py index 80e6766484d6b..80e9b1efad8eb 100644 --- a/tests/system/providers/pinecone/example_pinecone_cohere.py +++ b/providers/tests/system/pinecone/example_pinecone_cohere.py @@ -76,7 +76,7 @@ def delete_index(): create_index() >> embed_task >> transformed_output >> perform_ingestion >> delete_index() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/pinecone/example_pinecone_openai.py b/providers/tests/system/pinecone/example_pinecone_openai.py similarity index 98% rename from tests/system/providers/pinecone/example_pinecone_openai.py rename to providers/tests/system/pinecone/example_pinecone_openai.py index d338e25542ce0..dad83f48085a5 100644 --- a/tests/system/providers/pinecone/example_pinecone_openai.py +++ b/providers/tests/system/pinecone/example_pinecone_openai.py @@ -107,7 +107,7 @@ def delete_index(): create_index >> embed_task >> perform_ingestion >> delete_index() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/suite/__init__.py b/providers/tests/system/postgres/__init__.py similarity index 100% rename from tests/system/providers/google/suite/__init__.py rename to providers/tests/system/postgres/__init__.py diff --git a/tests/system/providers/postgres/example_postgres.py b/providers/tests/system/postgres/example_postgres.py similarity index 96% rename from tests/system/providers/postgres/example_postgres.py rename to providers/tests/system/postgres/example_postgres.py index aab5c1951799b..5ffaf6ea0098a 100644 --- a/tests/system/providers/postgres/example_postgres.py +++ b/providers/tests/system/postgres/example_postgres.py @@ -80,13 +80,13 @@ create_pet_table >> populate_pet_table >> get_all_pets >> get_birth_date # [END postgres_sql_execute_query_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/suite/resources/__init__.py b/providers/tests/system/presto/__init__.py similarity index 100% rename from tests/system/providers/google/suite/resources/__init__.py rename to providers/tests/system/presto/__init__.py diff --git a/tests/system/providers/presto/example_gcs_to_presto.py b/providers/tests/system/presto/example_gcs_to_presto.py similarity index 95% rename from tests/system/providers/presto/example_gcs_to_presto.py rename to providers/tests/system/presto/example_gcs_to_presto.py index 19b4a3a4fba5e..ceccf098471dc 100644 --- a/tests/system/providers/presto/example_gcs_to_presto.py +++ b/providers/tests/system/presto/example_gcs_to_presto.py @@ -50,7 +50,7 @@ # [END gcs_csv_to_presto_table] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/google/workplace/__init__.py b/providers/tests/system/qdrant/__init__.py similarity index 100% rename from tests/system/providers/google/workplace/__init__.py rename to providers/tests/system/qdrant/__init__.py diff --git a/tests/system/providers/qdrant/example_dag_qdrant.py b/providers/tests/system/qdrant/example_dag_qdrant.py similarity index 96% rename from tests/system/providers/qdrant/example_dag_qdrant.py rename to providers/tests/system/qdrant/example_dag_qdrant.py index 8f55d2d72c559..85c92d9c99312 100644 --- a/tests/system/providers/qdrant/example_dag_qdrant.py +++ b/providers/tests/system/qdrant/example_dag_qdrant.py @@ -43,7 +43,7 @@ # [END howto_operator_qdrant_ingest] -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/http/__init__.py b/providers/tests/system/redis/__init__.py similarity index 100% rename from tests/system/providers/http/__init__.py rename to providers/tests/system/redis/__init__.py diff --git a/tests/system/providers/redis/example_redis_publish.py b/providers/tests/system/redis/example_redis_publish.py similarity index 95% rename from tests/system/providers/redis/example_redis_publish.py rename to providers/tests/system/redis/example_redis_publish.py index 9d50593c04003..335256f68e57d 100644 --- a/tests/system/providers/redis/example_redis_publish.py +++ b/providers/tests/system/redis/example_redis_publish.py @@ -73,13 +73,13 @@ publish_task >> key_sensor_task - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/influxdb/__init__.py b/providers/tests/system/salesforce/__init__.py similarity index 100% rename from tests/system/providers/influxdb/__init__.py rename to providers/tests/system/salesforce/__init__.py diff --git a/tests/system/providers/salesforce/example_bulk.py b/providers/tests/system/salesforce/example_bulk.py similarity index 97% rename from tests/system/providers/salesforce/example_bulk.py rename to providers/tests/system/salesforce/example_bulk.py index 06764aaf54b42..7d43de648805d 100644 --- a/tests/system/providers/salesforce/example_bulk.py +++ b/providers/tests/system/salesforce/example_bulk.py @@ -90,7 +90,7 @@ # [END howto_salesforce_bulk_delete_operation] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/salesforce/example_salesforce_apex_rest.py b/providers/tests/system/salesforce/example_salesforce_apex_rest.py similarity index 95% rename from tests/system/providers/salesforce/example_salesforce_apex_rest.py rename to providers/tests/system/salesforce/example_salesforce_apex_rest.py index 9bfdb94839621..2961ffa4e4784 100644 --- a/tests/system/providers/salesforce/example_salesforce_apex_rest.py +++ b/providers/tests/system/salesforce/example_salesforce_apex_rest.py @@ -41,7 +41,7 @@ # [END howto_salesforce_apex_rest_operator] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/jdbc/__init__.py b/providers/tests/system/samba/__init__.py similarity index 100% rename from tests/system/providers/jdbc/__init__.py rename to providers/tests/system/samba/__init__.py diff --git a/tests/system/providers/samba/example_gcs_to_samba.py b/providers/tests/system/samba/example_gcs_to_samba.py similarity index 96% rename from tests/system/providers/samba/example_gcs_to_samba.py rename to providers/tests/system/samba/example_gcs_to_samba.py index c03a6f129ceae..d7717fcbfc727 100644 --- a/tests/system/providers/samba/example_gcs_to_samba.py +++ b/providers/tests/system/samba/example_gcs_to_samba.py @@ -30,7 +30,8 @@ from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.providers.samba.transfers.gcs_to_samba import GCSToSambaOperator from airflow.utils.trigger_rule import TriggerRule -from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID + +from providers.tests.system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID @@ -137,14 +138,14 @@ >> delete_bucket ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/jenkins/__init__.py b/providers/tests/system/sftp/__init__.py similarity index 100% rename from tests/system/providers/jenkins/__init__.py rename to providers/tests/system/sftp/__init__.py diff --git a/tests/system/providers/sftp/example_sftp_sensor.py b/providers/tests/system/sftp/example_sftp_sensor.py similarity index 96% rename from tests/system/providers/sftp/example_sftp_sensor.py rename to providers/tests/system/sftp/example_sftp_sensor.py index df705b4b871f9..a3bde8a03af0a 100644 --- a/tests/system/providers/sftp/example_sftp_sensor.py +++ b/providers/tests/system/sftp/example_sftp_sensor.py @@ -89,14 +89,14 @@ def sftp_sensor_decorator(): >> remove_file_task_end ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/mssql/__init__.py b/providers/tests/system/singularity/__init__.py similarity index 100% rename from tests/system/providers/microsoft/mssql/__init__.py rename to providers/tests/system/singularity/__init__.py diff --git a/tests/system/providers/singularity/example_singularity.py b/providers/tests/system/singularity/example_singularity.py similarity index 95% rename from tests/system/providers/singularity/example_singularity.py rename to providers/tests/system/singularity/example_singularity.py index 4b60c080dcd2c..e80a0f40eb955 100644 --- a/tests/system/providers/singularity/example_singularity.py +++ b/providers/tests/system/singularity/example_singularity.py @@ -50,7 +50,7 @@ t3 >> t4 -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/winrm/__init__.py b/providers/tests/system/slack/__init__.py similarity index 100% rename from tests/system/providers/microsoft/winrm/__init__.py rename to providers/tests/system/slack/__init__.py diff --git a/tests/system/providers/slack/example_slack.py b/providers/tests/system/slack/example_slack.py similarity index 97% rename from tests/system/providers/slack/example_slack.py rename to providers/tests/system/slack/example_slack.py index c20cf8afa8150..f122dffca2782 100644 --- a/tests/system/providers/slack/example_slack.py +++ b/providers/tests/system/slack/example_slack.py @@ -94,7 +94,7 @@ ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/slack/example_slack_webhook.py b/providers/tests/system/slack/example_slack_webhook.py similarity index 97% rename from tests/system/providers/slack/example_slack_webhook.py rename to providers/tests/system/slack/example_slack_webhook.py index 98905d1b9ee67..7e45dc92d921f 100644 --- a/tests/system/providers/slack/example_slack_webhook.py +++ b/providers/tests/system/slack/example_slack_webhook.py @@ -69,7 +69,7 @@ slack_webhook_operator_text >> slack_webhook_operator_blocks -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/slack/example_sql_to_slack.py b/providers/tests/system/slack/example_sql_to_slack.py similarity index 96% rename from tests/system/providers/slack/example_sql_to_slack.py rename to providers/tests/system/slack/example_sql_to_slack.py index 2e672df9aa686..457bcecaea671 100644 --- a/tests/system/providers/slack/example_sql_to_slack.py +++ b/providers/tests/system/slack/example_sql_to_slack.py @@ -52,7 +52,7 @@ # [END howto_operator_sql_to_slack_api_file] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/slack/example_sql_to_slack_webhook.py b/providers/tests/system/slack/example_sql_to_slack_webhook.py similarity index 95% rename from tests/system/providers/slack/example_sql_to_slack_webhook.py rename to providers/tests/system/slack/example_sql_to_slack_webhook.py index 97bb8ae46ed2d..44fad156657be 100644 --- a/tests/system/providers/slack/example_sql_to_slack_webhook.py +++ b/providers/tests/system/slack/example_sql_to_slack_webhook.py @@ -51,7 +51,7 @@ # [END howto_operator_sql_to_slack_webhook] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/mysql/__init__.py b/providers/tests/system/snowflake/__init__.py similarity index 100% rename from tests/system/providers/mysql/__init__.py rename to providers/tests/system/snowflake/__init__.py diff --git a/tests/system/providers/snowflake/example_copy_into_snowflake.py b/providers/tests/system/snowflake/example_copy_into_snowflake.py similarity index 96% rename from tests/system/providers/snowflake/example_copy_into_snowflake.py rename to providers/tests/system/snowflake/example_copy_into_snowflake.py index e25112b95f3e4..32eb9699d74b6 100644 --- a/tests/system/providers/snowflake/example_copy_into_snowflake.py +++ b/providers/tests/system/snowflake/example_copy_into_snowflake.py @@ -60,7 +60,7 @@ # [END howto_operator_s3_copy_into_snowflake] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/snowflake/example_snowflake.py b/providers/tests/system/snowflake/example_snowflake.py similarity index 97% rename from tests/system/providers/snowflake/example_snowflake.py rename to providers/tests/system/snowflake/example_snowflake.py index 1a6f12080d74e..e851b13c08200 100644 --- a/tests/system/providers/snowflake/example_snowflake.py +++ b/providers/tests/system/snowflake/example_snowflake.py @@ -96,7 +96,7 @@ ) -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/snowflake/example_snowflake_snowflake_op_template_file.sql b/providers/tests/system/snowflake/example_snowflake_snowflake_op_template_file.sql similarity index 100% rename from tests/system/providers/snowflake/example_snowflake_snowflake_op_template_file.sql rename to providers/tests/system/snowflake/example_snowflake_snowflake_op_template_file.sql diff --git a/tests/system/providers/snowflake/example_snowpark_decorator.py b/providers/tests/system/snowflake/example_snowpark_decorator.py similarity index 97% rename from tests/system/providers/snowflake/example_snowpark_decorator.py rename to providers/tests/system/snowflake/example_snowpark_decorator.py index 1a303b1fdfa5f..3f74b97d0f23e 100644 --- a/tests/system/providers/snowflake/example_snowpark_decorator.py +++ b/providers/tests/system/snowflake/example_snowpark_decorator.py @@ -79,7 +79,8 @@ def check_num_rows(table_name: str): check_num_rows(table_name) # [END howto_decorator_snowpark] -from tests.system.utils import get_test_run # noqa: E402 + +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/snowflake/example_snowpark_operator.py b/providers/tests/system/snowflake/example_snowpark_operator.py similarity index 97% rename from tests/system/providers/snowflake/example_snowpark_operator.py rename to providers/tests/system/snowflake/example_snowpark_operator.py index 090a0f53a4bde..0543596ec9b1f 100644 --- a/tests/system/providers/snowflake/example_snowpark_operator.py +++ b/providers/tests/system/snowflake/example_snowpark_operator.py @@ -88,7 +88,8 @@ def check_num_rows(table_name: str): setup_data_operator >> check_num_rows_operator # [END howto_operator_snowpark] -from tests.system.utils import get_test_run # noqa: E402 + +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/neo4j/__init__.py b/providers/tests/system/sqlite/__init__.py similarity index 100% rename from tests/system/providers/neo4j/__init__.py rename to providers/tests/system/sqlite/__init__.py diff --git a/tests/system/providers/sqlite/create_table.sql b/providers/tests/system/sqlite/create_table.sql similarity index 100% rename from tests/system/providers/sqlite/create_table.sql rename to providers/tests/system/sqlite/create_table.sql diff --git a/tests/system/providers/sqlite/example_sqlite.py b/providers/tests/system/sqlite/example_sqlite.py similarity index 96% rename from tests/system/providers/sqlite/example_sqlite.py rename to providers/tests/system/sqlite/example_sqlite.py index 317da9af37808..1ea0270a0df27 100644 --- a/tests/system/providers/sqlite/example_sqlite.py +++ b/providers/tests/system/sqlite/example_sqlite.py @@ -91,13 +91,13 @@ def replace_sqlite_hook(): >> replace_sqlite_hook() ) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/openai/__init__.py b/providers/tests/system/tableau/__init__.py similarity index 100% rename from tests/system/providers/openai/__init__.py rename to providers/tests/system/tableau/__init__.py diff --git a/tests/system/providers/tableau/example_tableau.py b/providers/tests/system/tableau/example_tableau.py similarity index 97% rename from tests/system/providers/tableau/example_tableau.py rename to providers/tests/system/tableau/example_tableau.py index c5f6995b62fe0..1472511a66c95 100644 --- a/tests/system/providers/tableau/example_tableau.py +++ b/providers/tests/system/tableau/example_tableau.py @@ -71,7 +71,7 @@ # task_refresh_workbook_non_blocking >> task_check_job_status -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/opensearch/__init__.py b/providers/tests/system/telegram/__init__.py similarity index 100% rename from tests/system/providers/opensearch/__init__.py rename to providers/tests/system/telegram/__init__.py diff --git a/tests/system/providers/telegram/example_telegram.py b/providers/tests/system/telegram/example_telegram.py similarity index 95% rename from tests/system/providers/telegram/example_telegram.py rename to providers/tests/system/telegram/example_telegram.py index 18d734f3c69e8..a7e9018cf85f1 100644 --- a/tests/system/providers/telegram/example_telegram.py +++ b/providers/tests/system/telegram/example_telegram.py @@ -45,7 +45,7 @@ # [END howto_operator_telegram] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/opsgenie/__init__.py b/providers/tests/system/teradata/__init__.py similarity index 100% rename from tests/system/providers/opsgenie/__init__.py rename to providers/tests/system/teradata/__init__.py diff --git a/tests/system/providers/teradata/create_ssl_table.sql b/providers/tests/system/teradata/create_ssl_table.sql similarity index 100% rename from tests/system/providers/teradata/create_ssl_table.sql rename to providers/tests/system/teradata/create_ssl_table.sql diff --git a/tests/system/providers/teradata/create_table.sql b/providers/tests/system/teradata/create_table.sql similarity index 100% rename from tests/system/providers/teradata/create_table.sql rename to providers/tests/system/teradata/create_table.sql diff --git a/tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py b/providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py similarity index 98% rename from tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py rename to providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py index bcb1dd2fe6c5b..26fd27554d5d1 100644 --- a/tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py +++ b/providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py @@ -205,13 +205,13 @@ ) # [END azure_blob_to_teradata_transfer_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/teradata/example_s3_to_teradata_transfer.py b/providers/tests/system/teradata/example_s3_to_teradata_transfer.py similarity index 98% rename from tests/system/providers/teradata/example_s3_to_teradata_transfer.py rename to providers/tests/system/teradata/example_s3_to_teradata_transfer.py index ae8b827c1efae..ab361990b21e4 100644 --- a/tests/system/providers/teradata/example_s3_to_teradata_transfer.py +++ b/providers/tests/system/teradata/example_s3_to_teradata_transfer.py @@ -206,13 +206,13 @@ ) # [END s3_to_teradata_transfer_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/teradata/example_ssl_teradata.py b/providers/tests/system/teradata/example_ssl_teradata.py similarity index 97% rename from tests/system/providers/teradata/example_ssl_teradata.py rename to providers/tests/system/teradata/example_ssl_teradata.py index 1673bd791a79b..3836715a8e634 100644 --- a/tests/system/providers/teradata/example_ssl_teradata.py +++ b/providers/tests/system/teradata/example_ssl_teradata.py @@ -121,13 +121,13 @@ # [END teradata_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/teradata/example_teradata.py b/providers/tests/system/teradata/example_teradata.py similarity index 97% rename from tests/system/providers/teradata/example_teradata.py rename to providers/tests/system/teradata/example_teradata.py index 1fd587cdf8f71..ae41effce7fec 100644 --- a/tests/system/providers/teradata/example_teradata.py +++ b/providers/tests/system/teradata/example_teradata.py @@ -164,13 +164,13 @@ # [END teradata_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/teradata/example_teradata_call_sp.py b/providers/tests/system/teradata/example_teradata_call_sp.py similarity index 97% rename from tests/system/providers/teradata/example_teradata_call_sp.py rename to providers/tests/system/teradata/example_teradata_call_sp.py index 98ce85fffdfdc..f48583d83ec55 100644 --- a/tests/system/providers/teradata/example_teradata_call_sp.py +++ b/providers/tests/system/teradata/example_teradata_call_sp.py @@ -162,13 +162,13 @@ # [END howto_teradata_operator_for_sp] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/teradata/example_teradata_compute_cluster.py b/providers/tests/system/teradata/example_teradata_compute_cluster.py similarity index 97% rename from tests/system/providers/teradata/example_teradata_compute_cluster.py rename to providers/tests/system/teradata/example_teradata_compute_cluster.py index 3fefe9858770a..210cde7816bc2 100644 --- a/tests/system/providers/teradata/example_teradata_compute_cluster.py +++ b/providers/tests/system/teradata/example_teradata_compute_cluster.py @@ -146,13 +146,13 @@ # [END teradata_vantage_lake_compute_cluster_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/teradata/example_teradata_to_teradata_transfer.py b/providers/tests/system/teradata/example_teradata_to_teradata_transfer.py similarity index 97% rename from tests/system/providers/teradata/example_teradata_to_teradata_transfer.py rename to providers/tests/system/teradata/example_teradata_to_teradata_transfer.py index ac2517a33f519..7f315202606fc 100644 --- a/tests/system/providers/teradata/example_teradata_to_teradata_transfer.py +++ b/providers/tests/system/teradata/example_teradata_to_teradata_transfer.py @@ -148,13 +148,13 @@ ) # [END teradata_to_teradata_transfer_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/pgvector/__init__.py b/providers/tests/system/trino/__init__.py similarity index 100% rename from tests/system/providers/pgvector/__init__.py rename to providers/tests/system/trino/__init__.py diff --git a/tests/system/providers/trino/example_gcs_to_trino.py b/providers/tests/system/trino/example_gcs_to_trino.py similarity index 95% rename from tests/system/providers/trino/example_gcs_to_trino.py rename to providers/tests/system/trino/example_gcs_to_trino.py index e970e4d146718..e0249101eb1a4 100644 --- a/tests/system/providers/trino/example_gcs_to_trino.py +++ b/providers/tests/system/trino/example_gcs_to_trino.py @@ -50,7 +50,7 @@ # [END gcs_csv_to_trino_table] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/trino/example_trino.py b/providers/tests/system/trino/example_trino.py similarity index 97% rename from tests/system/providers/trino/example_trino.py rename to providers/tests/system/trino/example_trino.py index 91e1c9dfce6ad..d708d9b7b8fb7 100644 --- a/tests/system/providers/trino/example_trino.py +++ b/providers/tests/system/trino/example_trino.py @@ -91,7 +91,7 @@ # [END howto_operator_trino] -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/pinecone/__init__.py b/providers/tests/system/weaviate/__init__.py similarity index 100% rename from tests/system/providers/pinecone/__init__.py rename to providers/tests/system/weaviate/__init__.py diff --git a/tests/system/providers/weaviate/example_weaviate_cohere.py b/providers/tests/system/weaviate/example_weaviate_cohere.py similarity index 98% rename from tests/system/providers/weaviate/example_weaviate_cohere.py rename to providers/tests/system/weaviate/example_weaviate_cohere.py index d8662e464bd4b..1db5e2cccb72a 100644 --- a/tests/system/providers/weaviate/example_weaviate_cohere.py +++ b/providers/tests/system/weaviate/example_weaviate_cohere.py @@ -115,7 +115,7 @@ def delete_weaviate_collections(): example_weaviate_cohere() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/weaviate/example_weaviate_dynamic_mapping_dag.py b/providers/tests/system/weaviate/example_weaviate_dynamic_mapping_dag.py similarity index 97% rename from tests/system/providers/weaviate/example_weaviate_dynamic_mapping_dag.py rename to providers/tests/system/weaviate/example_weaviate_dynamic_mapping_dag.py index 5f998cc52e862..38c0168a9f615 100644 --- a/tests/system/providers/weaviate/example_weaviate_dynamic_mapping_dag.py +++ b/providers/tests/system/weaviate/example_weaviate_dynamic_mapping_dag.py @@ -92,7 +92,7 @@ def delete_weaviate_collection(collection_name): example_weaviate_dynamic_mapping_dag() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/weaviate/example_weaviate_openai.py b/providers/tests/system/weaviate/example_weaviate_openai.py similarity index 98% rename from tests/system/providers/weaviate/example_weaviate_openai.py rename to providers/tests/system/weaviate/example_weaviate_openai.py index adf20fd929f1b..edad9058f8738 100644 --- a/tests/system/providers/weaviate/example_weaviate_openai.py +++ b/providers/tests/system/weaviate/example_weaviate_openai.py @@ -123,7 +123,7 @@ def delete_weaviate_collection(): example_weaviate_openai() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/weaviate/example_weaviate_operator.py b/providers/tests/system/weaviate/example_weaviate_operator.py similarity index 99% rename from tests/system/providers/weaviate/example_weaviate_operator.py rename to providers/tests/system/weaviate/example_weaviate_operator.py index 081f2ef3903d7..309864ad64924 100644 --- a/tests/system/providers/weaviate/example_weaviate_operator.py +++ b/providers/tests/system/weaviate/example_weaviate_operator.py @@ -297,7 +297,7 @@ def delete_weaviate_docs_collection_without_vector(): example_weaviate_using_operator() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/weaviate/example_weaviate_using_hook.py b/providers/tests/system/weaviate/example_weaviate_using_hook.py similarity index 98% rename from tests/system/providers/weaviate/example_weaviate_using_hook.py rename to providers/tests/system/weaviate/example_weaviate_using_hook.py index 8ad7356465a48..601a222f94d4e 100644 --- a/tests/system/providers/weaviate/example_weaviate_using_hook.py +++ b/providers/tests/system/weaviate/example_weaviate_using_hook.py @@ -140,7 +140,7 @@ def delete_weaviate_collection_without_vector(): example_weaviate_dag_using_hook() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/weaviate/example_weaviate_vectorizer_dag.py b/providers/tests/system/weaviate/example_weaviate_vectorizer_dag.py similarity index 97% rename from tests/system/providers/weaviate/example_weaviate_vectorizer_dag.py rename to providers/tests/system/weaviate/example_weaviate_vectorizer_dag.py index 546148ad70ed2..f0306177bffd4 100644 --- a/tests/system/providers/weaviate/example_weaviate_vectorizer_dag.py +++ b/providers/tests/system/weaviate/example_weaviate_vectorizer_dag.py @@ -99,7 +99,7 @@ def delete_weaviate_collection(): example_weaviate_vectorizer_dag() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/weaviate/example_weaviate_without_vectorizer_dag.py b/providers/tests/system/weaviate/example_weaviate_without_vectorizer_dag.py similarity index 98% rename from tests/system/providers/weaviate/example_weaviate_without_vectorizer_dag.py rename to providers/tests/system/weaviate/example_weaviate_without_vectorizer_dag.py index c4062508e26d1..e2cf9e560e785 100644 --- a/tests/system/providers/weaviate/example_weaviate_without_vectorizer_dag.py +++ b/providers/tests/system/weaviate/example_weaviate_without_vectorizer_dag.py @@ -111,7 +111,7 @@ def delete_weaviate_collection(): example_weaviate_without_vectorizer_dag() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/weaviate/jeopardy_data_with_vectors.json b/providers/tests/system/weaviate/jeopardy_data_with_vectors.json similarity index 100% rename from tests/system/providers/weaviate/jeopardy_data_with_vectors.json rename to providers/tests/system/weaviate/jeopardy_data_with_vectors.json diff --git a/tests/system/providers/weaviate/jeopardy_data_without_vectors.json b/providers/tests/system/weaviate/jeopardy_data_without_vectors.json similarity index 100% rename from tests/system/providers/weaviate/jeopardy_data_without_vectors.json rename to providers/tests/system/weaviate/jeopardy_data_without_vectors.json diff --git a/tests/system/providers/weaviate/jeopardy_doc_data_without_vectors.json b/providers/tests/system/weaviate/jeopardy_doc_data_without_vectors.json similarity index 100% rename from tests/system/providers/weaviate/jeopardy_doc_data_without_vectors.json rename to providers/tests/system/weaviate/jeopardy_doc_data_without_vectors.json diff --git a/tests/system/providers/postgres/__init__.py b/providers/tests/system/yandex/__init__.py similarity index 100% rename from tests/system/providers/postgres/__init__.py rename to providers/tests/system/yandex/__init__.py diff --git a/tests/system/providers/yandex/example_yandexcloud.py b/providers/tests/system/yandex/example_yandexcloud.py similarity index 97% rename from tests/system/providers/yandex/example_yandexcloud.py rename to providers/tests/system/yandex/example_yandexcloud.py index ddebc46a3b50c..ab72adc14c051 100644 --- a/tests/system/providers/yandex/example_yandexcloud.py +++ b/providers/tests/system/yandex/example_yandexcloud.py @@ -32,7 +32,8 @@ from airflow import DAG from airflow.decorators import task from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook -from tests.system.utils import get_test_env_id + +from dev.tests_common.test_utils.system_tests import get_test_env_id ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_hook" @@ -196,14 +197,14 @@ def delete_cluster( spark_job >> delete_task - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/yandex/example_yandexcloud_dataproc.py b/providers/tests/system/yandex/example_yandexcloud_dataproc.py similarity index 96% rename from tests/system/providers/yandex/example_yandexcloud_dataproc.py rename to providers/tests/system/yandex/example_yandexcloud_dataproc.py index 7ff4aa541dd54..d97e3d592ef88 100644 --- a/tests/system/providers/yandex/example_yandexcloud_dataproc.py +++ b/providers/tests/system/yandex/example_yandexcloud_dataproc.py @@ -31,7 +31,8 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule -from tests.system.utils import get_test_env_id + +from dev.tests_common.test_utils.system_tests import get_test_env_id # should be filled with appropriate ids @@ -162,13 +163,13 @@ create_cluster >> create_mapreduce_job >> create_hive_query >> create_hive_query_from_file create_hive_query_from_file >> create_spark_job >> create_pyspark_job >> delete_cluster - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py b/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py similarity index 92% rename from tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py rename to providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py index 475bc789ec2b8..38401a1cdcd39 100644 --- a/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py +++ b/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py @@ -27,7 +27,8 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule -from tests.system.utils import get_test_env_id + +from dev.tests_common.test_utils.system_tests import get_test_env_id # should be filled with appropriate ids @@ -70,13 +71,13 @@ ) create_spark_job >> delete_cluster - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/yandex/example_yandexcloud_yq.py b/providers/tests/system/yandex/example_yandexcloud_yq.py similarity index 88% rename from tests/system/providers/yandex/example_yandexcloud_yq.py rename to providers/tests/system/yandex/example_yandexcloud_yq.py index 0ebef685e24b7..02478825ed5e0 100644 --- a/tests/system/providers/yandex/example_yandexcloud_yq.py +++ b/providers/tests/system/yandex/example_yandexcloud_yq.py @@ -21,7 +21,8 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator from airflow.providers.yandex.operators.yq import YQExecuteQueryOperator -from tests.system.utils import get_test_env_id + +from dev.tests_common.test_utils.system_tests import get_test_env_id ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_yq" @@ -39,13 +40,13 @@ yq_operator = YQExecuteQueryOperator(task_id="sample_query", sql="select 33 as d, 44 as t") yq_operator >> run_this_last - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/presto/__init__.py b/providers/tests/system/ydb/__init__.py similarity index 100% rename from tests/system/providers/presto/__init__.py rename to providers/tests/system/ydb/__init__.py diff --git a/tests/system/providers/ydb/example_ydb.py b/providers/tests/system/ydb/example_ydb.py similarity index 97% rename from tests/system/providers/ydb/example_ydb.py rename to providers/tests/system/ydb/example_ydb.py index 39156328f241b..d7293cb0d1b56 100644 --- a/tests/system/providers/ydb/example_ydb.py +++ b/providers/tests/system/ydb/example_ydb.py @@ -119,13 +119,13 @@ def populate_pet_table_via_bulk_upsert(): ) # [END ydb_operator_howto_guide] - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/qdrant/__init__.py b/providers/tests/system/zendesk/__init__.py similarity index 100% rename from tests/system/providers/qdrant/__init__.py rename to providers/tests/system/zendesk/__init__.py diff --git a/tests/system/providers/zendesk/example_zendesk_custom_get.py b/providers/tests/system/zendesk/example_zendesk_custom_get.py similarity index 95% rename from tests/system/providers/zendesk/example_zendesk_custom_get.py rename to providers/tests/system/zendesk/example_zendesk_custom_get.py index 701a109fd1159..c38d10a26abd3 100644 --- a/tests/system/providers/zendesk/example_zendesk_custom_get.py +++ b/providers/tests/system/zendesk/example_zendesk_custom_get.py @@ -45,7 +45,7 @@ def fetch_organizations() -> list[dict]: fetch_organizations() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/redis/__init__.py b/providers/tests/tableau/__init__.py similarity index 100% rename from tests/system/providers/redis/__init__.py rename to providers/tests/tableau/__init__.py diff --git a/tests/providers/teradata/triggers/__init__.py b/providers/tests/tableau/hooks/__init__.py similarity index 100% rename from tests/providers/teradata/triggers/__init__.py rename to providers/tests/tableau/hooks/__init__.py diff --git a/tests/providers/tableau/hooks/test_tableau.py b/providers/tests/tableau/hooks/test_tableau.py similarity index 95% rename from tests/providers/tableau/hooks/test_tableau.py rename to providers/tests/tableau/hooks/test_tableau.py index 8d10ce2fc368c..23d5e0188f609 100644 --- a/tests/providers/tableau/hooks/test_tableau.py +++ b/providers/tests/tableau/hooks/test_tableau.py @@ -115,10 +115,13 @@ def test_get_conn_auth_via_token_and_site_in_init(self, mock_server, mock_tablea """ Test get conn auth via token """ - with pytest.warns( - AirflowProviderDeprecationWarning, - match="Authentication via personal access token is deprecated..*", - ), TableauHook(site_id="test", tableau_conn_id="tableau_test_token") as tableau_hook: + with ( + pytest.warns( + AirflowProviderDeprecationWarning, + match="Authentication via personal access token is deprecated..*", + ), + TableauHook(site_id="test", tableau_conn_id="tableau_test_token") as tableau_hook, + ): mock_server.assert_called_once_with(tableau_hook.conn.host) mock_tableau_auth.assert_called_once_with( token_name=tableau_hook.conn.extra_dejson["token_name"], @@ -158,10 +161,13 @@ def test_get_conn_ssl_default(self, mock_server, mock_tableau_auth): """ Test get conn with default SSL parameters """ - with pytest.warns( - AirflowProviderDeprecationWarning, - match="Authentication via personal access token is deprecated..*", - ), TableauHook(tableau_conn_id="tableau_test_ssl_connection_default") as tableau_hook: + with ( + pytest.warns( + AirflowProviderDeprecationWarning, + match="Authentication via personal access token is deprecated..*", + ), + TableauHook(tableau_conn_id="tableau_test_ssl_connection_default") as tableau_hook, + ): mock_server.assert_called_once_with(tableau_hook.conn.host) mock_server.return_value.add_http_options.assert_called_once_with( options_dict={"verify": True, "cert": None} diff --git a/tests/system/providers/salesforce/__init__.py b/providers/tests/tableau/operators/__init__.py similarity index 100% rename from tests/system/providers/salesforce/__init__.py rename to providers/tests/tableau/operators/__init__.py diff --git a/tests/providers/tableau/operators/test_tableau.py b/providers/tests/tableau/operators/test_tableau.py similarity index 100% rename from tests/providers/tableau/operators/test_tableau.py rename to providers/tests/tableau/operators/test_tableau.py diff --git a/tests/system/providers/samba/__init__.py b/providers/tests/tableau/sensors/__init__.py similarity index 100% rename from tests/system/providers/samba/__init__.py rename to providers/tests/tableau/sensors/__init__.py diff --git a/tests/providers/tableau/sensors/test_tableau.py b/providers/tests/tableau/sensors/test_tableau.py similarity index 100% rename from tests/providers/tableau/sensors/test_tableau.py rename to providers/tests/tableau/sensors/test_tableau.py diff --git a/tests/providers/teradata/utils/__init__.py b/providers/tests/telegram/__init__.py similarity index 100% rename from tests/providers/teradata/utils/__init__.py rename to providers/tests/telegram/__init__.py diff --git a/tests/providers/trino/__init__.py b/providers/tests/telegram/hooks/__init__.py similarity index 100% rename from tests/providers/trino/__init__.py rename to providers/tests/telegram/hooks/__init__.py diff --git a/tests/providers/telegram/hooks/test_telegram.py b/providers/tests/telegram/hooks/test_telegram.py similarity index 100% rename from tests/providers/telegram/hooks/test_telegram.py rename to providers/tests/telegram/hooks/test_telegram.py diff --git a/tests/providers/trino/hooks/__init__.py b/providers/tests/telegram/operators/__init__.py similarity index 100% rename from tests/providers/trino/hooks/__init__.py rename to providers/tests/telegram/operators/__init__.py diff --git a/tests/providers/telegram/operators/test_telegram.py b/providers/tests/telegram/operators/test_telegram.py similarity index 100% rename from tests/providers/telegram/operators/test_telegram.py rename to providers/tests/telegram/operators/test_telegram.py diff --git a/tests/system/providers/sftp/__init__.py b/providers/tests/teradata/__init__.py similarity index 100% rename from tests/system/providers/sftp/__init__.py rename to providers/tests/teradata/__init__.py diff --git a/tests/providers/trino/transfers/__init__.py b/providers/tests/teradata/hooks/__init__.py similarity index 100% rename from tests/providers/trino/transfers/__init__.py rename to providers/tests/teradata/hooks/__init__.py diff --git a/tests/providers/teradata/hooks/test_teradata.py b/providers/tests/teradata/hooks/test_teradata.py similarity index 98% rename from tests/providers/teradata/hooks/test_teradata.py rename to providers/tests/teradata/hooks/test_teradata.py index 75367e4d406eb..21a19308b2da7 100644 --- a/tests/providers/teradata/hooks/test_teradata.py +++ b/providers/tests/teradata/hooks/test_teradata.py @@ -275,9 +275,12 @@ def test_bulk_insert_rows_without_fields(self): def test_bulk_insert_rows_no_rows(self): rows = [] - with pytest.raises(ValueError), pytest.warns( - AirflowProviderDeprecationWarning, - match="bulk_insert_rows is deprecated. Please use the insert_rows method instead.", + with ( + pytest.raises(ValueError), + pytest.warns( + AirflowProviderDeprecationWarning, + match="bulk_insert_rows is deprecated. Please use the insert_rows method instead.", + ), ): self.test_db_hook.bulk_insert_rows("table", rows) diff --git a/tests/providers/vertica/__init__.py b/providers/tests/teradata/operators/__init__.py similarity index 100% rename from tests/providers/vertica/__init__.py rename to providers/tests/teradata/operators/__init__.py diff --git a/tests/providers/teradata/operators/test_teradata.py b/providers/tests/teradata/operators/test_teradata.py similarity index 100% rename from tests/providers/teradata/operators/test_teradata.py rename to providers/tests/teradata/operators/test_teradata.py diff --git a/tests/providers/teradata/operators/test_teradata_compute_cluster.py b/providers/tests/teradata/operators/test_teradata_compute_cluster.py similarity index 100% rename from tests/providers/teradata/operators/test_teradata_compute_cluster.py rename to providers/tests/teradata/operators/test_teradata_compute_cluster.py diff --git a/tests/providers/vertica/hooks/__init__.py b/providers/tests/teradata/transfers/__init__.py similarity index 100% rename from tests/providers/vertica/hooks/__init__.py rename to providers/tests/teradata/transfers/__init__.py diff --git a/tests/providers/teradata/transfers/test_azure_blob_to_teradata.py b/providers/tests/teradata/transfers/test_azure_blob_to_teradata.py similarity index 100% rename from tests/providers/teradata/transfers/test_azure_blob_to_teradata.py rename to providers/tests/teradata/transfers/test_azure_blob_to_teradata.py diff --git a/tests/providers/teradata/transfers/test_s3_to_teradata.py b/providers/tests/teradata/transfers/test_s3_to_teradata.py similarity index 100% rename from tests/providers/teradata/transfers/test_s3_to_teradata.py rename to providers/tests/teradata/transfers/test_s3_to_teradata.py diff --git a/tests/providers/teradata/transfers/test_teradata_to_teradata.py b/providers/tests/teradata/transfers/test_teradata_to_teradata.py similarity index 100% rename from tests/providers/teradata/transfers/test_teradata_to_teradata.py rename to providers/tests/teradata/transfers/test_teradata_to_teradata.py diff --git a/tests/providers/vertica/operators/__init__.py b/providers/tests/teradata/triggers/__init__.py similarity index 100% rename from tests/providers/vertica/operators/__init__.py rename to providers/tests/teradata/triggers/__init__.py diff --git a/tests/providers/teradata/triggers/test_teradata_compute_cluster.py b/providers/tests/teradata/triggers/test_teradata_compute_cluster.py similarity index 100% rename from tests/providers/teradata/triggers/test_teradata_compute_cluster.py rename to providers/tests/teradata/triggers/test_teradata_compute_cluster.py diff --git a/tests/system/providers/__init__.py b/providers/tests/teradata/utils/__init__.py similarity index 100% rename from tests/system/providers/__init__.py rename to providers/tests/teradata/utils/__init__.py diff --git a/tests/providers/teradata/utils/test_constants.py b/providers/tests/teradata/utils/test_constants.py similarity index 100% rename from tests/providers/teradata/utils/test_constants.py rename to providers/tests/teradata/utils/test_constants.py diff --git a/tests/system/providers/amazon/__init__.py b/providers/tests/trino/__init__.py similarity index 100% rename from tests/system/providers/amazon/__init__.py rename to providers/tests/trino/__init__.py diff --git a/tests/system/providers/singularity/__init__.py b/providers/tests/trino/assets/__init__.py similarity index 100% rename from tests/system/providers/singularity/__init__.py rename to providers/tests/trino/assets/__init__.py diff --git a/tests/providers/trino/assets/test_trino.py b/providers/tests/trino/assets/test_trino.py similarity index 100% rename from tests/providers/trino/assets/test_trino.py rename to providers/tests/trino/assets/test_trino.py diff --git a/tests/system/providers/amazon/aws/__init__.py b/providers/tests/trino/hooks/__init__.py similarity index 100% rename from tests/system/providers/amazon/aws/__init__.py rename to providers/tests/trino/hooks/__init__.py diff --git a/tests/providers/trino/hooks/test_trino.py b/providers/tests/trino/hooks/test_trino.py similarity index 100% rename from tests/providers/trino/hooks/test_trino.py rename to providers/tests/trino/hooks/test_trino.py diff --git a/tests/system/providers/slack/__init__.py b/providers/tests/trino/operators/__init__.py similarity index 100% rename from tests/system/providers/slack/__init__.py rename to providers/tests/trino/operators/__init__.py diff --git a/tests/providers/trino/operators/test_trino.py b/providers/tests/trino/operators/test_trino.py similarity index 100% rename from tests/providers/trino/operators/test_trino.py rename to providers/tests/trino/operators/test_trino.py diff --git a/tests/system/providers/apache/__init__.py b/providers/tests/trino/transfers/__init__.py similarity index 100% rename from tests/system/providers/apache/__init__.py rename to providers/tests/trino/transfers/__init__.py diff --git a/tests/providers/trino/transfers/test_gcs_to_trino.py b/providers/tests/trino/transfers/test_gcs_to_trino.py similarity index 100% rename from tests/providers/trino/transfers/test_gcs_to_trino.py rename to providers/tests/trino/transfers/test_gcs_to_trino.py diff --git a/tests/system/providers/apache/beam/__init__.py b/providers/tests/vertica/__init__.py similarity index 100% rename from tests/system/providers/apache/beam/__init__.py rename to providers/tests/vertica/__init__.py diff --git a/tests/system/providers/apache/hive/__init__.py b/providers/tests/vertica/hooks/__init__.py similarity index 100% rename from tests/system/providers/apache/hive/__init__.py rename to providers/tests/vertica/hooks/__init__.py diff --git a/tests/providers/vertica/hooks/test_vertica.py b/providers/tests/vertica/hooks/test_vertica.py similarity index 100% rename from tests/providers/vertica/hooks/test_vertica.py rename to providers/tests/vertica/hooks/test_vertica.py diff --git a/tests/system/providers/apache/iceberg/__init__.py b/providers/tests/vertica/operators/__init__.py similarity index 100% rename from tests/system/providers/apache/iceberg/__init__.py rename to providers/tests/vertica/operators/__init__.py diff --git a/tests/providers/vertica/operators/test_vertica.py b/providers/tests/vertica/operators/test_vertica.py similarity index 100% rename from tests/providers/vertica/operators/test_vertica.py rename to providers/tests/vertica/operators/test_vertica.py diff --git a/tests/system/providers/snowflake/__init__.py b/providers/tests/weaviate/__init__.py similarity index 100% rename from tests/system/providers/snowflake/__init__.py rename to providers/tests/weaviate/__init__.py diff --git a/tests/system/providers/sqlite/__init__.py b/providers/tests/weaviate/hooks/__init__.py similarity index 100% rename from tests/system/providers/sqlite/__init__.py rename to providers/tests/weaviate/hooks/__init__.py diff --git a/tests/providers/weaviate/hooks/test_weaviate.py b/providers/tests/weaviate/hooks/test_weaviate.py similarity index 100% rename from tests/providers/weaviate/hooks/test_weaviate.py rename to providers/tests/weaviate/hooks/test_weaviate.py diff --git a/tests/system/providers/tableau/__init__.py b/providers/tests/weaviate/operators/__init__.py similarity index 100% rename from tests/system/providers/tableau/__init__.py rename to providers/tests/weaviate/operators/__init__.py diff --git a/tests/providers/weaviate/operators/test_weaviate.py b/providers/tests/weaviate/operators/test_weaviate.py similarity index 100% rename from tests/providers/weaviate/operators/test_weaviate.py rename to providers/tests/weaviate/operators/test_weaviate.py diff --git a/tests/system/providers/telegram/__init__.py b/providers/tests/yandex/__init__.py similarity index 100% rename from tests/system/providers/telegram/__init__.py rename to providers/tests/yandex/__init__.py diff --git a/tests/system/providers/teradata/__init__.py b/providers/tests/yandex/hooks/__init__.py similarity index 100% rename from tests/system/providers/teradata/__init__.py rename to providers/tests/yandex/hooks/__init__.py diff --git a/tests/providers/yandex/hooks/test_dataproc.py b/providers/tests/yandex/hooks/test_dataproc.py similarity index 100% rename from tests/providers/yandex/hooks/test_dataproc.py rename to providers/tests/yandex/hooks/test_dataproc.py diff --git a/tests/providers/yandex/hooks/test_yandex.py b/providers/tests/yandex/hooks/test_yandex.py similarity index 91% rename from tests/providers/yandex/hooks/test_yandex.py rename to providers/tests/yandex/hooks/test_yandex.py index e3eb36e2cfc8b..fbf77115053b2 100644 --- a/tests/providers/yandex/hooks/test_yandex.py +++ b/providers/tests/yandex/hooks/test_yandex.py @@ -27,7 +27,8 @@ from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars class TestYandexHook: @@ -62,9 +63,12 @@ def test_provider_user_agent(self, mock_get_credentials, mock_get_connection): sdk_prefix = "MyAirflow" hook = YandexCloudBaseHook() - with conf_vars({("yandex", "sdk_user_agent_prefix"): sdk_prefix}), pytest.warns( - AirflowProviderDeprecationWarning, - match="Using `provider_user_agent` in `YandexCloudBaseHook` is deprecated. Please use it in `utils.user_agent` instead.", + with ( + conf_vars({("yandex", "sdk_user_agent_prefix"): sdk_prefix}), + pytest.warns( + AirflowProviderDeprecationWarning, + match="Using `provider_user_agent` in `YandexCloudBaseHook` is deprecated. Please use it in `utils.user_agent` instead.", + ), ): assert hook.provider_user_agent().startswith(sdk_prefix) @@ -90,9 +94,12 @@ def test_sdk_user_agent(self, mock_get_credentials, mock_get_connection): ) @mock.patch("airflow.providers.yandex.utils.credentials.get_credentials", new=MagicMock()) def test_backcompat_prefix_works(self, uri): - with mock.patch.dict(os.environ, {"AIRFLOW_CONN_MY_CONN": uri}), pytest.warns( - AirflowProviderDeprecationWarning, - match="Using `connection_id` is deprecated. Please use `yandex_conn_id` parameter.", + with ( + mock.patch.dict(os.environ, {"AIRFLOW_CONN_MY_CONN": uri}), + pytest.warns( + AirflowProviderDeprecationWarning, + match="Using `connection_id` is deprecated. Please use `yandex_conn_id` parameter.", + ), ): hook = YandexCloudBaseHook("my_conn") assert hook.default_folder_id == "abc" diff --git a/tests/providers/yandex/hooks/test_yq.py b/providers/tests/yandex/hooks/test_yq.py similarity index 100% rename from tests/providers/yandex/hooks/test_yq.py rename to providers/tests/yandex/hooks/test_yq.py diff --git a/tests/system/providers/trino/__init__.py b/providers/tests/yandex/links/__init__.py similarity index 100% rename from tests/system/providers/trino/__init__.py rename to providers/tests/yandex/links/__init__.py diff --git a/tests/providers/yandex/links/test_yq.py b/providers/tests/yandex/links/test_yq.py similarity index 94% rename from tests/providers/yandex/links/test_yq.py rename to providers/tests/yandex/links/test_yq.py index d46862f1c737f..2ed720e7e141c 100644 --- a/tests/providers/yandex/links/test_yq.py +++ b/providers/tests/yandex/links/test_yq.py @@ -23,8 +23,9 @@ from airflow.models.taskinstance import TaskInstance from airflow.models.xcom import XCom from airflow.providers.yandex.links.yq import YQLink -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.mock_operators import MockOperator yandexcloud = pytest.importorskip("yandexcloud") diff --git a/tests/system/providers/weaviate/__init__.py b/providers/tests/yandex/operators/__init__.py similarity index 100% rename from tests/system/providers/weaviate/__init__.py rename to providers/tests/yandex/operators/__init__.py diff --git a/tests/providers/yandex/operators/test_dataproc.py b/providers/tests/yandex/operators/test_dataproc.py similarity index 100% rename from tests/providers/yandex/operators/test_dataproc.py rename to providers/tests/yandex/operators/test_dataproc.py diff --git a/tests/providers/yandex/operators/test_yq.py b/providers/tests/yandex/operators/test_yq.py similarity index 98% rename from tests/providers/yandex/operators/test_yq.py rename to providers/tests/yandex/operators/test_yq.py index 034f0505517ba..7e586e929eaff 100644 --- a/tests/providers/yandex/operators/test_yq.py +++ b/providers/tests/yandex/operators/test_yq.py @@ -22,7 +22,7 @@ import pytest -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS yandexcloud = pytest.importorskip("yandexcloud") diff --git a/tests/system/providers/yandex/__init__.py b/providers/tests/yandex/secrets/__init__.py similarity index 100% rename from tests/system/providers/yandex/__init__.py rename to providers/tests/yandex/secrets/__init__.py diff --git a/tests/providers/yandex/secrets/test_lockbox.py b/providers/tests/yandex/secrets/test_lockbox.py similarity index 100% rename from tests/providers/yandex/secrets/test_lockbox.py rename to providers/tests/yandex/secrets/test_lockbox.py diff --git a/tests/system/providers/ydb/__init__.py b/providers/tests/yandex/utils/__init__.py similarity index 100% rename from tests/system/providers/ydb/__init__.py rename to providers/tests/yandex/utils/__init__.py diff --git a/tests/providers/yandex/utils/test_credentials.py b/providers/tests/yandex/utils/test_credentials.py similarity index 100% rename from tests/providers/yandex/utils/test_credentials.py rename to providers/tests/yandex/utils/test_credentials.py diff --git a/tests/providers/yandex/utils/test_defaults.py b/providers/tests/yandex/utils/test_defaults.py similarity index 100% rename from tests/providers/yandex/utils/test_defaults.py rename to providers/tests/yandex/utils/test_defaults.py diff --git a/tests/providers/yandex/utils/test_fields.py b/providers/tests/yandex/utils/test_fields.py similarity index 100% rename from tests/providers/yandex/utils/test_fields.py rename to providers/tests/yandex/utils/test_fields.py diff --git a/tests/providers/yandex/utils/test_user_agent.py b/providers/tests/yandex/utils/test_user_agent.py similarity index 100% rename from tests/providers/yandex/utils/test_user_agent.py rename to providers/tests/yandex/utils/test_user_agent.py diff --git a/tests/system/providers/zendesk/__init__.py b/providers/tests/ydb/__init__.py similarity index 100% rename from tests/system/providers/zendesk/__init__.py rename to providers/tests/ydb/__init__.py diff --git a/tests/test_utils/operators/__init__.py b/providers/tests/ydb/hooks/__init__.py similarity index 100% rename from tests/test_utils/operators/__init__.py rename to providers/tests/ydb/hooks/__init__.py diff --git a/tests/providers/ydb/hooks/test_ydb.py b/providers/tests/ydb/hooks/test_ydb.py similarity index 100% rename from tests/providers/ydb/hooks/test_ydb.py rename to providers/tests/ydb/hooks/test_ydb.py diff --git a/tests/test_utils/perf/__init__.py b/providers/tests/ydb/operators/__init__.py similarity index 100% rename from tests/test_utils/perf/__init__.py rename to providers/tests/ydb/operators/__init__.py diff --git a/tests/providers/ydb/operators/test_ydb.py b/providers/tests/ydb/operators/test_ydb.py similarity index 100% rename from tests/providers/ydb/operators/test_ydb.py rename to providers/tests/ydb/operators/test_ydb.py diff --git a/tests/providers/ydb/utils/test_defaults.py b/providers/tests/ydb/utils/__init__.py similarity index 100% rename from tests/providers/ydb/utils/test_defaults.py rename to providers/tests/ydb/utils/__init__.py diff --git a/tests/providers/ydb/utils/test_credentials.py b/providers/tests/ydb/utils/test_credentials.py similarity index 100% rename from tests/providers/ydb/utils/test_credentials.py rename to providers/tests/ydb/utils/test_credentials.py diff --git a/tests/system/providers/google/leveldb/__init__.py b/providers/tests/ydb/utils/test_defaults.py similarity index 99% rename from tests/system/providers/google/leveldb/__init__.py rename to providers/tests/ydb/utils/test_defaults.py index 217e5db960782..13a83393a9124 100644 --- a/tests/system/providers/google/leveldb/__init__.py +++ b/providers/tests/ydb/utils/test_defaults.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/system/providers/microsoft/__init__.py b/providers/tests/zendesk/__init__.py similarity index 99% rename from tests/system/providers/microsoft/__init__.py rename to providers/tests/zendesk/__init__.py index 217e5db960782..13a83393a9124 100644 --- a/tests/system/providers/microsoft/__init__.py +++ b/providers/tests/zendesk/__init__.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/system/providers/dbt/__init__.py b/providers/tests/zendesk/hooks/__init__.py similarity index 99% rename from tests/system/providers/dbt/__init__.py rename to providers/tests/zendesk/hooks/__init__.py index 217e5db960782..13a83393a9124 100644 --- a/tests/system/providers/dbt/__init__.py +++ b/providers/tests/zendesk/hooks/__init__.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/providers/zendesk/hooks/test_zendesk.py b/providers/tests/zendesk/hooks/test_zendesk.py similarity index 100% rename from tests/providers/zendesk/hooks/test_zendesk.py rename to providers/tests/zendesk/hooks/test_zendesk.py diff --git a/pyproject.toml b/pyproject.toml index bc5842fc51c68..038393b139a02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,7 +205,6 @@ include = [ "/airflow/git_version" ] exclude = [ - "/airflow/providers/", "/airflow/www/node_modules/" ] artifacts = [ @@ -219,9 +218,6 @@ artifacts = [ include = [ "/airflow", ] -exclude = [ - "/airflow/providers/", -] artifacts = [ "/airflow/www/static/dist/", "/airflow/git_version" @@ -240,9 +236,6 @@ line-length = 110 extend-exclude = [ ".eggs", "*/_vendor/*", - # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can - # ignore them in ruff - "airflow/providers/common/sql/*/*.pyi", "tests/dags/test_imports.py", ] @@ -337,7 +330,21 @@ docstring-code-format = true required-imports = ["from __future__ import annotations"] combine-as-imports = true -[tool.ruff.lint.per-file-ignores] + +section-order = [ + "future", + "standard-library", + "third-party", + "first-party", + "local-folder", + "testing" +] + +# Make sure we put the "dev" imports at the end, not as a third-party module +[tool.ruff.lint.isort.sections] +testing = ["dev", "providers.tests"] + +[tool.ruff.lint.extend-per-file-ignores] "airflow/__init__.py" = ["F401", "TCH004"] "airflow/models/__init__.py" = ["F401", "TCH004"] "airflow/models/sqla_models.py" = ["F401"] @@ -370,6 +377,7 @@ combine-as-imports = true # TRY002: Use `raise Exception` "dev/perf/*" = ["TID253"] "dev/check_files.py" = ["S101"] +"dev/tests_common/*" = ["S101", "TRY002"] "dev/breeze/tests/*" = ["TID253", "S101", "TRY002"] "tests/*" = ["D", "TID253", "S101", "TRY002"] "docker_tests/*" = ["D", "TID253", "S101", "TRY002"] @@ -382,47 +390,12 @@ combine-as-imports = true "airflow/api/auth/backend/kerberos_auth.py" = ["E402"] "airflow/security/kerberos.py" = ["E402"] "airflow/security/utils.py" = ["E402"] -"tests/providers/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py" = ["E402"] -"tests/providers/common/io/xcom/test_backend.py" = ["E402"] -"tests/providers/elasticsearch/log/elasticmock/__init__.py" = ["E402"] -"tests/providers/elasticsearch/log/elasticmock/utilities/__init__.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_batch_prediction_job.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_hyperparameter_tuning_job.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_auto_ml.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_dataset.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_endpoint_service.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_generative_model.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_model_service.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_pipeline_job.py" = ["E402"] -"tests/providers/google/cloud/hooks/vertex_ai/test_prediction_service.py" = ["E402"] -"tests/providers/google/cloud/links/test_translate.py" = ["E402"] -"tests/providers/google/cloud/operators/test_automl.py"= ["E402"] -"tests/providers/google/cloud/operators/test_vertex_ai.py" = ["E402"] -"tests/providers/google/cloud/operators/vertex_ai/test_generative_model.py" = ["E402"] -"tests/providers/google/cloud/triggers/test_vertex_ai.py" = ["E402"] -"tests/providers/openai/hooks/test_openai.py" = ["E402"] -"tests/providers/openai/operators/test_openai.py" = ["E402"] -"tests/providers/openai/triggers/test_openai.py" = ["E402"] -"tests/providers/opensearch/conftest.py" = ["E402"] -"tests/providers/opensearch/hooks/test_opensearch.py" = ["E402"] -"tests/providers/opensearch/log/test_os_json_formatter.py" = ["E402"] -"tests/providers/opensearch/log/test_os_response.py" = ["E402"] -"tests/providers/opensearch/log/test_os_task_handler.py" = ["E402"] -"tests/providers/opensearch/operators/test_opensearch.py" = ["E402"] -"tests/providers/qdrant/hooks/test_qdrant.py" = ["E402"] -"tests/providers/qdrant/operators/test_qdrant.py" = ["E402"] -"tests/providers/snowflake/operators/test_snowflake_sql.py" = ["E402"] -"tests/providers/yandex/*/*.py" = ["E402"] # All the modules which do not follow B028 yet: https://docs.astral.sh/ruff/rules/no-explicit-stacklevel/ "helm_tests/airflow_aux/test_basic_helm_chart.py" = ["B028"] -# https://github.com/apache/airflow/issues/39252 -"airflow/providers/amazon/aws/hooks/eks.py" = ["W605"] - # Test compat imports banned imports to allow testing against older airflow versions -"tests/test_utils/compat.py" = ["TID251", "F401"] +"dev/tests_common/test_utils/compat.py" = ["TID251", "F401"] [tool.ruff.lint.flake8-tidy-imports] # Disallow all relative imports. @@ -512,11 +485,10 @@ norecursedirs = [ "airflow", "tests/_internals", "tests/dags_with_system_exit", - "tests/test_utils", "tests/dags_corrupted", "tests/dags", - "tests/system/providers/google/cloud/dataproc/resources", - "tests/system/providers/google/cloud/gcs/resources", + "providers/tests/system/google/cloud/dataproc/resources", + "providers/tests/system/google/cloud/gcs/resources", ] log_level = "INFO" filterwarnings = [ @@ -576,9 +548,9 @@ omit = [ "airflow/contrib/**", "airflow/example_dags/**", "airflow/migrations/**", - "airflow/providers/**/example_dags/**", "airflow/www/node_modules/**", - "airflow/providers/google/ads/_vendor/**", + "providers/src/airflow/providers/**/example_dags/**", + "providers/src/airflow/providers/google/ads/_vendor/**", ] [tool.coverage.report] @@ -609,6 +581,16 @@ show_error_codes = true disable_error_code = [ "annotation-unchecked", ] +# Since there are no __init__.py files in +# providers/src/apache/airflow/providers we need to tell MyPy where the "base" +# is, otherwise when it sees +# providers/src/apache/airflow/providers/redis/__init__.py, it thinks this is +# the toplevel `redis` folder. +explicit_package_bases = true +mypy_path = [ + "$MYPY_CONFIG_FILE_DIR", + "$MYPY_CONFIG_FILE_DIR/providers/src", +] [[tool.mypy.overrides]] module="airflow.config_templates.default_webserver_config" @@ -640,3 +622,16 @@ module=[ "apache_beam.*" ] ignore_errors = true + +[tool.uv] +dev-dependencies = [ + "local-providers", +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +local-providers = { workspace = true } + +[tool.uv.workspace] +members = ["providers"] diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 2a55d8733c328..66a4736c5fd8d 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -85,6 +85,9 @@ services: - type: bind source: ../../../logs target: /root/airflow/logs + - type: bind + source: ../../../providers + target: /opt/airflow/providers - type: bind source: ../../../pyproject.toml target: /opt/airflow/pyproject.toml diff --git a/scripts/ci/kubernetes/k8s_requirements.txt b/scripts/ci/kubernetes/k8s_requirements.txt index 50e9399f5f221..1e587051f5845 100644 --- a/scripts/ci/kubernetes/k8s_requirements.txt +++ b/scripts/ci/kubernetes/k8s_requirements.txt @@ -1 +1,3 @@ --e .[devel-devscripts,devel-tests,cncf.kubernetes] --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.9.txt" +--constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.9.txt +-e .[devel-devscripts,devel-tests,cncf.kubernetes] +-e ./providers diff --git a/scripts/ci/pre_commit/check_providers_subpackages_all_have_init.py b/scripts/ci/pre_commit/check_providers_subpackages_all_have_init.py index da17f794eaeb6..af3afe0abc076 100755 --- a/scripts/ci/pre_commit/check_providers_subpackages_all_have_init.py +++ b/scripts/ci/pre_commit/check_providers_subpackages_all_have_init.py @@ -19,25 +19,27 @@ import os import sys -from glob import glob from pathlib import Path -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) -ACCEPTED_NON_INIT_DIRS = ["adr", "doc", "templates"] +ROOT_DIR = Path(__file__).parents[3].resolve() +ACCEPTED_NON_INIT_DIRS = ["adr", "doc", "templates", "__pycache__"] -def check_dir_init_file(provider_files: list[str]) -> None: +def check_dir_init_file(folders: list[Path]) -> None: missing_init_dirs: list[Path] = [] - for candidate_path in provider_files: - if candidate_path.endswith("/__pycache__"): - continue - path = Path(candidate_path) - if path.is_dir() and not (path / "__init__.py").exists(): - if path.name not in ACCEPTED_NON_INIT_DIRS: - missing_init_dirs.append(path) + folders = list(folders) + for path in folders: + for root, dirs, files in os.walk(path): + # Edit it in place, so we don't recurse to folders we don't care about + dirs[:] = [d for d in dirs if d not in ACCEPTED_NON_INIT_DIRS] + + if "__init__.py" in files: + continue + + missing_init_dirs.append(Path(root)) if missing_init_dirs: - with open(os.path.join(ROOT_DIR, "scripts/ci/license-templates/LICENSE.txt")) as license: + with ROOT_DIR.joinpath("scripts/ci/license-templates/LICENSE.txt").open() as license: license_txt = license.readlines() prefixed_licensed_txt = [f"# {line}" if line != "\n" else "#\n" for line in license_txt] @@ -51,7 +53,11 @@ def check_dir_init_file(provider_files: list[str]) -> None: if __name__ == "__main__": - all_provider_subpackage_dirs = sorted(glob(f"{ROOT_DIR}/airflow/providers/**/*", recursive=True)) - check_dir_init_file(all_provider_subpackage_dirs) - all_test_provider_subpackage_dirs = sorted(glob(f"{ROOT_DIR}/tests/providers/**/*", recursive=True)) - check_dir_init_file(all_test_provider_subpackage_dirs) + providers_root = Path(f"{ROOT_DIR}/providers") + providers_ns = providers_root.joinpath("src", "airflow", "providers") + providers_tests = providers_root.joinpath("tests") + + providers_pkgs = sorted(map(lambda f: f.parent, providers_ns.rglob("provider.yaml"))) + check_dir_init_file(providers_pkgs) + + check_dir_init_file([providers_root / "tests"]) diff --git a/scripts/ci/pre_commit/check_system_tests.py b/scripts/ci/pre_commit/check_system_tests.py index 4c82272ad7875..c7c305460cb3e 100755 --- a/scripts/ci/pre_commit/check_system_tests.py +++ b/scripts/ci/pre_commit/check_system_tests.py @@ -37,13 +37,13 @@ WATCHER_APPEND_INSTRUCTION = "list(dag.tasks) >> watcher()" PYTEST_FUNCTION = """ -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) """ PYTEST_FUNCTION_PATTERN = re.compile( - r"from tests\.system\.utils import get_test_run(?: # noqa: E402)?\s+" + r"from dev\.tests_common\.test_utils\.system_tests import get_test_run(?: # noqa: E402)?\s+" r"(?:# .+\))?\s+" r"test_run = get_test_run\(dag\)" ) @@ -51,11 +51,11 @@ def _check_file(file: Path): content = file.read_text() - if "from tests.system.utils.watcher import watcher" in content: + if "from dev.tests_common.test_utils.watcher import watcher" in content: index = content.find(WATCHER_APPEND_INSTRUCTION) if index == -1: errors.append( - f"[red]The example {file} imports tests.system.utils.watcher " + f"[red]The example {file} imports dev.tests_common.test_utils.watcher " f"but does not use it properly![/]\n\n" "[yellow]Make sure you have:[/]\n\n" f" {WATCHER_APPEND_INSTRUCTION}\n\n" diff --git a/scripts/ci/pre_commit/check_system_tests_hidden_in_index.py b/scripts/ci/pre_commit/check_system_tests_hidden_in_index.py index fde6f38f45a9a..1c1fdb02c1793 100755 --- a/scripts/ci/pre_commit/check_system_tests_hidden_in_index.py +++ b/scripts/ci/pre_commit/check_system_tests_hidden_in_index.py @@ -54,10 +54,10 @@ def check_system_test_entry_hidden(provider_index: Path): :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/{provider_path}/index> + System Tests <_api/tests/system/{provider_path}/index> """ index_text = provider_index.read_text() - system_tests_path = AIRFLOW_SOURCES_ROOT / "tests" / "system" / "providers" / provider_path + system_tests_path = AIRFLOW_SOURCES_ROOT / "providers" / "tests" / "system" / provider_path index_text_manual = index_text.split( ".. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!" )[0] diff --git a/scripts/ci/pre_commit/mypy_folder.py b/scripts/ci/pre_commit/mypy_folder.py index 2e50a3334f298..b2d7c76580276 100755 --- a/scripts/ci/pre_commit/mypy_folder.py +++ b/scripts/ci/pre_commit/mypy_folder.py @@ -31,7 +31,7 @@ initialize_breeze_precommit(__name__, __file__) -ALLOWED_FOLDERS = ["airflow", "airflow/providers", "dev", "docs"] +ALLOWED_FOLDERS = ["airflow", "providers/src/airflow/providers", "dev", "docs"] if len(sys.argv) < 2: console.print(f"[yellow]You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}\n") @@ -43,12 +43,10 @@ sys.exit(1) arguments = [mypy_folder] -if mypy_folder == "airflow/providers": +if mypy_folder == "providers/src/airflow/providers": arguments.extend( [ - "tests/providers", - "tests/system/providers", - "tests/integration/providers", + "providers/tests", "--namespace-packages", ] ) @@ -57,14 +55,6 @@ arguments.extend( [ "tests", - "--exclude", - "airflow/providers", - "--exclude", - "tests/providers", - "--exclude", - "tests/system/providers", - "--exclude", - "tests/integration/providers", ] ) diff --git a/scripts/ci/pre_commit/update_common_sql_api_stubs.py b/scripts/ci/pre_commit/update_common_sql_api_stubs.py index 954302804e6f1..371c758146a2e 100755 --- a/scripts/ci/pre_commit/update_common_sql_api_stubs.py +++ b/scripts/ci/pre_commit/update_common_sql_api_stubs.py @@ -39,10 +39,12 @@ from common_precommit_black_utils import black_format from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH -PROVIDERS_ROOT = (AIRFLOW_SOURCES_ROOT_PATH / "airflow" / "providers").resolve(strict=True) +PROVIDERS_ROOT = (AIRFLOW_SOURCES_ROOT_PATH / "providers" / "src" / "airflow" / "providers").resolve( + strict=True +) COMMON_SQL_ROOT = (PROVIDERS_ROOT / "common" / "sql").resolve(strict=True) OUT_DIR = AIRFLOW_SOURCES_ROOT_PATH / "out" -OUT_DIR_PROVIDERS = OUT_DIR / "airflow" / "providers" +OUT_DIR_PROVIDERS = OUT_DIR / PROVIDERS_ROOT.relative_to(AIRFLOW_SOURCES_ROOT_PATH) COMMON_SQL_PACKAGE_PREFIX = "airflow.providers.common.sql." @@ -317,7 +319,7 @@ def compare_stub_files(generated_stub_path: Path, force_override: bool) -> tuple shutil.rmtree(OUT_DIR, ignore_errors=True) subprocess.run( - ["stubgen", *[os.fspath(path) for path in COMMON_SQL_ROOT.rglob("*.py")]], + ["stubgen", f"--out={ OUT_DIR }", COMMON_SQL_ROOT], cwd=AIRFLOW_SOURCES_ROOT_PATH, ) total_removals, total_additions = 0, 0 diff --git a/scripts/ci/pre_commit/update_example_dags_paths.py b/scripts/ci/pre_commit/update_example_dags_paths.py index 8b2c461ec8c1f..17d2a2ccea453 100755 --- a/scripts/ci/pre_commit/update_example_dags_paths.py +++ b/scripts/ci/pre_commit/update_example_dags_paths.py @@ -34,7 +34,7 @@ console = Console(color_system="standard", width=200) AIRFLOW_SOURCES_ROOT = Path(__file__).parents[3].resolve() - +PROVIDERS_SRC = AIRFLOW_SOURCES_ROOT / "providers" / "src" / "airflow" / "providers" EXAMPLE_DAGS_URL_MATCHER = re.compile( r"^(.*)(https://github.com/apache/airflow/tree/(.*)/airflow/providers/(.*)/example_dags)(/?>.*)$" @@ -45,10 +45,7 @@ def get_provider_and_version(url_path: str) -> tuple[str, str]: candidate_folders = url_path.split("/") while candidate_folders: try: - with open( - (AIRFLOW_SOURCES_ROOT / "airflow" / "providers").joinpath(*candidate_folders) - / "provider.yaml" - ) as f: + with PROVIDERS_SRC.joinpath(*candidate_folders, "provider.yaml").open() as f: provider_info = yaml.safe_load(f) version = provider_info["versions"][0] provider = "-".join(candidate_folders) @@ -68,13 +65,11 @@ def replace_match(file: Path, line: str) -> str | None: if match: url_path_to_dir = match.group(4) folders = url_path_to_dir.split("/") - example_dags_folder = (AIRFLOW_SOURCES_ROOT / "airflow" / "providers").joinpath( - *folders - ) / "example_dags" + example_dags_folder = PROVIDERS_SRC.joinpath(*folders, "example_dags") provider, version = get_provider_and_version(url_path_to_dir) proper_system_tests_url = ( f"https://github.com/apache/airflow/tree/providers-{provider}/{version}" - f"/tests/system/providers/{url_path_to_dir}" + f"/providers/tests/system/{url_path_to_dir}" ) if not example_dags_folder.exists(): if proper_system_tests_url in file.read_text(): diff --git a/scripts/ci/pre_commit/update_providers_dependencies.py b/scripts/ci/pre_commit/update_providers_dependencies.py index 903a692384558..8a0e797d0ab6a 100755 --- a/scripts/ci/pre_commit/update_providers_dependencies.py +++ b/scripts/ci/pre_commit/update_providers_dependencies.py @@ -34,9 +34,10 @@ AIRFLOW_SOURCES_ROOT = Path(__file__).parents[3].resolve() -AIRFLOW_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" -AIRFLOW_TESTS_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT / "tests" / "providers" -AIRFLOW_SYSTEM_TESTS_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT / "system" / "tests" / "providers" +AIRFLOW_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT / "providers" +AIRFLOW_PROVIDERS_SRC_DIR = AIRFLOW_PROVIDERS_DIR / "src" / "airflow" / "providers" +AIRFLOW_TESTS_PROVIDERS_DIR = AIRFLOW_PROVIDERS_DIR / "tests" +AIRFLOW_SYSTEM_TESTS_PROVIDERS_DIR = AIRFLOW_TESTS_PROVIDERS_DIR / "tests" / "system" DEPENDENCIES_JSON_FILE_PATH = AIRFLOW_SOURCES_ROOT / "generated" / "provider_dependencies.json" @@ -95,16 +96,18 @@ def visit_ImportFrom(self, node: ImportFrom): def find_all_providers_and_provider_files(): - for root, _, filenames in os.walk(AIRFLOW_PROVIDERS_DIR): + for root, _, filenames in os.walk(AIRFLOW_PROVIDERS_SRC_DIR): for filename in filenames: if filename == "provider.yaml": provider_file = Path(root, filename) - provider_name = str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR)).replace( + provider_name = str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_SRC_DIR)).replace( os.sep, "." ) provider_info = yaml.safe_load(provider_file.read_text()) if provider_info["state"] == "suspended": - suspended_paths.append(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR).as_posix()) + suspended_paths.append( + provider_file.parent.relative_to(AIRFLOW_PROVIDERS_SRC_DIR).as_posix() + ) ALL_PROVIDERS[provider_name] = provider_info path = Path(root, filename) if path.is_file() and path.name.endswith(".py"): @@ -145,7 +148,7 @@ def get_imports_from_file(file_path: Path) -> list[str]: def get_provider_id_from_file_name(file_path: Path) -> str | None: # is_relative_to is only available in Python 3.9 - we should simplify this check when we are Python 3.9+ try: - relative_path = file_path.relative_to(AIRFLOW_PROVIDERS_DIR) + relative_path = file_path.relative_to(AIRFLOW_PROVIDERS_SRC_DIR) except ValueError: try: relative_path = file_path.relative_to(AIRFLOW_SYSTEM_TESTS_PROVIDERS_DIR) @@ -173,7 +176,14 @@ def check_if_different_provider_used(file_path: Path) -> None: imported_provider = get_provider_id_from_import(import_name, file_path) if imported_provider is not None and imported_provider not in ALL_PROVIDERS: warnings.append(f"The provider {imported_provider} from {file_path} cannot be found.") - elif imported_provider and file_provider != imported_provider: + continue + + if imported_provider == "standard": + # Standard -- i.e. BashOperator is used in a lot of example dags, but we don't want to mark this + # as a provider cross dependency + if file_path.name == "celery_executor_utils.py" or "/example_dags/" in file_path.as_posix(): + continue + if imported_provider and file_provider != imported_provider: ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider) diff --git a/scripts/ci/pre_commit/version_heads_map.py b/scripts/ci/pre_commit/version_heads_map.py index 10a6dee2eaf23..6796819444d8c 100755 --- a/scripts/ci/pre_commit/version_heads_map.py +++ b/scripts/ci/pre_commit/version_heads_map.py @@ -29,8 +29,9 @@ DB_FILE = PROJECT_SOURCE_ROOT_DIR / "airflow" / "utils" / "db.py" MIGRATION_PATH = PROJECT_SOURCE_ROOT_DIR / "airflow" / "migrations" / "versions" -FAB_DB_FILE = PROJECT_SOURCE_ROOT_DIR / "airflow" / "providers" / "fab" / "auth_manager" / "models" / "db.py" -FAB_MIGRATION_PATH = PROJECT_SOURCE_ROOT_DIR / "airflow" / "providers" / "fab" / "migrations" / "versions" +PROVIDERS_SRC = PROJECT_SOURCE_ROOT_DIR / "providers" / "src" +FAB_DB_FILE = PROVIDERS_SRC / "airflow" / "providers" / "fab" / "auth_manager" / "models" / "db.py" +FAB_MIGRATION_PATH = PROVIDERS_SRC / "airflow" / "providers" / "fab" / "migrations" / "versions" sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is importable diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 5db10ad967695..324bca7ccfe09 100644 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -48,8 +48,13 @@ function install_airflow() { # Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then + # We need _a _ file in there otherwise the editable install doesn't include anything in the .pth file + mkdir -p ./providers/src/airflow/providers/ + touch ./providers/src/airflow/providers/__init__.py + trap 'rm -f ./providers/src/airflow/providers/__init__.py 2>/dev/null' EXIT + # When installing from sources - we always use `--editable` mode - installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" + installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then diff --git a/scripts/in_container/install_devel_deps.py b/scripts/in_container/install_devel_deps.py index 7a2410f385dfc..84f28bb2cc48b 100755 --- a/scripts/in_container/install_devel_deps.py +++ b/scripts/in_container/install_devel_deps.py @@ -27,7 +27,8 @@ def get_devel_test_deps() -> list[str]: - devel_deps: list[str] = [] + # Pre-install the tests_common pytest plugin/utils, in case sources aren't mounted + devel_deps: list[str] = ["./dev/tests_common"] hatch_build_content = (AIRFLOW_SOURCES_DIR / "hatch_build.py").read_text().splitlines() store = False for line in hatch_build_content: diff --git a/scripts/in_container/run_mypy.sh b/scripts/in_container/run_mypy.sh index 0245825a72647..976963e444a05 100755 --- a/scripts/in_container/run_mypy.sh +++ b/scripts/in_container/run_mypy.sh @@ -30,8 +30,8 @@ then for folder in ${SUSPENDED_PROVIDERS_FOLDERS=} do ADDITIONAL_MYPY_OPTIONS+=( - "--exclude" "airflow/providers/${folder}/*" - "--exclude" "tests/providers/${folder}/*" + "--exclude" "providers/src/airflow/providers/${folder}/*" + "--exclude" "providers/tests/${folder}/*" ) done fi diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 95978a04ddd09..ab5ebcac697b9 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -74,6 +74,7 @@ ) ROOT_DIR = pathlib.Path(__file__).resolve().parents[2] +PROVIDERS_SRC = ROOT_DIR / "providers" / "src" DOCS_DIR = ROOT_DIR.joinpath("docs") PROVIDER_DATA_SCHEMA_PATH = ROOT_DIR.joinpath("airflow", "provider.yaml.schema.json") PROVIDER_ISSUE_TEMPLATE_PATH = ROOT_DIR.joinpath( @@ -97,11 +98,13 @@ suspended_integrations: set[str] = set() -def _filepath_to_module(filepath: pathlib.Path) -> str: - p = filepath.resolve().relative_to(ROOT_DIR).as_posix() - if p.endswith(".py"): - p = p[:-3] - return p.replace("/", ".") +def _filepath_to_module(filepath: pathlib.Path | str) -> str: + if isinstance(filepath, str): + filepath = pathlib.Path(filepath) + if filepath.name == "provider.yaml": + filepath = filepath.parent + p = filepath.resolve().relative_to(PROVIDERS_SRC).with_suffix("") + return p.as_posix().replace("/", ".") def _load_schema() -> dict[str, Any]: @@ -270,8 +273,8 @@ def check_if_objects_exist_and_belong_to_package( def parse_module_data(provider_data, resource_type, yaml_file_path): - package_dir = ROOT_DIR.joinpath(yaml_file_path).parent - provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") + provider_dir = pathlib.Path(yaml_file_path).parent + package_dir = ROOT_DIR.joinpath(provider_dir) py_files = itertools.chain( package_dir.glob(f"**/{resource_type}/*.py"), package_dir.glob(f"{resource_type}/*.py"), @@ -280,7 +283,7 @@ def parse_module_data(provider_data, resource_type, yaml_file_path): ) expected_modules = {_filepath_to_module(f) for f in py_files if f.name != "__init__.py"} resource_data = provider_data.get(resource_type, []) - return expected_modules, provider_package, resource_data + return expected_modules, _filepath_to_module(provider_dir), resource_data def run_check(title: str): @@ -398,12 +401,9 @@ def check_completeness_of_list_of_transfers(yaml_files: dict[str, dict]) -> tupl current_modules, provider_package, yaml_file_path, resource_type, ObjectType.MODULE ) try: - package_name = os.fspath(ROOT_DIR.joinpath(yaml_file_path).parent.relative_to(ROOT_DIR)).replace( - "/", "." - ) assert_sets_equal( set(expected_modules), - f"Found list of transfer modules in provider package: {package_name}", + f"Found list of transfer modules in provider package: {provider_package}", set(current_modules), f"Currently configured list of transfer modules in {yaml_file_path}", ) @@ -423,7 +423,7 @@ def check_hook_class_name_entries_in_connection_types(yaml_files: dict[str, dict num_errors = 0 num_connection_types = 0 for yaml_file_path, provider_data in yaml_files.items(): - provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") + provider_package = _filepath_to_module(yaml_file_path) connection_types = provider_data.get(resource_type) if connection_types: num_connection_types += len(connection_types) @@ -440,7 +440,7 @@ def check_plugin_classes(yaml_files: dict[str, dict]) -> tuple[int, int]: num_errors = 0 num_plugins = 0 for yaml_file_path, provider_data in yaml_files.items(): - provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") + provider_package = _filepath_to_module(yaml_file_path) plugins = provider_data.get(resource_type) if plugins: num_plugins += len(plugins) @@ -460,7 +460,7 @@ def check_extra_link_classes(yaml_files: dict[str, dict]) -> tuple[int, int]: num_errors = 0 num_extra_links = 0 for yaml_file_path, provider_data in yaml_files.items(): - provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") + provider_package = _filepath_to_module(yaml_file_path) extra_links = provider_data.get(resource_type) if extra_links: num_extra_links += len(extra_links) @@ -476,7 +476,7 @@ def check_notification_classes(yaml_files: dict[str, dict]) -> tuple[int, int]: num_errors = 0 num_notifications = 0 for yaml_file_path, provider_data in yaml_files.items(): - provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") + provider_package = _filepath_to_module(yaml_file_path) notifications = provider_data.get(resource_type) if notifications: num_notifications += len(notifications) @@ -692,7 +692,9 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]): ProvidersManager().initialize_providers_configuration() architecture = Architecture.get_current() console.print(f"Verifying packages on {architecture} architecture. Platform: {platform.machine()}.") - provider_files_pattern = pathlib.Path(ROOT_DIR, "airflow", "providers").rglob("provider.yaml") + provider_files_pattern = pathlib.Path(ROOT_DIR, "providers", "src", "airflow", "providers").rglob( + "provider.yaml" + ) all_provider_files = sorted(str(path) for path in provider_files_pattern) if len(sys.argv) > 1: paths = [os.fspath(ROOT_DIR / f) for f in sorted(sys.argv[1:])] diff --git a/tests/always/test_connection.py b/tests/always/test_connection.py index 5e41ddda798c0..0f6b27c62d4ef 100644 --- a/tests/always/test_connection.py +++ b/tests/always/test_connection.py @@ -33,7 +33,8 @@ from airflow.models import Connection, crypto from airflow.providers.sqlite.hooks.sqlite import SqliteHook from airflow.providers_manager import HookInfo -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars ConnectionParts = namedtuple("ConnectionParts", ["conn_type", "login", "password", "host", "port", "schema"]) diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 9d10ce5cad199..9def5f42206f8 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -29,12 +29,13 @@ from airflow.models import DagBag from airflow.utils import yaml -from tests.test_utils.asserts import assert_queries_count + +from dev.tests_common.test_utils.asserts import assert_queries_count AIRFLOW_SOURCES_ROOT = Path(__file__).resolve().parents[2] AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" CURRENT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" -PROVIDERS_PREFIXES = ("airflow/providers/", "tests/system/providers/") +PROVIDERS_PREFIXES = ("providers/src/airflow/providers/", "providers/tests/system/") OPTIONAL_PROVIDERS_DEPENDENCIES: dict[str, dict[str, str | None]] = { # Some examples or system tests may depend on additional packages # that are not included in certain CI checks. @@ -50,17 +51,17 @@ # Generally, these should be resolved as soon as a parameter or operator is deprecated. # If the deprecation is postponed, the item should be added to this tuple, # and a corresponding Issue should be created on GitHub. - "tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py", - "tests/system/providers/google/cloud/dataflow/example_dataflow_sql.py", - "tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py", - "tests/system/providers/google/cloud/datapipelines/example_datapipeline.py", - "tests/system/providers/google/cloud/gcs/example_gcs_sensor.py", - "tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py", - "tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py", - "tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py", - "tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py", - "tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py", - "tests/system/providers/google/cloud/life_sciences/example_life_sciences.py", + "providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py", + "providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py", + "providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py", + "providers/tests/system/google/cloud/datapipelines/example_datapipeline.py", + "providers/tests/system/google/cloud/gcs/example_gcs_sensor.py", + "providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py", + "providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py", + "providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py", + "providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py", + "providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py", + "providers/tests/system/google/cloud/life_sciences/example_life_sciences.py", # Deprecated Operators/Hooks, which replaced by common.sql Operators/Hooks ) @@ -89,7 +90,7 @@ def get_suspended_providers_folders() -> list[str]: suspended_providers.append( provider_path.parent.relative_to(AIRFLOW_SOURCES_ROOT) .as_posix() - .replace("airflow/providers/", "") + .replace("providers/src/airflow/providers/", "") ) return suspended_providers @@ -107,7 +108,7 @@ def get_python_excluded_providers_folders() -> list[str]: excluded_providers.append( provider_path.parent.relative_to(AIRFLOW_SOURCES_ROOT) .as_posix() - .replace("airflow/providers/", "") + .replace("providers/src/airflow/providers/", "") ) return excluded_providers diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index b27729a68a261..9db0b22df84fc 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -21,169 +21,159 @@ import itertools import mmap import os +import pathlib import pytest -ROOT_FOLDER = os.path.realpath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) -) +ROOT_FOLDER = pathlib.Path(__file__).parents[2] +PROVIDERS_SRC = ROOT_FOLDER.joinpath("providers", "src") +PROVIDERS_TESTS = ROOT_FOLDER.joinpath("providers", "tests") class TestProjectStructure: def test_reference_to_providers_from_core(self): - for filename in glob.glob(f"{ROOT_FOLDER}/example_dags/**/*.py", recursive=True): + for filename in ROOT_FOLDER.glob("example_dags/**/*.py"): self.assert_file_not_contains(filename, "providers") def test_deprecated_packages(self): - path_pattern = f"{ROOT_FOLDER}/airflow/contrib/**/*.py" - - for filename in glob.glob(path_pattern, recursive=True): - if filename.endswith("/__init__.py"): + for filename in ROOT_FOLDER.glob("airflow/contrib/**/*.py"): + if filename.name == "__init__.py": self.assert_file_contains(filename, "This package is deprecated.") else: self.assert_file_contains(filename, "This module is deprecated.") - def assert_file_not_contains(self, filename: str, pattern: str): + def assert_file_not_contains(self, filename: pathlib.Path, pattern: str): with open(filename, "rb", 0) as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as content: if content.find(bytes(pattern, "utf-8")) != -1: pytest.fail(f"File {filename} not contains pattern - {pattern}") - def assert_file_contains(self, filename: str, pattern: str): + def assert_file_contains(self, filename: pathlib.Path, pattern: str): with open(filename, "rb", 0) as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as content: if content.find(bytes(pattern, "utf-8")) == -1: pytest.fail(f"File {filename} contains illegal pattern - {pattern}") def test_providers_modules_should_have_tests(self): """ - Assert every module in /airflow/providers has a corresponding test_ file in tests/airflow/providers. + Assert every module in /providers/src/airflow/providers has a corresponding test_ file in providers/providers. """ # The test below had a but for quite a while and we missed a lot of modules to have tess # We should make sure that one goes to 0 OVERLOOKED_TESTS = [ - "tests/providers/amazon/aws/executors/batch/test_boto_schema.py", - "tests/providers/amazon/aws/executors/batch/test_batch_executor_config.py", - "tests/providers/amazon/aws/executors/batch/test_utils.py", - "tests/providers/amazon/aws/executors/ecs/test_boto_schema.py", - "tests/providers/amazon/aws/executors/ecs/test_ecs_executor_config.py", - "tests/providers/amazon/aws/executors/ecs/test_utils.py", - "tests/providers/amazon/aws/executors/utils/test_base_config_keys.py", - "tests/providers/amazon/aws/operators/test_emr.py", - "tests/providers/amazon/aws/operators/test_sagemaker.py", - "tests/providers/amazon/aws/sensors/test_emr.py", - "tests/providers/amazon/aws/sensors/test_sagemaker.py", - "tests/providers/amazon/aws/test_exceptions.py", - "tests/providers/amazon/aws/triggers/test_step_function.py", - "tests/providers/amazon/aws/utils/test_rds.py", - "tests/providers/amazon/aws/utils/test_sagemaker.py", - "tests/providers/amazon/aws/utils/test_asset_compat_lineage_collector.py", - "tests/providers/amazon/aws/waiters/test_base_waiter.py", - "tests/providers/apache/cassandra/hooks/test_cassandra.py", - "tests/providers/apache/drill/operators/test_drill.py", - "tests/providers/apache/druid/operators/test_druid_check.py", - "tests/providers/apache/hdfs/hooks/test_hdfs.py", - "tests/providers/apache/hdfs/log/test_hdfs_task_handler.py", - "tests/providers/apache/hdfs/sensors/test_hdfs.py", - "tests/providers/apache/hive/plugins/test_hive.py", - "tests/providers/celery/executors/test_celery_executor_utils.py", - "tests/providers/celery/executors/test_default_celery.py", - "tests/providers/cloudant/test_cloudant_fake.py", - "tests/providers/cncf/kubernetes/backcompat/test_backwards_compat_converters.py", - "tests/providers/cncf/kubernetes/executors/test_kubernetes_executor_types.py", - "tests/providers/cncf/kubernetes/executors/test_kubernetes_executor_utils.py", - "tests/providers/cncf/kubernetes/operators/test_kubernetes_pod.py", - "tests/providers/cncf/kubernetes/test_k8s_model.py", - "tests/providers/cncf/kubernetes/test_kube_client.py", - "tests/providers/cncf/kubernetes/test_kube_config.py", - "tests/providers/cncf/kubernetes/test_pod_generator_deprecated.py", - "tests/providers/cncf/kubernetes/test_pod_launcher_deprecated.py", - "tests/providers/cncf/kubernetes/test_python_kubernetes_script.py", - "tests/providers/cncf/kubernetes/test_secret.py", - "tests/providers/cncf/kubernetes/triggers/test_kubernetes_pod.py", - "tests/providers/cncf/kubernetes/utils/test_delete_from.py", - "tests/providers/cncf/kubernetes/utils/test_k8s_hashlib_wrapper.py", - "tests/providers/cncf/kubernetes/utils/test_xcom_sidecar.py", - "tests/providers/google/cloud/fs/test_gcs.py", - "tests/providers/google/cloud/links/test_automl.py", - "tests/providers/google/cloud/links/test_base.py", - "tests/providers/google/cloud/links/test_bigquery.py", - "tests/providers/google/cloud/links/test_bigquery_dts.py", - "tests/providers/google/cloud/links/test_bigtable.py", - "tests/providers/google/cloud/links/test_cloud_build.py", - "tests/providers/google/cloud/links/test_cloud_functions.py", - "tests/providers/google/cloud/links/test_cloud_memorystore.py", - "tests/providers/google/cloud/links/test_cloud_sql.py", - "tests/providers/google/cloud/links/test_cloud_storage_transfer.py", - "tests/providers/google/cloud/links/test_cloud_tasks.py", - "tests/providers/google/cloud/links/test_compute.py", - "tests/providers/google/cloud/links/test_data_loss_prevention.py", - "tests/providers/google/cloud/links/test_datacatalog.py", - "tests/providers/google/cloud/links/test_dataflow.py", - "tests/providers/google/cloud/links/test_dataform.py", - "tests/providers/google/cloud/links/test_datafusion.py", - "tests/providers/google/cloud/links/test_dataplex.py", - "tests/providers/google/cloud/links/test_dataprep.py", - "tests/providers/google/cloud/links/test_dataproc.py", - "tests/providers/google/cloud/links/test_datastore.py", - "tests/providers/google/cloud/links/test_kubernetes_engine.py", - "tests/providers/google/cloud/links/test_life_sciences.py", - "tests/providers/google/cloud/links/test_mlengine.py", - "tests/providers/google/cloud/links/test_pubsub.py", - "tests/providers/google/cloud/links/test_spanner.py", - "tests/providers/google/cloud/links/test_stackdriver.py", - "tests/providers/google/cloud/links/test_vertex_ai.py", - "tests/providers/google/cloud/links/test_workflows.py", - "tests/providers/google/cloud/operators/vertex_ai/test_auto_ml.py", - "tests/providers/google/cloud/operators/vertex_ai/test_batch_prediction_job.py", - "tests/providers/google/cloud/operators/vertex_ai/test_custom_job.py", - "tests/providers/google/cloud/operators/vertex_ai/test_dataset.py", - "tests/providers/google/cloud/operators/vertex_ai/test_endpoint_service.py", - "tests/providers/google/cloud/operators/vertex_ai/test_hyperparameter_tuning_job.py", - "tests/providers/google/cloud/operators/vertex_ai/test_model_service.py", - "tests/providers/google/cloud/operators/vertex_ai/test_pipeline_job.py", - "tests/providers/google/cloud/sensors/test_dataform.py", - "tests/providers/google/cloud/transfers/test_bigquery_to_sql.py", - "tests/providers/google/cloud/transfers/test_presto_to_gcs.py", - "tests/providers/google/cloud/utils/test_bigquery.py", - "tests/providers/google/cloud/utils/test_bigquery_get_data.py", - "tests/providers/google/cloud/utils/test_dataform.py", - "tests/providers/google/common/links/test_storage.py", - "tests/providers/google/common/test_consts.py", - "tests/providers/google/test_go_module_utils.py", - "tests/providers/microsoft/azure/operators/test_adls.py", - "tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py", - "tests/providers/openlineage/utils/test_asset_compat_lineage_collector.py", - "tests/providers/slack/notifications/test_slack_notifier.py", - "tests/providers/snowflake/triggers/test_snowflake_trigger.py", - "tests/providers/yandex/hooks/test_yandexcloud_dataproc.py", - "tests/providers/yandex/operators/test_yandexcloud_dataproc.py", - "tests/providers/fab/migrations/test_env.py", + "providers/tests/amazon/aws/executors/batch/test_boto_schema.py", + "providers/tests/amazon/aws/executors/batch/test_batch_executor_config.py", + "providers/tests/amazon/aws/executors/batch/test_utils.py", + "providers/tests/amazon/aws/executors/ecs/test_boto_schema.py", + "providers/tests/amazon/aws/executors/ecs/test_ecs_executor_config.py", + "providers/tests/amazon/aws/executors/ecs/test_utils.py", + "providers/tests/amazon/aws/executors/utils/test_base_config_keys.py", + "providers/tests/amazon/aws/operators/test_emr.py", + "providers/tests/amazon/aws/operators/test_sagemaker.py", + "providers/tests/amazon/aws/sensors/test_emr.py", + "providers/tests/amazon/aws/sensors/test_sagemaker.py", + "providers/tests/amazon/aws/test_exceptions.py", + "providers/tests/amazon/aws/triggers/test_step_function.py", + "providers/tests/amazon/aws/utils/test_rds.py", + "providers/tests/amazon/aws/utils/test_sagemaker.py", + "providers/tests/amazon/aws/waiters/test_base_waiter.py", + "providers/tests/apache/cassandra/hooks/test_cassandra.py", + "providers/tests/apache/drill/operators/test_drill.py", + "providers/tests/apache/druid/operators/test_druid_check.py", + "providers/tests/apache/hdfs/hooks/test_hdfs.py", + "providers/tests/apache/hdfs/log/test_hdfs_task_handler.py", + "providers/tests/apache/hdfs/sensors/test_hdfs.py", + "providers/tests/apache/hive/plugins/test_hive.py", + "providers/tests/celery/executors/test_celery_executor_utils.py", + "providers/tests/celery/executors/test_default_celery.py", + "providers/tests/cloudant/test_cloudant_fake.py", + "providers/tests/cncf/kubernetes/backcompat/test_backwards_compat_converters.py", + "providers/tests/cncf/kubernetes/executors/test_kubernetes_executor_types.py", + "providers/tests/cncf/kubernetes/executors/test_kubernetes_executor_utils.py", + "providers/tests/cncf/kubernetes/operators/test_kubernetes_pod.py", + "providers/tests/cncf/kubernetes/test_k8s_model.py", + "providers/tests/cncf/kubernetes/test_kube_client.py", + "providers/tests/cncf/kubernetes/test_kube_config.py", + "providers/tests/cncf/kubernetes/test_pod_generator_deprecated.py", + "providers/tests/cncf/kubernetes/test_pod_launcher_deprecated.py", + "providers/tests/cncf/kubernetes/test_python_kubernetes_script.py", + "providers/tests/cncf/kubernetes/test_secret.py", + "providers/tests/cncf/kubernetes/triggers/test_kubernetes_pod.py", + "providers/tests/cncf/kubernetes/utils/test_delete_from.py", + "providers/tests/cncf/kubernetes/utils/test_k8s_hashlib_wrapper.py", + "providers/tests/cncf/kubernetes/utils/test_xcom_sidecar.py", + "providers/tests/google/cloud/fs/test_gcs.py", + "providers/tests/google/cloud/links/test_automl.py", + "providers/tests/google/cloud/links/test_base.py", + "providers/tests/google/cloud/links/test_bigquery.py", + "providers/tests/google/cloud/links/test_bigquery_dts.py", + "providers/tests/google/cloud/links/test_bigtable.py", + "providers/tests/google/cloud/links/test_cloud_build.py", + "providers/tests/google/cloud/links/test_cloud_functions.py", + "providers/tests/google/cloud/links/test_cloud_memorystore.py", + "providers/tests/google/cloud/links/test_cloud_sql.py", + "providers/tests/google/cloud/links/test_cloud_storage_transfer.py", + "providers/tests/google/cloud/links/test_cloud_tasks.py", + "providers/tests/google/cloud/links/test_compute.py", + "providers/tests/google/cloud/links/test_data_loss_prevention.py", + "providers/tests/google/cloud/links/test_datacatalog.py", + "providers/tests/google/cloud/links/test_dataflow.py", + "providers/tests/google/cloud/links/test_dataform.py", + "providers/tests/google/cloud/links/test_datafusion.py", + "providers/tests/google/cloud/links/test_dataplex.py", + "providers/tests/google/cloud/links/test_dataprep.py", + "providers/tests/google/cloud/links/test_dataproc.py", + "providers/tests/google/cloud/links/test_datastore.py", + "providers/tests/google/cloud/links/test_kubernetes_engine.py", + "providers/tests/google/cloud/links/test_life_sciences.py", + "providers/tests/google/cloud/links/test_mlengine.py", + "providers/tests/google/cloud/links/test_pubsub.py", + "providers/tests/google/cloud/links/test_spanner.py", + "providers/tests/google/cloud/links/test_stackdriver.py", + "providers/tests/google/cloud/links/test_vertex_ai.py", + "providers/tests/google/cloud/links/test_workflows.py", + "providers/tests/google/cloud/operators/vertex_ai/test_auto_ml.py", + "providers/tests/google/cloud/operators/vertex_ai/test_batch_prediction_job.py", + "providers/tests/google/cloud/operators/vertex_ai/test_custom_job.py", + "providers/tests/google/cloud/operators/vertex_ai/test_endpoint_service.py", + "providers/tests/google/cloud/operators/vertex_ai/test_hyperparameter_tuning_job.py", + "providers/tests/google/cloud/operators/vertex_ai/test_model_service.py", + "providers/tests/google/cloud/operators/vertex_ai/test_pipeline_job.py", + "providers/tests/google/cloud/sensors/test_dataform.py", + "providers/tests/google/cloud/transfers/test_bigquery_to_sql.py", + "providers/tests/google/cloud/transfers/test_presto_to_gcs.py", + "providers/tests/google/cloud/utils/test_bigquery.py", + "providers/tests/google/cloud/utils/test_bigquery_get_data.py", + "providers/tests/google/cloud/utils/test_dataform.py", + "providers/tests/google/common/links/test_storage.py", + "providers/tests/google/common/test_consts.py", + "providers/tests/google/test_go_module_utils.py", + "providers/tests/microsoft/azure/operators/test_adls.py", + "providers/tests/microsoft/azure/transfers/test_azure_blob_to_gcs.py", + "providers/tests/slack/notifications/test_slack_notifier.py", + "providers/tests/snowflake/triggers/test_snowflake_trigger.py", + "providers/tests/yandex/hooks/test_yandexcloud_dataproc.py", + "providers/tests/yandex/operators/test_yandexcloud_dataproc.py", + "providers/tests/fab/migrations/test_env.py", ] # TODO: Should we extend this test to cover other directories? - modules_files = list(glob.glob(f"{ROOT_FOLDER}/airflow/providers/**/*.py", recursive=True)) + modules_files = PROVIDERS_SRC.joinpath("airflow", "providers").glob("**/*.py") + # Exclude __init__.py + modules_files = filter(lambda f: f.name != "__init__.py", modules_files) # Make path relative - modules_files = list(os.path.relpath(f, ROOT_FOLDER) for f in modules_files) + modules_files = map(lambda f: f.relative_to(PROVIDERS_SRC / "airflow" / "providers"), modules_files) # Exclude example_dags - modules_files = list(f for f in modules_files if "/example_dags/" not in f) + modules_files = (f for f in modules_files if "example_dags" not in f.parts) # Exclude _vendor - modules_files = list(f for f in modules_files if "/_vendor/" not in f) - # Exclude __init__.py - modules_files = list(f for f in modules_files if not f.endswith("__init__.py")) + modules_files = (f for f in modules_files if "_vendor" not in f.parts) # Exclude versions file - modules_files = list(f for f in modules_files if "/versions/" not in f) - # Change airflow/ to tests/ - expected_test_files = list( - f'tests/{f.partition("/")[2]}' for f in modules_files if not f.endswith("__init__.py") - ) + modules_files = (f for f in modules_files if "versions" not in f.parts) + # Change src/airflow/providers/ to tests/ + test_folder = pathlib.Path("providers/tests") # Add test_ prefix to filename - expected_test_files = list( - f'{f.rpartition("/")[0]}/test_{f.rpartition("/")[2]}' - for f in expected_test_files - if not f.endswith("__init__.py") - ) + expected_test_files = (test_folder.joinpath(f.with_name("test_" + f.name)) for f in modules_files) - current_test_files = glob.glob(f"{ROOT_FOLDER}/tests/providers/**/*.py", recursive=True) + current_test_files = PROVIDERS_TESTS.glob("**/*.py") # Make path relative current_test_files = (os.path.relpath(f, ROOT_FOLDER) for f in current_test_files) # Exclude __init__.py @@ -219,9 +209,9 @@ def get_imports_from_file(filepath: str): return import_names -def filepath_to_module(filepath: str): - filepath = os.path.relpath(os.path.abspath(filepath), ROOT_FOLDER) - return filepath.replace("/", ".")[: -(len(".py"))] +def filepath_to_module(path: pathlib.Path): + path = path.relative_to(PROVIDERS_SRC) + return path.as_posix().replace("/", ".")[: -(len(".py"))] def print_sorted(container: set, indent: str = " ") -> None: @@ -235,24 +225,22 @@ class ProjectStructureTest: CLASS_SUFFIXES = ["Operator", "Sensor"] def class_paths(self): - """Override this method if your classes are located under different paths""" for resource_type in self.CLASS_DIRS: - python_files = glob.glob( - f"{ROOT_FOLDER}/airflow/providers/{self.PROVIDER}/**/{resource_type}/**.py", recursive=True + python_files = PROVIDERS_SRC.glob( + f"airflow/providers/{self.PROVIDER}/**/{resource_type}/**/*.py", ) # Make path relative - resource_files = (os.path.relpath(f, ROOT_FOLDER) for f in python_files) - resource_files = (f for f in resource_files if not f.endswith("__init__.py")) + resource_files = filter(lambda f: f.name != "__init__.py", python_files) yield from resource_files def list_of_classes(self): classes = {} for operator_file in self.class_paths(): - operators_paths = self.get_classes_from_file(f"{ROOT_FOLDER}/{operator_file}") + operators_paths = self.get_classes_from_file(operator_file) classes.update(operators_paths) return classes - def get_classes_from_file(self, filepath: str): + def get_classes_from_file(self, filepath: pathlib.Path): with open(filepath) as py_file: content = py_file.read() doc_node = ast.parse(content, filepath) @@ -282,11 +270,12 @@ def example_paths(self): """Override this method if your example dags are located elsewhere""" # old_design: yield from glob.glob( - f"{ROOT_FOLDER}/airflow/providers/{self.PROVIDER}/**/example_dags/example_*.py", recursive=True + f"{ROOT_FOLDER}/providers/src/airflow/providers/{self.PROVIDER}/**/example_dags/example_*.py", + recursive=True, ) # new_design: yield from glob.glob( - f"{ROOT_FOLDER}/tests/system/providers/{self.PROVIDER}/**/example_*.py", recursive=True + f"{ROOT_FOLDER}/providers/tests/system/{self.PROVIDER}/**/example_*.py", recursive=True ) def test_missing_examples(self): diff --git a/tests/always/test_secrets.py b/tests/always/test_secrets.py index 9e1381faf1fef..a8b2296de1ccd 100644 --- a/tests/always/test_secrets.py +++ b/tests/always/test_secrets.py @@ -24,8 +24,9 @@ from airflow.configuration import ensure_secrets_loaded, initialize_secrets_backends from airflow.models import Connection, Variable from airflow.secrets.cache import SecretCache -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_variables + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_variables class TestConnectionsFromSecrets: diff --git a/tests/always/test_secrets_backends.py b/tests/always/test_secrets_backends.py index e862ed3263a6a..c5093d2e562a7 100644 --- a/tests/always/test_secrets_backends.py +++ b/tests/always/test_secrets_backends.py @@ -28,7 +28,8 @@ from airflow.secrets.environment_variables import EnvironmentVariablesBackend from airflow.secrets.metastore import MetastoreBackend from airflow.utils.session import create_session -from tests.test_utils.db import clear_db_connections, clear_db_variables + +from dev.tests_common.test_utils.db import clear_db_connections, clear_db_variables pytestmark = pytest.mark.db_test diff --git a/tests/always/test_secrets_local_filesystem.py b/tests/always/test_secrets_local_filesystem.py index 44b93dedb8222..40472ec64d5f5 100644 --- a/tests/always/test_secrets_local_filesystem.py +++ b/tests/always/test_secrets_local_filesystem.py @@ -29,7 +29,8 @@ from airflow.models import Variable from airflow.secrets import local_filesystem from airflow.secrets.local_filesystem import LocalFilesystemBackend -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars @contextmanager diff --git a/tests/api_connexion/conftest.py b/tests/api_connexion/conftest.py index 6a23b2cf11d93..8ab76fbb76298 100644 --- a/tests/api_connexion/conftest.py +++ b/tests/api_connexion/conftest.py @@ -19,8 +19,9 @@ import pytest from airflow.www import app -from tests.test_utils.config import conf_vars -from tests.test_utils.decorators import dont_initialize_flask_app_submodules + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules @pytest.fixture(scope="session") @@ -38,7 +39,7 @@ def minimal_app_for_api(): def factory(): with conf_vars( { - ("api", "auth_backends"): "tests.test_utils.remote_user_api_auth_backend", + ("api", "auth_backends"): "dev.tests_common.test_utils.remote_user_api_auth_backend", ( "core", "auth_manager", diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index 40ad59067f594..39b0129c3317d 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -36,11 +36,12 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_assets, clear_db_runs -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_assets, clear_db_runs +from dev.tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_backfill_endpoint.py b/tests/api_connexion/endpoints/test_backfill_endpoint.py index dd086339b73ac..67ec6316e2ffd 100644 --- a/tests/api_connexion/endpoints/test_backfill_endpoint.py +++ b/tests/api_connexion/endpoints/test_backfill_endpoint.py @@ -30,8 +30,14 @@ from airflow.operators.empty import EmptyOperator from airflow.utils import timezone from airflow.utils.session import provide_session -from tests.test_utils.api_connexion_utils import create_user, delete_user -from tests.test_utils.db import clear_db_backfills, clear_db_dags, clear_db_runs, clear_db_serialized_dags + +from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user +from dev.tests_common.test_utils.db import ( + clear_db_backfills, + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, +) pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag] diff --git a/tests/api_connexion/endpoints/test_config_endpoint.py b/tests/api_connexion/endpoints/test_config_endpoint.py index bd88c491c952b..c46a6571c23b9 100644 --- a/tests/api_connexion/endpoints/test_config_endpoint.py +++ b/tests/api_connexion/endpoints/test_config_endpoint.py @@ -21,8 +21,8 @@ import pytest -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.config import conf_vars +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_connection_endpoint.py b/tests/api_connexion/endpoints/test_connection_endpoint.py index a140046656e31..fb2d53074cd87 100644 --- a/tests/api_connexion/endpoints/test_connection_endpoint.py +++ b/tests/api_connexion/endpoints/test_connection_endpoint.py @@ -25,10 +25,11 @@ from airflow.models import Connection from airflow.secrets.environment_variables import CONN_ENV_PREFIX from airflow.utils.session import provide_session -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_connections -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_connections +from dev.tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py index 907009d26df89..86e0a3b4b3ec6 100644 --- a/tests/api_connexion/endpoints/test_dag_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_endpoint.py @@ -30,10 +30,11 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.session import provide_session from airflow.utils.state import TaskInstanceState -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from dev.tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_parsing.py b/tests/api_connexion/endpoints/test_dag_parsing.py index ae42a565dd052..0fca8ee63c973 100644 --- a/tests/api_connexion/endpoints/test_dag_parsing.py +++ b/tests/api_connexion/endpoints/test_dag_parsing.py @@ -24,8 +24,9 @@ from airflow.models import DagBag from airflow.models.dagbag import DagPriorityParsingRequest -from tests.test_utils.api_connexion_utils import create_user, delete_user -from tests.test_utils.db import clear_db_dag_parsing_requests + +from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_dag_parsing_requests pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_run_endpoint.py b/tests/api_connexion/endpoints/test_dag_run_endpoint.py index 6262d4e83479e..b881f5ea98638 100644 --- a/tests/api_connexion/endpoints/test_dag_run_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_run_endpoint.py @@ -34,11 +34,12 @@ from airflow.utils.session import create_session, provide_session from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from dev.tests_common.test_utils.www import _check_last_log if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/api_connexion/endpoints/test_dag_source_endpoint.py b/tests/api_connexion/endpoints/test_dag_source_endpoint.py index f4df56ba629ae..80f32dc715a3c 100644 --- a/tests/api_connexion/endpoints/test_dag_source_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_source_endpoint.py @@ -23,8 +23,9 @@ import pytest from airflow.models import DagBag -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py index 9ab5b49765931..a447e2a6a4b23 100644 --- a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py @@ -26,8 +26,9 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import create_user, delete_user -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags + +from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_warning_endpoint.py b/tests/api_connexion/endpoints/test_dag_warning_endpoint.py index f156d8921c0e6..b9c00991413bb 100644 --- a/tests/api_connexion/endpoints/test_dag_warning_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_warning_endpoint.py @@ -23,8 +23,9 @@ from airflow.models.dag import DagModel from airflow.models.dagwarning import DagWarning from airflow.utils.session import create_session -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.db import clear_db_dag_warnings, clear_db_dags + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_event_log_endpoint.py b/tests/api_connexion/endpoints/test_event_log_endpoint.py index e5ca3d301765a..42a610890ede2 100644 --- a/tests/api_connexion/endpoints/test_event_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_event_log_endpoint.py @@ -21,9 +21,10 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import Log from airflow.utils import timezone -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_logs + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_logs pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_extra_link_endpoint.py b/tests/api_connexion/endpoints/test_extra_link_endpoint.py index 2c3eacdc91dc0..373571a42ee47 100644 --- a/tests/api_connexion/endpoints/test_extra_link_endpoint.py +++ b/tests/api_connexion/endpoints/test_extra_link_endpoint.py @@ -30,11 +30,12 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import create_user, delete_user -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink -from tests.test_utils.db import clear_db_runs, clear_db_xcom -from tests.test_utils.mock_operators import CustomOperator -from tests.test_utils.mock_plugins import mock_plugin_manager + +from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom +from dev.tests_common.test_utils.mock_operators import CustomOperator +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/api_connexion/endpoints/test_import_error_endpoint.py b/tests/api_connexion/endpoints/test_import_error_endpoint.py index af2b83ebb1eed..f4f7d03b323f2 100644 --- a/tests/api_connexion/endpoints/test_import_error_endpoint.py +++ b/tests/api_connexion/endpoints/test_import_error_endpoint.py @@ -23,10 +23,11 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.utils import timezone from airflow.utils.session import provide_session -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.compat import ParseImportError -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_import_errors + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.compat import ParseImportError +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_import_errors pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_log_endpoint.py b/tests/api_connexion/endpoints/test_log_endpoint.py index 2b112e3221843..21d4e8feb83ad 100644 --- a/tests/api_connexion/endpoints/test_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_log_endpoint.py @@ -32,8 +32,9 @@ from airflow.operators.empty import EmptyOperator from airflow.utils import timezone from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_runs pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py index fc53b8952f4aa..e41bbcc6c2e88 100644 --- a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py @@ -32,9 +32,10 @@ from airflow.utils.session import provide_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from dev.tests_common.test_utils.mock_operators import MockOperator pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_plugin_endpoint.py b/tests/api_connexion/endpoints/test_plugin_endpoint.py index 6670c684cb45d..2831af06bf3d9 100644 --- a/tests/api_connexion/endpoints/test_plugin_endpoint.py +++ b/tests/api_connexion/endpoints/test_plugin_endpoint.py @@ -28,10 +28,11 @@ from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.timetables.base import Timetable from airflow.utils.module_loading import qualname -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.compat import BaseOperatorLink -from tests.test_utils.config import conf_vars -from tests.test_utils.mock_plugins import mock_plugin_manager + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.compat import BaseOperatorLink +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_pool_endpoint.py b/tests/api_connexion/endpoints/test_pool_endpoint.py index 2cc095d077aa9..455c417bd2581 100644 --- a/tests/api_connexion/endpoints/test_pool_endpoint.py +++ b/tests/api_connexion/endpoints/test_pool_endpoint.py @@ -21,10 +21,11 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models.pool import Pool from airflow.utils.session import provide_session -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_pools -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_pools +from dev.tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_provider_endpoint.py b/tests/api_connexion/endpoints/test_provider_endpoint.py index b4cf8f10a92ae..92e14887ec038 100644 --- a/tests/api_connexion/endpoints/test_provider_endpoint.py +++ b/tests/api_connexion/endpoints/test_provider_endpoint.py @@ -21,7 +21,8 @@ import pytest from airflow.providers_manager import ProviderInfo -from tests.test_utils.api_connexion_utils import create_user, delete_user + +from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_task_endpoint.py b/tests/api_connexion/endpoints/test_task_endpoint.py index b2e068bd507fe..60237946a82ca 100644 --- a/tests/api_connexion/endpoints/test_task_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_endpoint.py @@ -27,8 +27,9 @@ from airflow.models.expandinput import EXPAND_INPUT_EMPTY from airflow.models.serialized_dag import SerializedDagModel from airflow.operators.empty import EmptyOperator -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_task_instance_endpoint.py index b5b3163e988d0..809c2fab9aa2d 100644 --- a/tests/api_connexion/endpoints/test_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_instance_endpoint.py @@ -35,9 +35,10 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from dev.tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_variable_endpoint.py b/tests/api_connexion/endpoints/test_variable_endpoint.py index aa5f7c99674f8..e4302636184d3 100644 --- a/tests/api_connexion/endpoints/test_variable_endpoint.py +++ b/tests/api_connexion/endpoints/test_variable_endpoint.py @@ -22,10 +22,11 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import Variable -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_variables -from tests.test_utils.www import _check_last_log + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_variables +from dev.tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_xcom_endpoint.py b/tests/api_connexion/endpoints/test_xcom_endpoint.py index 809e537f9f88d..0293e672c47d2 100644 --- a/tests/api_connexion/endpoints/test_xcom_endpoint.py +++ b/tests/api_connexion/endpoints/test_xcom_endpoint.py @@ -30,9 +30,10 @@ from airflow.utils.session import create_session from airflow.utils.timezone import utcnow from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom + +from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_connection_schema.py b/tests/api_connexion/schemas/test_connection_schema.py index cf614eb3b114e..d7a0de4e89f05 100644 --- a/tests/api_connexion/schemas/test_connection_schema.py +++ b/tests/api_connexion/schemas/test_connection_schema.py @@ -30,7 +30,8 @@ ) from airflow.models import Connection from airflow.utils.session import create_session, provide_session -from tests.test_utils.db import clear_db_connections + +from dev.tests_common.test_utils.db import clear_db_connections pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_dag_run_schema.py b/tests/api_connexion/schemas/test_dag_run_schema.py index 01ed33fd59886..b874ade4442ff 100644 --- a/tests/api_connexion/schemas/test_dag_run_schema.py +++ b/tests/api_connexion/schemas/test_dag_run_schema.py @@ -29,8 +29,9 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/api_connexion/schemas/test_dataset_schema.py b/tests/api_connexion/schemas/test_dataset_schema.py index 4a6fdf6e6d513..769de6f1b10eb 100644 --- a/tests/api_connexion/schemas/test_dataset_schema.py +++ b/tests/api_connexion/schemas/test_dataset_schema.py @@ -30,7 +30,8 @@ from airflow.assets import Asset from airflow.models.asset import AssetAliasModel, AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator -from tests.test_utils.db import clear_db_assets, clear_db_dags + +from dev.tests_common.test_utils.db import clear_db_assets, clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_error_schema.py b/tests/api_connexion/schemas/test_error_schema.py index fb5e9ea219d22..5c358f2e2c374 100644 --- a/tests/api_connexion/schemas/test_error_schema.py +++ b/tests/api_connexion/schemas/test_error_schema.py @@ -25,8 +25,9 @@ ) from airflow.utils import timezone from airflow.utils.session import provide_session -from tests.test_utils.compat import ParseImportError -from tests.test_utils.db import clear_db_import_errors + +from dev.tests_common.test_utils.compat import ParseImportError +from dev.tests_common.test_utils.db import clear_db_import_errors pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_plugin_schema.py b/tests/api_connexion/schemas/test_plugin_schema.py index 199db02c7268a..ee079ce664348 100644 --- a/tests/api_connexion/schemas/test_plugin_schema.py +++ b/tests/api_connexion/schemas/test_plugin_schema.py @@ -28,7 +28,8 @@ ) from airflow.hooks.base import BaseHook from airflow.plugins_manager import AirflowPlugin -from tests.test_utils.compat import BaseOperatorLink + +from dev.tests_common.test_utils.compat import BaseOperatorLink class PluginHook(BaseHook): ... diff --git a/tests/api_connexion/schemas/test_pool_schemas.py b/tests/api_connexion/schemas/test_pool_schemas.py index 110103073aaa5..23788426c5018 100644 --- a/tests/api_connexion/schemas/test_pool_schemas.py +++ b/tests/api_connexion/schemas/test_pool_schemas.py @@ -21,7 +21,8 @@ from airflow.api_connexion.schemas.pool_schema import PoolCollection, pool_collection_schema, pool_schema from airflow.models.pool import Pool from airflow.utils.session import provide_session -from tests.test_utils.db import clear_db_pools + +from dev.tests_common.test_utils.db import clear_db_pools pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_xcom_schema.py b/tests/api_connexion/schemas/test_xcom_schema.py index e1f97dad0520d..06c36c31b7767 100644 --- a/tests/api_connexion/schemas/test_xcom_schema.py +++ b/tests/api_connexion/schemas/test_xcom_schema.py @@ -30,7 +30,8 @@ from airflow.models import DagRun, XCom from airflow.utils.dates import parse_execution_date from airflow.utils.session import create_session -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/test_auth.py b/tests/api_connexion/test_auth.py index 54e5632ad84d1..e568d618ad662 100644 --- a/tests/api_connexion/test_auth.py +++ b/tests/api_connexion/test_auth.py @@ -22,9 +22,10 @@ from airflow.auth.managers.simple.simple_auth_manager import SimpleAuthManager from airflow.auth.managers.simple.user import SimpleAuthManagerUser -from tests.test_utils.api_connexion_utils import assert_401 -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_pools + +from dev.tests_common.test_utils.api_connexion_utils import assert_401 +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_pools pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/test_parameters.py b/tests/api_connexion/test_parameters.py index d5fdbb79b9cc1..57c9c05415f65 100644 --- a/tests/api_connexion/test_parameters.py +++ b/tests/api_connexion/test_parameters.py @@ -29,7 +29,8 @@ validate_istimezone, ) from airflow.utils import timezone -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars class TestValidateIsTimezone: diff --git a/tests/api_connexion/test_security.py b/tests/api_connexion/test_security.py index c6a112b1a1bb9..2051cf613510e 100644 --- a/tests/api_connexion/test_security.py +++ b/tests/api_connexion/test_security.py @@ -18,7 +18,7 @@ import pytest -from tests.test_utils.api_connexion_utils import create_user, delete_user +from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_fastapi/views/public/test_connections.py b/tests/api_fastapi/views/public/test_connections.py index 0b76fdbe4f1b2..a5cb301a43136 100644 --- a/tests/api_fastapi/views/public/test_connections.py +++ b/tests/api_fastapi/views/public/test_connections.py @@ -20,7 +20,8 @@ from airflow.models import Connection from airflow.utils.session import provide_session -from tests.test_utils.db import clear_db_connections + +from dev.tests_common.test_utils.db import clear_db_connections pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/public/test_dags.py b/tests/api_fastapi/views/public/test_dags.py index 58b3daf35c64a..7ac93a2f2e07d 100644 --- a/tests/api_fastapi/views/public/test_dags.py +++ b/tests/api_fastapi/views/public/test_dags.py @@ -27,7 +27,8 @@ from airflow.utils.session import provide_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/public/test_variables.py b/tests/api_fastapi/views/public/test_variables.py index 0ee2cebd77340..7957d0bf22249 100644 --- a/tests/api_fastapi/views/public/test_variables.py +++ b/tests/api_fastapi/views/public/test_variables.py @@ -20,7 +20,8 @@ from airflow.models.variable import Variable from airflow.utils.session import provide_session -from tests.test_utils.db import clear_db_variables + +from dev.tests_common.test_utils.db import clear_db_variables pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/ui/test_assets.py b/tests/api_fastapi/views/ui/test_assets.py index 99cd9bc1e9960..d0e93c709b076 100644 --- a/tests/api_fastapi/views/ui/test_assets.py +++ b/tests/api_fastapi/views/ui/test_assets.py @@ -20,7 +20,8 @@ from airflow.assets import Asset from airflow.operators.empty import EmptyOperator -from tests.conftest import initial_db_init + +from dev.tests_common.test_utils.db import initial_db_init pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/ui/test_dashboard.py b/tests/api_fastapi/views/ui/test_dashboard.py index 970b79ad3568c..bee683535704f 100644 --- a/tests/api_fastapi/views/ui/test_dashboard.py +++ b/tests/api_fastapi/views/ui/test_dashboard.py @@ -26,7 +26,8 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/api_internal/endpoints/test_rpc_api_endpoint.py b/tests/api_internal/endpoints/test_rpc_api_endpoint.py index f12e0ae087bbb..364b0f44eaf27 100644 --- a/tests/api_internal/endpoints/test_rpc_api_endpoint.py +++ b/tests/api_internal/endpoints/test_rpc_api_endpoint.py @@ -34,8 +34,9 @@ from airflow.utils.jwt_signer import JWTSigner from airflow.utils.state import State from airflow.www import app -from tests.test_utils.config import conf_vars -from tests.test_utils.decorators import dont_initialize_flask_app_submodules + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules # Note: Sounds a bit strange to disable internal API tests in isolation mode but... # As long as the test is modelled to run its own internal API endpoints, it is conflicting diff --git a/tests/api_internal/test_internal_api_call.py b/tests/api_internal/test_internal_api_call.py index d779b504ea479..02ae2d9f55125 100644 --- a/tests/api_internal/test_internal_api_call.py +++ b/tests/api_internal/test_internal_api_call.py @@ -34,7 +34,8 @@ from airflow.serialization.serialized_objects import BaseSerialization from airflow.settings import _ENABLE_AIP_44 from airflow.utils.state import State -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars if TYPE_CHECKING: from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic diff --git a/tests/assets/test_manager.py b/tests/assets/test_manager.py index 950949fe474e3..cd4563d9a9c4e 100644 --- a/tests/assets/test_manager.py +++ b/tests/assets/test_manager.py @@ -48,7 +48,7 @@ @pytest.fixture def clear_assets(): - from tests.test_utils.db import clear_db_assets + from dev.tests_common.test_utils.db import clear_db_assets clear_db_assets() yield diff --git a/tests/assets/tests_asset.py b/tests/assets/tests_asset.py index afbb46827f3a9..10d91e7a00fb7 100644 --- a/tests/assets/tests_asset.py +++ b/tests/assets/tests_asset.py @@ -43,7 +43,7 @@ @pytest.fixture def clear_assets(): - from tests.test_utils.db import clear_db_assets + from dev.tests_common.test_utils.db import clear_db_assets clear_db_assets() yield diff --git a/tests/auth/managers/simple/views/test_auth.py b/tests/auth/managers/simple/views/test_auth.py index 197ed0e615cae..a4e2a12fdcf1c 100644 --- a/tests/auth/managers/simple/views/test_auth.py +++ b/tests/auth/managers/simple/views/test_auth.py @@ -23,7 +23,8 @@ from airflow.auth.managers.simple.simple_auth_manager import SimpleAuthManager from airflow.www import app as application -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars @pytest.fixture diff --git a/tests/cli/commands/test_celery_command.py b/tests/cli/commands/test_celery_command.py index 476e6279fbf12..c29d835d8ee07 100644 --- a/tests/cli/commands/test_celery_command.py +++ b/tests/cli/commands/test_celery_command.py @@ -30,7 +30,8 @@ from airflow.cli.commands import celery_command from airflow.configuration import conf from airflow.executors import executor_loader -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_config_command.py b/tests/cli/commands/test_config_command.py index 030303c28ec4d..90d38fa27ad03 100644 --- a/tests/cli/commands/test_config_command.py +++ b/tests/cli/commands/test_config_command.py @@ -22,7 +22,8 @@ from airflow.cli import cli_parser from airflow.cli.commands import config_command -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars STATSD_CONFIG_BEGIN_WITH = "# `StatsD `" diff --git a/tests/cli/commands/test_connection_command.py b/tests/cli/commands/test_connection_command.py index fbfd3d2945e4a..06578eb1492c1 100644 --- a/tests/cli/commands/test_connection_command.py +++ b/tests/cli/commands/test_connection_command.py @@ -33,7 +33,8 @@ from airflow.models import Connection from airflow.utils.db import merge_conn from airflow.utils.session import create_session -from tests.test_utils.db import clear_db_connections + +from dev.tests_common.test_utils.db import clear_db_connections pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py index 338ae44a2b95a..00364794f1d53 100644 --- a/tests/cli/commands/test_dag_command.py +++ b/tests/cli/commands/test_dag_command.py @@ -47,8 +47,9 @@ from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType from tests.models import TEST_DAGS_FOLDER -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs DEFAULT_DATE = timezone.make_aware(datetime(2015, 1, 1), timezone=timezone.utc) if pendulum.__version__.startswith("3"): diff --git a/tests/cli/commands/test_dag_processor_command.py b/tests/cli/commands/test_dag_processor_command.py index 2b84f506e78f1..643710ce1e415 100644 --- a/tests/cli/commands/test_dag_processor_command.py +++ b/tests/cli/commands/test_dag_processor_command.py @@ -24,7 +24,8 @@ from airflow.cli import cli_parser from airflow.cli.commands import dag_processor_command from airflow.configuration import conf -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_info_command.py b/tests/cli/commands/test_info_command.py index 44287bb0ec162..c6bea2d8a5bb8 100644 --- a/tests/cli/commands/test_info_command.py +++ b/tests/cli/commands/test_info_command.py @@ -32,7 +32,8 @@ from airflow.config_templates import airflow_local_settings from airflow.logging_config import configure_logging from airflow.version import version as airflow_version -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars def capture_show_output(instance): diff --git a/tests/cli/commands/test_internal_api_command.py b/tests/cli/commands/test_internal_api_command.py index 194f7874839c8..11123c7c211a1 100644 --- a/tests/cli/commands/test_internal_api_command.py +++ b/tests/cli/commands/test_internal_api_command.py @@ -32,7 +32,8 @@ from airflow.cli.commands.internal_api_command import GunicornMonitor from airflow.settings import _ENABLE_AIP_44 from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars console = Console(width=400, color_system="standard") diff --git a/tests/cli/commands/test_jobs_command.py b/tests/cli/commands/test_jobs_command.py index 28c739a2f2607..61540a68c51c6 100644 --- a/tests/cli/commands/test_jobs_command.py +++ b/tests/cli/commands/test_jobs_command.py @@ -27,7 +27,8 @@ from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.utils.session import create_session from airflow.utils.state import JobState, State -from tests.test_utils.db import clear_db_jobs + +from dev.tests_common.test_utils.db import clear_db_jobs @pytest.mark.skip_if_database_isolation_mode diff --git a/tests/cli/commands/test_kerberos_command.py b/tests/cli/commands/test_kerberos_command.py index 35e90fb54923d..4a5e73ece788f 100644 --- a/tests/cli/commands/test_kerberos_command.py +++ b/tests/cli/commands/test_kerberos_command.py @@ -23,7 +23,8 @@ from airflow.cli import cli_parser from airflow.cli.commands import kerberos_command from airflow.security.kerberos import KerberosMode -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_kubernetes_command.py b/tests/cli/commands/test_kubernetes_command.py index d489aa3ea93f1..1e53bdc77d8d8 100644 --- a/tests/cli/commands/test_kubernetes_command.py +++ b/tests/cli/commands/test_kubernetes_command.py @@ -28,7 +28,8 @@ from airflow.cli import cli_parser from airflow.cli.commands import kubernetes_command from airflow.executors import executor_loader -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_plugins_command.py b/tests/cli/commands/test_plugins_command.py index 288b4453bb159..703b6879d7a3a 100644 --- a/tests/cli/commands/test_plugins_command.py +++ b/tests/cli/commands/test_plugins_command.py @@ -29,7 +29,8 @@ from airflow.listeners.listener import get_listener_manager from airflow.plugins_manager import AirflowPlugin from tests.plugins.test_plugin import AirflowTestPlugin as ComplexAirflowPlugin -from tests.test_utils.mock_plugins import mock_plugin_manager + +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] @@ -88,15 +89,15 @@ def test_should_display_one_plugin(self): } ], "global_operator_extra_links": [ - "", - "", + "", + "", ], "timetables": ["tests.plugins.test_plugin.CustomCronDataIntervalTimetable"], "operator_extra_links": [ - "", - "", - "", - "", + "", + "", + "", + "", ], "hooks": ["tests.plugins.test_plugin.PluginHook"], "listeners": [ diff --git a/tests/cli/commands/test_rotate_fernet_key_command.py b/tests/cli/commands/test_rotate_fernet_key_command.py index d45753d75f70d..d638c2010e187 100644 --- a/tests/cli/commands/test_rotate_fernet_key_command.py +++ b/tests/cli/commands/test_rotate_fernet_key_command.py @@ -26,8 +26,9 @@ from airflow.hooks.base import BaseHook from airflow.models import Connection, Variable from airflow.utils.session import provide_session -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_connections, clear_db_variables + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_connections, clear_db_variables pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_scheduler_command.py b/tests/cli/commands/test_scheduler_command.py index 2e28196cb5c3c..8fce5d6e289b3 100644 --- a/tests/cli/commands/test_scheduler_command.py +++ b/tests/cli/commands/test_scheduler_command.py @@ -29,7 +29,8 @@ from airflow.executors import executor_loader from airflow.utils.scheduler_health import HealthServer, serve_health_check from airflow.utils.serve_logs import serve_logs -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index 3397005b80eb0..fb512792eb324 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -51,9 +51,10 @@ from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_pools, clear_db_runs + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_pools, clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/cli/commands/test_variable_command.py b/tests/cli/commands/test_variable_command.py index f95e7b1648277..d35184c73c80a 100644 --- a/tests/cli/commands/test_variable_command.py +++ b/tests/cli/commands/test_variable_command.py @@ -29,7 +29,8 @@ from airflow.cli.commands import variable_command from airflow.models import Variable from airflow.utils.session import create_session -from tests.test_utils.db import clear_db_variables + +from dev.tests_common.test_utils.db import clear_db_variables pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_webserver_command.py b/tests/cli/commands/test_webserver_command.py index 5531f674689b9..b72f4e4596035 100644 --- a/tests/cli/commands/test_webserver_command.py +++ b/tests/cli/commands/test_webserver_command.py @@ -31,7 +31,8 @@ from airflow.cli.commands import webserver_command from airflow.cli.commands.webserver_command import GunicornMonitor from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars console = Console(width=400, color_system="standard") diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 9f0a63af06978..61e27b0e74de4 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -21,12 +21,12 @@ import pytest -from airflow.cli import cli_parser from airflow.executors import local_executor from airflow.models.dagbag import DagBag from airflow.providers.celery.executors import celery_executor, celery_kubernetes_executor from airflow.providers.cncf.kubernetes.executors import kubernetes_executor, local_kubernetes_executor -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars # Create custom executors here because conftest is imported first custom_executor_module = type(sys)("custom_executor") @@ -61,4 +61,6 @@ def dagbag(): @pytest.fixture(scope="session") def parser(): + from airflow.cli import cli_parser + return cli_parser.get_parser() diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index 2244b6dbd5860..46ac36803face 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -43,7 +43,8 @@ from airflow.executors.local_executor import LocalExecutor from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/conftest.py b/tests/conftest.py index 60d009416fe8e..f956865e7a1c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,126 +16,33 @@ # under the License. from __future__ import annotations -import json import os -import platform -import re -import subprocess import sys -from contextlib import ExitStack, suppress -from datetime import datetime, timedelta, timezone -from pathlib import Path from typing import TYPE_CHECKING import pytest -import time_machine -from itsdangerous import URLSafeSerializer - -if TYPE_CHECKING: - from tests._internals.capture_warnings import CaptureWarningsPlugin # noqa: F401 - from tests._internals.forbidden_warnings import ForbiddenWarningsPlugin # noqa: F401 # We should set these before loading _any_ of the rest of airflow so that the # unit test mode config is set as early as possible. assert "airflow" not in sys.modules, "No airflow module can be imported before these lines" -keep_env_variables = "--keep-env-variables" in sys.argv - -if not keep_env_variables: - # Clear all Environment Variables that might have side effect, - # For example, defined in /files/airflow-breeze-config/variables.env - _AIRFLOW_CONFIG_PATTERN = re.compile(r"^AIRFLOW__(.+)__(.+)$") - _KEEP_CONFIGS_SETTINGS: dict[str, dict[str, set[str]]] = { - # Keep always these configurations - "always": { - "database": {"sql_alchemy_conn"}, - "core": {"sql_alchemy_conn"}, - "celery": {"result_backend", "broker_url"}, - }, - # Keep per enabled integrations - "celery": {"celery": {"*"}, "celery_broker_transport_options": {"*"}}, - "kerberos": {"kerberos": {"*"}}, - } - if os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true": - _KEEP_CONFIGS_SETTINGS["always"].update( - { - "core": { - "internal_api_url", - "fernet_key", - "database_access_isolation", - "internal_api_secret_key", - "internal_api_clock_grace", - }, - } - ) - _ENABLED_INTEGRATIONS = {e.split("_", 1)[-1].lower() for e in os.environ if e.startswith("INTEGRATION_")} - _KEEP_CONFIGS: dict[str, set[str]] = {} - for keep_settings_key in ("always", *_ENABLED_INTEGRATIONS): - if keep_settings := _KEEP_CONFIGS_SETTINGS.get(keep_settings_key): - for section, options in keep_settings.items(): - if section not in _KEEP_CONFIGS: - _KEEP_CONFIGS[section] = options - else: - _KEEP_CONFIGS[section].update(options) - for env_key in os.environ.copy(): - if m := _AIRFLOW_CONFIG_PATTERN.match(env_key): - section, option = m.group(1).lower(), m.group(2).lower() - if not (ko := _KEEP_CONFIGS.get(section)) or not ("*" in ko or option in ko): - del os.environ[env_key] - -SUPPORTED_DB_BACKENDS = ("sqlite", "postgres", "mysql") - -# A bit of a Hack - but we need to check args before they are parsed by pytest in order to -# configure the DB before Airflow gets initialized (which happens at airflow import time). -# Using env variables also handles the case, when python-xdist is used - python-xdist spawns separate -# processes and does not pass all args to them (it's done via env variables) so we are doing the -# same here and detect whether `--skip-db-tests` or `--run-db-tests-only` is passed to pytest -# and set env variables so the processes spawned by python-xdist can read the status from there -skip_db_tests = "--skip-db-tests" in sys.argv or os.environ.get("_AIRFLOW_SKIP_DB_TESTS") == "true" -run_db_tests_only = ( - "--run-db-tests-only" in sys.argv or os.environ.get("_AIRFLOW_RUN_DB_TESTS_ONLY") == "true" -) - -if skip_db_tests: - if run_db_tests_only: - raise Exception("You cannot specify both --skip-db-tests and --run-db-tests-only together") - # Make sure sqlalchemy will not be usable for pure unit tests even if initialized - os.environ["AIRFLOW__CORE__SQL_ALCHEMY_CONN"] = "bad_schema:///" - os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"] = "bad_schema:///" - os.environ["_IN_UNIT_TESTS"] = "true" - # Set it here to pass the flag to python-xdist spawned processes - os.environ["_AIRFLOW_SKIP_DB_TESTS"] = "true" - -if run_db_tests_only: - # Set it here to pass the flag to python-xdist spawned processes - os.environ["_AIRFLOW_RUN_DB_TESTS_ONLY"] = "true" - -AIRFLOW_TESTS_DIR = Path(os.path.dirname(os.path.realpath(__file__))).resolve() -AIRFLOW_SOURCES_ROOT_DIR = AIRFLOW_TESTS_DIR.parent.parent - -os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = os.fspath(AIRFLOW_TESTS_DIR / "plugins") -os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = os.fspath(AIRFLOW_TESTS_DIR / "dags") -os.environ["AIRFLOW__CORE__UNIT_TEST_MODE"] = "True" -os.environ["AWS_DEFAULT_REGION"] = os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" -os.environ["CREDENTIALS_DIR"] = os.environ.get("CREDENTIALS_DIR") or "/files/airflow-breeze-config/keys" -os.environ["AIRFLOW_ENABLE_AIP_44"] = os.environ.get("AIRFLOW_ENABLE_AIP_44") or "true" - -if platform.system() == "Darwin": - # mocks from unittest.mock work correctly in subprocesses only if they are created by "fork" method - # but macOS uses "spawn" by default - os.environ["AIRFLOW__CORE__MP_START_METHOD"] = "fork" +pytest_plugins = "dev.tests_common.pytest_plugin" # Ignore files that are really test dags to be ignored by pytest collect_ignore = [ "tests/dags/subdir1/test_ignore_this.py", "tests/dags/test_invalid_dup_task.py", "tests/dags_corrupted/test_impersonation_custom.py", - "tests/test_utils/perf/dags/elastic_dag.py", + "dev.tests_common.test_utils/perf/dags/elastic_dag.py", ] -# https://docs.pytest.org/en/stable/reference/reference.html#stash -capture_warnings_key = pytest.StashKey["CaptureWarningsPlugin"]() -forbidden_warnings_key = pytest.StashKey["ForbiddenWarningsPlugin"]() + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + dep_path = [config.rootpath.joinpath("tests", "deprecations_ignore.yml")] + config.inicfg["airflow_deprecations_ignore"] = ( + config.inicfg.get("airflow_deprecations_ignore", []) + dep_path # type: ignore[assignment,operator] + ) @pytest.fixture @@ -151,1308 +58,6 @@ def reset_environment(): os.environ[key] = init_env[key] -@pytest.fixture -def secret_key() -> str: - """Return secret key configured.""" - from airflow.configuration import conf - - the_key = conf.get("webserver", "SECRET_KEY") - if the_key is None: - raise RuntimeError( - "The secret key SHOULD be configured as `[webserver] secret_key` in the " - "configuration/environment at this stage! " - ) - return the_key - - -@pytest.fixture -def url_safe_serializer(secret_key) -> URLSafeSerializer: - return URLSafeSerializer(secret_key) - - -@pytest.fixture -def reset_db(): - """Resets Airflow db.""" - - from airflow.utils import db - - db.resetdb() - - -ALLOWED_TRACE_SQL_COLUMNS = ["num", "time", "trace", "sql", "parameters", "count"] - - -@pytest.fixture(autouse=True) -def trace_sql(request): - from tests.test_utils.perf.perf_kit.sqlalchemy import ( # isort: skip - count_queries, - trace_queries, - ) - - """Displays queries from the tests to console.""" - trace_sql_option = request.config.option.trace_sql - if not trace_sql_option: - yield - return - - terminal_reporter = request.config.pluginmanager.getplugin("terminalreporter") - # if no terminal reporter plugin is present, nothing we can do here; - # this can happen when this function executes in a worker node - # when using pytest-xdist, for example - if terminal_reporter is None: - yield - return - - columns = [col.strip() for col in trace_sql_option.split(",")] - - def pytest_print(text): - return terminal_reporter.write_line(text) - - with ExitStack() as exit_stack: - if columns == ["num"]: - # It is very unlikely that the user wants to display only numbers, but probably - # the user just wants to count the queries. - exit_stack.enter_context(count_queries(print_fn=pytest_print)) - elif any(c in columns for c in ["time", "trace", "sql", "parameters"]): - exit_stack.enter_context( - trace_queries( - display_num="num" in columns, - display_time="time" in columns, - display_trace="trace" in columns, - display_sql="sql" in columns, - display_parameters="parameters" in columns, - print_fn=pytest_print, - ) - ) - - yield - - -@pytest.fixture(autouse=True, scope="session") -def set_db_isolation_mode(): - if os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true": - from airflow.api_internal.internal_api_call import InternalApiConfig - - InternalApiConfig.set_use_internal_api("tests", allow_tests_to_use_db=True) - - -def skip_if_database_isolation_mode(item): - if os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true": - for _ in item.iter_markers(name="skip_if_database_isolation_mode"): - pytest.skip("This test is skipped because it is not allowed in database isolation mode.") - - -def pytest_addoption(parser: pytest.Parser): - """Add options parser for custom plugins.""" - group = parser.getgroup("airflow") - group.addoption( - "--with-db-init", - action="store_true", - dest="db_init", - help="Forces database initialization before tests", - ) - group.addoption( - "--integration", - action="append", - dest="integration", - metavar="INTEGRATIONS", - help="only run tests matching integration specified: " - "[cassandra,kerberos,mongo,celery,statsd,trino]. ", - ) - group.addoption( - "--keep-env-variables", - action="store_true", - dest="keep_env_variables", - help="do not clear environment variables that might have side effect while running tests", - ) - group.addoption( - "--skip-db-tests", - action="store_true", - dest="skip_db_tests", - help="skip tests that require database", - ) - group.addoption( - "--run-db-tests-only", - action="store_true", - dest="run_db_tests_only", - help="only run tests requiring database", - ) - group.addoption( - "--backend", - action="store", - dest="backend", - metavar="BACKEND", - help="only run tests matching the backend: [sqlite,postgres,mysql].", - ) - group.addoption( - "--system", - action="append", - dest="system", - metavar="SYSTEMS", - help="only run tests matching the system specified [google.cloud, google.marketing_platform]", - ) - group.addoption( - "--include-long-running", - action="store_true", - dest="include_long_running", - help="Includes long running tests (marked with long_running marker). They are skipped by default.", - ) - group.addoption( - "--include-quarantined", - action="store_true", - dest="include_quarantined", - help="Includes quarantined tests (marked with quarantined marker). They are skipped by default.", - ) - group.addoption( - "--exclude-virtualenv-operator", - action="store_true", - dest="exclude_virtualenv_operator", - help="Excludes virtualenv operators tests (marked with virtualenv_test marker).", - ) - group.addoption( - "--exclude-external-python-operator", - action="store_true", - dest="exclude_external_python_operator", - help="Excludes external python operator tests (marked with external_python_test marker).", - ) - allowed_trace_sql_columns_list = ",".join(ALLOWED_TRACE_SQL_COLUMNS) - group.addoption( - "--trace-sql", - action="store", - dest="trace_sql", - help=( - "Trace SQL statements. As an argument, you must specify the columns to be " - f"displayed as a comma-separated list. Supported values: [f{allowed_trace_sql_columns_list}]" - ), - metavar="COLUMNS", - ) - group.addoption( - "--no-db-cleanup", - action="store_false", - dest="db_cleanup", - help="Disable DB clear before each test module.", - ) - group.addoption( - "--disable-forbidden-warnings", - action="store_true", - dest="disable_forbidden_warnings", - help="Disable raising an error if forbidden warnings detected.", - ) - group.addoption( - "--disable-capture-warnings", - action="store_true", - dest="disable_capture_warnings", - help="Disable internal capture warnings.", - ) - group.addoption( - "--warning-output-path", - action="store", - dest="warning_output_path", - metavar="PATH", - help=( - "Path for resulting captured warnings. Absolute or relative to the `tests` directory. " - "If not provided or environment variable `CAPTURE_WARNINGS_OUTPUT` not set " - "then 'warnings.txt' will be used." - ), - ) - parser.addini( - name="forbidden_warnings", - type="linelist", - help="List of internal Airflow warnings which are prohibited during tests execution.", - ) - - -def initial_db_init(): - from flask import Flask - - from airflow.configuration import conf - from airflow.utils import db - from airflow.www.extensions.init_appbuilder import init_appbuilder - from airflow.www.extensions.init_auth_manager import get_auth_manager - from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS - - db.resetdb() - db.bootstrap_dagbag() - # minimal app to add roles - flask_app = Flask(__name__) - flask_app.config["SQLALCHEMY_DATABASE_URI"] = conf.get("database", "SQL_ALCHEMY_CONN") - init_appbuilder(flask_app) - if AIRFLOW_V_2_8_PLUS: - get_auth_manager().init() - - -@pytest.fixture(autouse=True, scope="session") -def initialize_airflow_tests(request): - """Helper that setups Airflow testing environment.""" - print(" AIRFLOW ".center(60, "=")) - - # Setup test environment for breeze - home = os.path.expanduser("~") - airflow_home = os.environ.get("AIRFLOW_HOME") or os.path.join(home, "airflow") - - print(f"Home of the user: {home}\nAirflow home {airflow_home}") - - # Initialize Airflow db if required - lock_file = os.path.join(airflow_home, ".airflow_db_initialised") - if not skip_db_tests: - if request.config.option.db_init: - print("Initializing the DB - forced with --with-db-init switch.") - initial_db_init() - elif not os.path.exists(lock_file): - print( - "Initializing the DB - first time after entering the container.\n" - "You can force re-initialization the database by adding --with-db-init switch to run-tests." - ) - initial_db_init() - # Create pid file - with open(lock_file, "w+"): - pass - else: - print( - "Skipping initializing of the DB as it was initialized already.\n" - "You can re-initialize the database by adding --with-db-init flag when running tests." - ) - integration_kerberos = os.environ.get("INTEGRATION_KERBEROS") - if integration_kerberos == "true": - # Initialize kerberos - kerberos = os.environ.get("KRB5_KTNAME") - if kerberos: - subprocess.check_call(["kinit", "-kt", kerberos, "bob@EXAMPLE.COM"]) - else: - print("Kerberos enabled! Please setup KRB5_KTNAME environment variable") - sys.exit(1) - - -def pytest_configure(config: pytest.Config) -> None: - if (backend := config.getoption("backend", default=None)) and backend not in SUPPORTED_DB_BACKENDS: - msg = ( - f"Provided DB backend {backend!r} not supported, " - f"expected one of: {', '.join(map(repr, SUPPORTED_DB_BACKENDS))}" - ) - pytest.exit(msg, returncode=6) - - config.addinivalue_line("markers", "integration(name): mark test to run with named integration") - config.addinivalue_line("markers", "backend(name): mark test to run with named backend") - config.addinivalue_line("markers", "system(name): mark test to run with named system") - config.addinivalue_line("markers", "platform(name): mark test to run with specific platform/environment") - config.addinivalue_line("markers", "long_running: mark test that run for a long time (many minutes)") - config.addinivalue_line( - "markers", "quarantined: mark test that are in quarantine (i.e. flaky, need to be isolated and fixed)" - ) - config.addinivalue_line( - "markers", "credential_file(name): mark tests that require credential file in CREDENTIALS_DIR" - ) - config.addinivalue_line( - "markers", "need_serialized_dag: mark tests that require dags in serialized form to be present" - ) - config.addinivalue_line( - "markers", - "db_test: mark tests that require database to be present", - ) - config.addinivalue_line( - "markers", - "non_db_test_override: you can mark individual tests with this marker to override the db_test marker", - ) - config.addinivalue_line( - "markers", - "virtualenv_operator: virtualenv operator tests are 'long', we should run them separately", - ) - config.addinivalue_line( - "markers", - "external_python_operator: external python operator tests are 'long', we should run them separately", - ) - config.addinivalue_line("markers", "enable_redact: do not mock redact secret masker") - config.addinivalue_line("markers", "skip_if_database_isolation_mode: skip if DB isolation is enabled") - - os.environ["_AIRFLOW__SKIP_DATABASE_EXECUTOR_COMPATIBILITY_CHECK"] = "1" - - # Setup internal warnings plugins - if "ignore" in sys.warnoptions: - config.option.disable_forbidden_warnings = True - config.option.disable_capture_warnings = True - if not config.pluginmanager.get_plugin("warnings"): - # Internal forbidden warnings plugin depends on builtin pytest warnings plugin - config.option.disable_forbidden_warnings = True - - forbidden_warnings: list[str] | None = config.getini("forbidden_warnings") - if not config.option.disable_forbidden_warnings and forbidden_warnings: - from tests._internals.forbidden_warnings import ForbiddenWarningsPlugin - - forbidden_warnings_plugin = ForbiddenWarningsPlugin( - config=config, - forbidden_warnings=tuple(map(str.strip, forbidden_warnings)), - ) - config.pluginmanager.register(forbidden_warnings_plugin) - config.stash[forbidden_warnings_key] = forbidden_warnings_plugin - - if not config.option.disable_capture_warnings: - from tests._internals.capture_warnings import CaptureWarningsPlugin - - capture_warnings_plugin = CaptureWarningsPlugin( - config=config, output_path=config.getoption("warning_output_path", default=None) - ) - config.pluginmanager.register(capture_warnings_plugin) - config.stash[capture_warnings_key] = capture_warnings_plugin - - -def pytest_unconfigure(config: pytest.Config) -> None: - os.environ.pop("_AIRFLOW__SKIP_DATABASE_EXECUTOR_COMPATIBILITY_CHECK", None) - if forbidden_warnings_plugin := config.stash.get(forbidden_warnings_key, None): - del config.stash[forbidden_warnings_key] - config.pluginmanager.unregister(forbidden_warnings_plugin) - if capture_warnings_plugin := config.stash.get(capture_warnings_key, None): - del config.stash[capture_warnings_key] - config.pluginmanager.unregister(capture_warnings_plugin) - - -def skip_if_not_marked_with_integration(selected_integrations, item): - for marker in item.iter_markers(name="integration"): - integration_name = marker.args[0] - if integration_name in selected_integrations or "all" in selected_integrations: - return - pytest.skip( - f"The test is skipped because it does not have the right integration marker. " - f"Only tests marked with pytest.mark.integration(INTEGRATION) are run with INTEGRATION " - f"being one of {selected_integrations}. {item}" - ) - - -def skip_if_not_marked_with_backend(selected_backend, item): - for marker in item.iter_markers(name="backend"): - backend_names = marker.args - if selected_backend in backend_names: - return - pytest.skip( - f"The test is skipped because it does not have the right backend marker. " - f"Only tests marked with pytest.mark.backend('{selected_backend}') are run: {item}" - ) - - -def skip_if_platform_doesnt_match(marker): - allowed_platforms = ("linux", "breeze") - if not (args := marker.args): - pytest.fail(f"No platform specified, expected one of: {', '.join(map(repr, allowed_platforms))}") - elif not all(a in allowed_platforms for a in args): - pytest.fail( - f"Allowed platforms {', '.join(map(repr, allowed_platforms))}; " - f"but got: {', '.join(map(repr, args))}" - ) - if "linux" in args: - if not sys.platform.startswith("linux"): - pytest.skip("Test expected to run on Linux platform.") - if "breeze" in args: - if not os.path.isfile("/.dockerenv") or os.environ.get("BREEZE", "").lower() != "true": - raise pytest.skip( - "Test expected to run into Airflow Breeze container. " - "Maybe because it is to dangerous to run it outside." - ) - - -def skip_if_not_marked_with_system(selected_systems, item): - for marker in item.iter_markers(name="system"): - systems_name = marker.args[0] - if systems_name in selected_systems or "all" in selected_systems: - return - pytest.skip( - f"The test is skipped because it does not have the right system marker. " - f"Only tests marked with pytest.mark.system(SYSTEM) are run with SYSTEM " - f"being one of {selected_systems}. {item}" - ) - - -def skip_system_test(item): - for marker in item.iter_markers(name="system"): - pytest.skip( - f"The test is skipped because it has system marker. System tests are only run when " - f"--system flag with the right system ({marker.args[0]}) is passed to pytest. {item}" - ) - - -def skip_long_running_test(item): - for _ in item.iter_markers(name="long_running"): - pytest.skip( - f"The test is skipped because it has long_running marker. " - f"And --include-long-running flag is not passed to pytest. {item}" - ) - - -def skip_quarantined_test(item): - for _ in item.iter_markers(name="quarantined"): - pytest.skip( - f"The test is skipped because it has quarantined marker. " - f"And --include-quarantined flag is not passed to pytest. {item}" - ) - - -def skip_virtualenv_operator_test(item): - for _ in item.iter_markers(name="virtualenv_operator"): - pytest.skip( - f"The test is skipped because it has virtualenv_operator marker. " - f"And --exclude-virtualenv-operator flag is not passed to pytest. {item}" - ) - - -def skip_external_python_operator_test(item): - for _ in item.iter_markers(name="external_python_operator"): - pytest.skip( - f"The test is skipped because it has external_python_operator marker. " - f"And --exclude-external-python-operator flag is not passed to pytest. {item}" - ) - - -def skip_db_test(item): - if next(item.iter_markers(name="db_test"), None): - if next(item.iter_markers(name="non_db_test_override"), None): - # non_db_test can override the db_test set for example on module or class level - return - else: - pytest.skip( - f"The test is skipped as it is DB test " - f"and --skip-db-tests is flag is passed to pytest. {item}" - ) - if next(item.iter_markers(name="backend"), None): - # also automatically skip tests marked with `backend` marker as they are implicitly - # db tests - pytest.skip( - f"The test is skipped as it is DB test " - f"and --skip-db-tests is flag is passed to pytest. {item}" - ) - - -def only_run_db_test(item): - if next(item.iter_markers(name="db_test"), None) and not next( - item.iter_markers(name="non_db_test_override"), None - ): - # non_db_test at individual level can override the db_test set for example on module or class level - return - else: - if next(item.iter_markers(name="backend"), None): - # Also do not skip the tests marked with `backend` marker - as it is implicitly a db test - return - pytest.skip( - f"The test is skipped as it is not a DB tests " - f"and --run-db-tests-only flag is passed to pytest. {item}" - ) - - -def skip_if_integration_disabled(marker, item): - integration_name = marker.args[0] - environment_variable_name = "INTEGRATION_" + integration_name.upper() - environment_variable_value = os.environ.get(environment_variable_name) - if not environment_variable_value or environment_variable_value != "true": - pytest.skip( - f"The test requires {integration_name} integration started and " - f"{environment_variable_name} environment variable to be set to true (it is '{environment_variable_value}')." - f" It can be set by specifying '--integration {integration_name}' at breeze startup" - f": {item}" - ) - - -def skip_if_wrong_backend(marker: pytest.Mark, item: pytest.Item) -> None: - if not (backend_names := marker.args): - reason = ( - "`pytest.mark.backend` expect to get at least one of the following backends: " - f"{', '.join(map(repr, SUPPORTED_DB_BACKENDS))}." - ) - pytest.fail(reason) - elif unsupported_backends := list(filter(lambda b: b not in SUPPORTED_DB_BACKENDS, backend_names)): - reason = ( - "Airflow Tests supports only the following backends in `pytest.mark.backend` marker: " - f"{', '.join(map(repr, SUPPORTED_DB_BACKENDS))}, " - f"but got {', '.join(map(repr, unsupported_backends))}." - ) - pytest.fail(reason) - - env_name = "BACKEND" - if not (backend := os.environ.get(env_name)) or backend not in backend_names: - reason = ( - f"The test {item.nodeid!r} requires one of {', '.join(map(repr, backend_names))} backend started " - f"and {env_name!r} environment variable to be set (currently it set to {backend!r}). " - f"It can be set by specifying backend at breeze startup." - ) - pytest.skip(reason) - - -def skip_if_credential_file_missing(item): - for marker in item.iter_markers(name="credential_file"): - credential_file = marker.args[0] - credential_path = os.path.join(os.environ.get("CREDENTIALS_DIR"), credential_file) - if not os.path.exists(credential_path): - pytest.skip(f"The test requires credential file {credential_path}: {item}") - - -def pytest_runtest_setup(item): - selected_integrations_list = item.config.option.integration - selected_systems_list = item.config.option.system - - include_long_running = item.config.option.include_long_running - include_quarantined = item.config.option.include_quarantined - exclude_virtualenv_operator = item.config.option.exclude_virtualenv_operator - exclude_external_python_operator = item.config.option.exclude_external_python_operator - - for marker in item.iter_markers(name="integration"): - skip_if_integration_disabled(marker, item) - if selected_integrations_list: - skip_if_not_marked_with_integration(selected_integrations_list, item) - if selected_systems_list: - skip_if_not_marked_with_system(selected_systems_list, item) - else: - skip_system_test(item) - for marker in item.iter_markers(name="platform"): - skip_if_platform_doesnt_match(marker) - for marker in item.iter_markers(name="backend"): - skip_if_wrong_backend(marker, item) - skip_if_database_isolation_mode(item) - selected_backend = item.config.option.backend - if selected_backend: - skip_if_not_marked_with_backend(selected_backend, item) - if not include_long_running: - skip_long_running_test(item) - if not include_quarantined: - skip_quarantined_test(item) - if exclude_virtualenv_operator: - skip_virtualenv_operator_test(item) - if exclude_external_python_operator: - skip_external_python_operator_test(item) - if skip_db_tests: - skip_db_test(item) - if run_db_tests_only: - only_run_db_test(item) - skip_if_credential_file_missing(item) - - -@pytest.fixture -def frozen_sleep(monkeypatch): - """Use time-machine to "stub" sleep. - - This means the ``sleep()`` takes no time, but ``datetime.now()`` appears to move forwards. - - If your module under test does ``import time`` and then ``time.sleep``: - - .. code-block:: python - - def test_something(frozen_sleep): - my_mod.fn_under_test() - - If your module under test does ``from time import sleep`` then you will - have to mock that sleep function directly: - - .. code-block:: python - - def test_something(frozen_sleep, monkeypatch): - monkeypatch.setattr("my_mod.sleep", frozen_sleep) - my_mod.fn_under_test() - """ - traveller = None - - def fake_sleep(seconds): - nonlocal traveller - utcnow = datetime.now(tz=timezone.utc) - if traveller is not None: - traveller.stop() - traveller = time_machine.travel(utcnow + timedelta(seconds=seconds)) - traveller.start() - - monkeypatch.setattr("time.sleep", fake_sleep) - yield fake_sleep - - if traveller is not None: - traveller.stop() - - -@pytest.fixture(scope="session") -def app(): - from tests.test_utils.config import conf_vars - - with conf_vars({("fab", "auth_rate_limited"): "False"}): - from airflow.www import app - - yield app.create_app(testing=True) - - -@pytest.fixture -def dag_maker(request): - """Fixture to help create DAG, DagModel, and SerializedDAG automatically. - - You have to use the dag_maker as a context manager and it takes - the same argument as DAG:: - - with dag_maker(dag_id="mydag") as dag: - task1 = EmptyOperator(task_id="mytask") - task2 = EmptyOperator(task_id="mytask2") - - If the DagModel you want to use needs different parameters than the one - automatically created by the dag_maker, you have to update the DagModel as below:: - - dag_maker.dag_model.is_active = False - session.merge(dag_maker.dag_model) - session.commit() - - For any test you use the dag_maker, make sure to create a DagRun:: - - dag_maker.create_dagrun() - - The dag_maker.create_dagrun takes the same arguments as dag.create_dagrun - - If you want to operate on serialized DAGs, then either pass - ``serialized=True`` to the ``dag_maker()`` call, or you can mark your - test/class/file with ``@pytest.mark.need_serialized_dag(True)``. In both of - these cases the ``dag`` returned by the context manager will be a - lazily-evaluated proxy object to the SerializedDAG. - """ - import lazy_object_proxy - - # IMPORTANT: Delay _all_ imports from `airflow.*` to _inside a method_. - # This fixture is "called" early on in the pytest collection process, and - # if we import airflow.* here the wrong (non-test) config will be loaded - # and "baked" in to various constants - - want_serialized = False - - # Allow changing default serialized behaviour with `@pytest.mark.need_serialized_dag` or - # `@pytest.mark.need_serialized_dag(False)` - serialized_marker = request.node.get_closest_marker("need_serialized_dag") - if serialized_marker: - (want_serialized,) = serialized_marker.args or (True,) - - from airflow.utils.log.logging_mixin import LoggingMixin - - class DagFactory(LoggingMixin): - _own_session = False - - def __init__(self): - from airflow.models import DagBag - - # Keep all the serialized dags we've created in this test - self.dagbag = DagBag(os.devnull, include_examples=False, read_dags_from_db=False) - - def __enter__(self): - self.dag.__enter__() - if self.want_serialized: - return lazy_object_proxy.Proxy(self._serialized_dag) - return self.dag - - def _serialized_dag(self): - return self.serialized_model.dag - - def get_serialized_data(self): - try: - data = self.serialized_model.data - except AttributeError: - raise RuntimeError("DAG serialization not requested") - if isinstance(data, str): - return json.loads(data) - return data - - def _bag_dag_compat(self, dag): - # This is a compatibility shim for the old bag_dag method in Airflow <3.0 - # TODO: Remove this when we drop support for Airflow <3.0 in Providers - if hasattr(dag, "parent_dag"): - return self.dagbag.bag_dag(dag, root_dag=dag) - return self.dagbag.bag_dag(dag) - - def __exit__(self, type, value, traceback): - from airflow.models import DagModel - from airflow.models.serialized_dag import SerializedDagModel - - dag = self.dag - dag.__exit__(type, value, traceback) - if type is not None: - return - - dag.clear(session=self.session) - dag.sync_to_db(processor_subdir=self.processor_subdir, session=self.session) - self.dag_model = self.session.get(DagModel, dag.dag_id) - - if self.want_serialized: - self.serialized_model = SerializedDagModel( - dag, processor_subdir=self.dag_model.processor_subdir - ) - self.session.merge(self.serialized_model) - serialized_dag = self._serialized_dag() - self._bag_dag_compat(serialized_dag) - self.session.flush() - else: - self._bag_dag_compat(self.dag) - - def create_dagrun(self, **kwargs): - from airflow.utils import timezone - from airflow.utils.state import State - from airflow.utils.types import DagRunType - from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS - - if AIRFLOW_V_3_0_PLUS: - from airflow.utils.types import DagRunTriggeredByType - - dag = self.dag - kwargs = { - "state": State.RUNNING, - "start_date": self.start_date, - "session": self.session, - **kwargs, - } - # Need to provide run_id if the user does not either provide one - # explicitly, or pass run_type for inference in dag.create_dagrun(). - if "run_id" not in kwargs and "run_type" not in kwargs: - kwargs["run_id"] = "test" - - if "run_type" not in kwargs: - kwargs["run_type"] = DagRunType.from_run_id(kwargs["run_id"]) - if kwargs.get("execution_date") is None: - if kwargs["run_type"] == DagRunType.MANUAL: - kwargs["execution_date"] = self.start_date - else: - kwargs["execution_date"] = dag.next_dagrun_info(None).logical_date - if "data_interval" not in kwargs: - logical_date = timezone.coerce_datetime(kwargs["execution_date"]) - if kwargs["run_type"] == DagRunType.MANUAL: - data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date) - else: - data_interval = dag.infer_automated_data_interval(logical_date) - kwargs["data_interval"] = data_interval - if AIRFLOW_V_3_0_PLUS and "triggered_by" not in kwargs: - kwargs["triggered_by"] = DagRunTriggeredByType.TEST - - self.dag_run = dag.create_dagrun(**kwargs) - for ti in self.dag_run.task_instances: - ti.refresh_from_task(dag.get_task(ti.task_id)) - if self.want_serialized: - self.session.commit() - return self.dag_run - - def create_dagrun_after(self, dagrun, **kwargs): - next_info = self.dag.next_dagrun_info(self.dag.get_run_data_interval(dagrun)) - if next_info is None: - raise ValueError(f"cannot create run after {dagrun}") - return self.create_dagrun( - execution_date=next_info.logical_date, - data_interval=next_info.data_interval, - **kwargs, - ) - - def __call__( - self, - dag_id="test_dag", - schedule=timedelta(days=1), - serialized=want_serialized, - fileloc=None, - processor_subdir=None, - session=None, - **kwargs, - ): - from airflow import settings - from airflow.models.dag import DAG - from airflow.utils import timezone - - if session is None: - self._own_session = True - session = settings.Session() - - self.kwargs = kwargs - self.session = session - self.start_date = self.kwargs.get("start_date", None) - default_args = kwargs.get("default_args", None) - if default_args and not self.start_date: - if "start_date" in default_args: - self.start_date = default_args.get("start_date") - if not self.start_date: - if hasattr(request.module, "DEFAULT_DATE"): - self.start_date = getattr(request.module, "DEFAULT_DATE") - else: - DEFAULT_DATE = timezone.datetime(2016, 1, 1) - self.start_date = DEFAULT_DATE - self.kwargs["start_date"] = self.start_date - # Set schedule argument to explicitly set value, or a default if no - # other scheduling arguments are set. - self.dag = DAG(dag_id, schedule=schedule, **self.kwargs) - self.dag.fileloc = fileloc or request.module.__file__ - self.want_serialized = serialized - self.processor_subdir = processor_subdir - - return self - - def cleanup(self): - from airflow.models import DagModel, DagRun, TaskInstance, XCom - from airflow.models.serialized_dag import SerializedDagModel - from airflow.models.taskmap import TaskMap - from airflow.utils.retries import run_with_db_retries - from tests.test_utils.compat import AssetEvent - - for attempt in run_with_db_retries(logger=self.log): - with attempt: - dag_ids = list(self.dagbag.dag_ids) - if not dag_ids: - return - # To isolate problems here with problems from elsewhere on the session object - self.session.rollback() - - self.session.query(SerializedDagModel).filter( - SerializedDagModel.dag_id.in_(dag_ids) - ).delete(synchronize_session=False) - self.session.query(DagRun).filter(DagRun.dag_id.in_(dag_ids)).delete( - synchronize_session=False, - ) - self.session.query(TaskInstance).filter(TaskInstance.dag_id.in_(dag_ids)).delete( - synchronize_session=False, - ) - self.session.query(XCom).filter(XCom.dag_id.in_(dag_ids)).delete( - synchronize_session=False, - ) - self.session.query(DagModel).filter(DagModel.dag_id.in_(dag_ids)).delete( - synchronize_session=False, - ) - self.session.query(TaskMap).filter(TaskMap.dag_id.in_(dag_ids)).delete( - synchronize_session=False, - ) - self.session.query(AssetEvent).filter(AssetEvent.source_dag_id.in_(dag_ids)).delete( - synchronize_session=False, - ) - self.session.commit() - if self._own_session: - self.session.expunge_all() - - factory = DagFactory() - - try: - yield factory - finally: - factory.cleanup() - with suppress(AttributeError): - del factory.session - - -@pytest.fixture -def create_dummy_dag(dag_maker): - """Create a `DAG` with a single `EmptyOperator` task. - - DagRun and DagModel is also created. - - Apart from the already existing arguments, any other argument in kwargs - is passed to the DAG and not to the EmptyOperator task. - - If you have an argument that you want to pass to the EmptyOperator that - is not here, please use `default_args` so that the DAG will pass it to the - Task:: - - dag, task = create_dummy_dag(default_args={"start_date": timezone.datetime(2016, 1, 1)}) - - You cannot be able to alter the created DagRun or DagModel, use `dag_maker` fixture instead. - """ - from airflow.operators.empty import EmptyOperator - from airflow.utils.types import DagRunType - - def create_dag( - dag_id="dag", - task_id="op1", - task_display_name=None, - max_active_tis_per_dag=16, - max_active_tis_per_dagrun=None, - pool="default_pool", - executor_config=None, - trigger_rule="all_done", - on_success_callback=None, - on_execute_callback=None, - on_failure_callback=None, - on_retry_callback=None, - email=None, - with_dagrun_type=DagRunType.SCHEDULED, - **kwargs, - ): - op_kwargs = {} - from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS - - if AIRFLOW_V_2_9_PLUS: - op_kwargs["task_display_name"] = task_display_name - with dag_maker(dag_id, **kwargs) as dag: - op = EmptyOperator( - task_id=task_id, - max_active_tis_per_dag=max_active_tis_per_dag, - max_active_tis_per_dagrun=max_active_tis_per_dagrun, - executor_config=executor_config or {}, - on_success_callback=on_success_callback, - on_execute_callback=on_execute_callback, - on_failure_callback=on_failure_callback, - on_retry_callback=on_retry_callback, - email=email, - pool=pool, - trigger_rule=trigger_rule, - **op_kwargs, - ) - if with_dagrun_type is not None: - dag_maker.create_dagrun(run_type=with_dagrun_type) - return dag, op - - return create_dag - - -if TYPE_CHECKING: - from airflow.models.taskinstance import TaskInstance - - -@pytest.fixture -def create_task_instance(dag_maker, create_dummy_dag): - """Create a TaskInstance, and associated DB rows (DagRun, DagModel, etc). - - Uses ``create_dummy_dag`` to create the dag structure. - """ - from airflow.operators.empty import EmptyOperator - - def maker( - execution_date=None, - dagrun_state=None, - state=None, - run_id=None, - run_type=None, - data_interval=None, - external_executor_id=None, - dag_id="dag", - task_id="op1", - task_display_name=None, - max_active_tis_per_dag=16, - max_active_tis_per_dagrun=None, - pool="default_pool", - executor_config=None, - trigger_rule="all_done", - on_success_callback=None, - on_execute_callback=None, - on_failure_callback=None, - on_retry_callback=None, - email=None, - map_index=-1, - **kwargs, - ) -> TaskInstance: - from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS - - if AIRFLOW_V_3_0_PLUS: - from airflow.utils.types import DagRunTriggeredByType - - if execution_date is None: - from airflow.utils import timezone - - execution_date = timezone.utcnow() - with dag_maker(dag_id, **kwargs): - op_kwargs = {} - from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS - - if AIRFLOW_V_2_9_PLUS: - op_kwargs["task_display_name"] = task_display_name - task = EmptyOperator( - task_id=task_id, - max_active_tis_per_dag=max_active_tis_per_dag, - max_active_tis_per_dagrun=max_active_tis_per_dagrun, - executor_config=executor_config or {}, - on_success_callback=on_success_callback, - on_execute_callback=on_execute_callback, - on_failure_callback=on_failure_callback, - on_retry_callback=on_retry_callback, - email=email, - pool=pool, - trigger_rule=trigger_rule, - **op_kwargs, - ) - - dagrun_kwargs = { - "execution_date": execution_date, - "state": dagrun_state, - } - dagrun_kwargs.update({"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {}) - if run_id is not None: - dagrun_kwargs["run_id"] = run_id - if run_type is not None: - dagrun_kwargs["run_type"] = run_type - if data_interval is not None: - dagrun_kwargs["data_interval"] = data_interval - dagrun = dag_maker.create_dagrun(**dagrun_kwargs) - (ti,) = dagrun.task_instances - ti.task = task - ti.state = state - ti.external_executor_id = external_executor_id - ti.map_index = map_index - - dag_maker.session.flush() - return ti - - return maker - - -@pytest.fixture -def create_serialized_task_instance_of_operator(dag_maker): - def _create_task_instance( - operator_class, - *, - dag_id, - execution_date=None, - session=None, - **operator_kwargs, - ) -> TaskInstance: - with dag_maker(dag_id=dag_id, serialized=True, session=session): - operator_class(**operator_kwargs) - if execution_date is None: - dagrun_kwargs = {} - else: - dagrun_kwargs = {"execution_date": execution_date} - (ti,) = dag_maker.create_dagrun(**dagrun_kwargs).task_instances - return ti - - return _create_task_instance - - -@pytest.fixture -def create_task_instance_of_operator(dag_maker): - def _create_task_instance( - operator_class, - *, - dag_id, - execution_date=None, - session=None, - **operator_kwargs, - ) -> TaskInstance: - with dag_maker(dag_id=dag_id, session=session, serialized=True): - operator_class(**operator_kwargs) - if execution_date is None: - dagrun_kwargs = {} - else: - dagrun_kwargs = {"execution_date": execution_date} - (ti,) = dag_maker.create_dagrun(**dagrun_kwargs).task_instances - return ti - - return _create_task_instance - - -@pytest.fixture -def create_task_of_operator(dag_maker): - def _create_task_of_operator(operator_class, *, dag_id, session=None, **operator_kwargs): - with dag_maker(dag_id=dag_id, session=session): - task = operator_class(**operator_kwargs) - return task - - return _create_task_of_operator - - -@pytest.fixture -def session(): - from airflow.utils.session import create_session - - with create_session() as session: - yield session - session.rollback() - - -@pytest.fixture -def get_test_dag(): - def _get(dag_id): - from airflow.models.dagbag import DagBag - from airflow.models.serialized_dag import SerializedDagModel - - dag_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "dags", f"{dag_id}.py") - dagbag = DagBag(dag_folder=dag_file, include_examples=False) - - dag = dagbag.get_dag(dag_id) - dag.sync_to_db() - SerializedDagModel.write_dag(dag) - - return dag - - return _get - - -@pytest.fixture -def create_log_template(request): - from airflow import settings - from airflow.models.tasklog import LogTemplate - - session = settings.Session() - - def _create_log_template(filename_template, elasticsearch_id=""): - log_template = LogTemplate(filename=filename_template, elasticsearch_id=elasticsearch_id) - session.add(log_template) - session.commit() - - def _delete_log_template(): - from airflow.models import DagRun, TaskInstance - - session.query(TaskInstance).delete() - session.query(DagRun).delete() - session.delete(log_template) - session.commit() - - request.addfinalizer(_delete_log_template) - - return _create_log_template - - -@pytest.fixture -def reset_logging_config(): - import logging.config - - from airflow import settings - from airflow.utils.module_loading import import_string - - logging_config = import_string(settings.LOGGING_CLASS_PATH) - logging.config.dictConfig(logging_config) - - -@pytest.fixture(scope="session", autouse=True) -def suppress_info_logs_for_dag_and_fab(): - import logging - - from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS - - dag_logger = logging.getLogger("airflow.models.dag") - dag_logger.setLevel(logging.WARNING) - - if AIRFLOW_V_2_9_PLUS: - fab_logger = logging.getLogger("airflow.providers.fab.auth_manager.security_manager.override") - fab_logger.setLevel(logging.WARNING) - else: - fab_logger = logging.getLogger("airflow.www.fab_security") - fab_logger.setLevel(logging.WARNING) - - -@pytest.fixture(scope="module", autouse=True) -def _clear_db(request): - """Clear DB before each test module run.""" - from tests.test_utils.db import clear_all - - if not request.config.option.db_cleanup: - return - if skip_db_tests: - return - from airflow.configuration import conf - - sql_alchemy_conn = conf.get("database", "sql_alchemy_conn") - if sql_alchemy_conn.startswith("sqlite"): - sql_alchemy_file = sql_alchemy_conn.replace("sqlite:///", "") - if not os.path.exists(sql_alchemy_file): - print(f"The sqlite file `{sql_alchemy_file}` does not exist. Attempt to initialize it.") - initial_db_init() - - dist_option = getattr(request.config.option, "dist", "no") - if dist_option != "no" or hasattr(request.config, "workerinput"): - # Skip if pytest-xdist detected (controller or worker) - return - try: - clear_all() - except Exception as ex: - exc_name_parts = [type(ex).__name__] - exc_module = type(ex).__module__ - if exc_module != "builtins": - exc_name_parts.insert(0, exc_module) - extra_msg = "" if request.config.option.db_init else ", try to run with flag --with-db-init" - pytest.exit(f"Unable clear test DB{extra_msg}, got error {'.'.join(exc_name_parts)}: {ex}") - - -@pytest.fixture(autouse=True) -def clear_lru_cache(): - from airflow.executors.executor_loader import ExecutorLoader - from airflow.utils.entry_points import _get_grouped_entry_points - - ExecutorLoader.validate_database_executor_compatibility.cache_clear() - try: - _get_grouped_entry_points.cache_clear() - try: - yield - finally: - _get_grouped_entry_points.cache_clear() - finally: - ExecutorLoader.validate_database_executor_compatibility.cache_clear() - - -@pytest.fixture(autouse=True) -def refuse_to_run_test_from_wrongly_named_files(request: pytest.FixtureRequest): - filepath = request.node.path - is_system_test: bool = "tests/system/" in os.fspath(filepath) - test_name = request.node.name - if request.node.cls: - test_name = f"{request.node.cls.__name__}.{test_name}" - if is_system_test and not filepath.name.startswith(("example_", "test_")): - pytest.fail( - f"All test method files in tests/system must start with 'example_' or 'test_'. " - f"Seems that {os.fspath(filepath)!r} contains {test_name!r} that looks like a test case. " - f"Please rename the file to follow the example_* or test_* pattern if you want to run the tests " - f"in it." - ) - elif not is_system_test and not filepath.name.startswith("test_"): - pytest.fail( - f"All test method files in tests/ must start with 'test_'. Seems that {os.fspath(filepath)!r} " - f"contains {test_name!r} that looks like a test case. Please rename the file to " - f"follow the test_* pattern if you want to run the tests in it." - ) - - -@pytest.fixture(autouse=True) -def initialize_providers_manager(): - from airflow.providers_manager import ProvidersManager - - ProvidersManager().initialize_providers_configuration() - - -@pytest.fixture(autouse=True) -def close_all_sqlalchemy_sessions(): - from sqlalchemy.orm import close_all_sessions - - with suppress(Exception): - close_all_sessions() - yield - with suppress(Exception): - close_all_sessions() - - -@pytest.fixture -def cleanup_providers_manager(): - from airflow.providers_manager import ProvidersManager - - ProvidersManager()._cleanup() - ProvidersManager().initialize_providers_configuration() - try: - yield - finally: - ProvidersManager()._cleanup() - - -@pytest.fixture(autouse=True) -def _disable_redact(request: pytest.FixtureRequest, mocker): - """Disable redacted text in tests, except specific.""" - from airflow import settings - - if next(request.node.iter_markers("enable_redact"), None): - with pytest.MonkeyPatch.context() as mp_ctx: - mp_ctx.setattr(settings, "MASK_SECRETS_IN_LOGS", True) - yield - return - - mocked_redact = mocker.patch("airflow.utils.log.secrets_masker.SecretsMasker.redact") - mocked_redact.side_effect = lambda item, name=None, max_depth=None: item - with pytest.MonkeyPatch.context() as mp_ctx: - mp_ctx.setattr(settings, "MASK_SECRETS_IN_LOGS", False) - yield - return - - -@pytest.fixture -def airflow_root_path() -> Path: - import airflow - - return Path(airflow.__path__[0]).parent - - -@pytest.fixture -def hook_lineage_collector(): - from airflow.lineage import hook - - hook._hook_lineage_collector = None - hook._hook_lineage_collector = hook.HookLineageCollector() - yield hook.get_hook_lineage_collector() - hook._hook_lineage_collector = None - - -# This constant is set to True if tests are run with Airflow installed from Packages rather than running -# the tests within Airflow sources. While most tests in CI are run using Airflow sources, there are -# also compatibility tests that only use `tests` package and run against installed packages of Airflow in -# for supported Airflow versions. -RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES = not (Path(__file__).parents[1] / "airflow" / "__init__.py").exists() - - if TYPE_CHECKING: # Static checkers do not know about pytest fixtures' types and return, # In case if them distributed through third party packages. @@ -1489,15 +94,3 @@ def requests_mock() -> RequestsMockFixture: ... # time-machine @pytest.fixture # type: ignore[no-redef] def time_machine() -> TimeMachineFixture: ... - - -@pytest.fixture -def clean_dags_and_dagruns(): - """Fixture that cleans the database before and after every test.""" - from tests.test_utils.db import clear_db_dags, clear_db_runs - - clear_db_runs() - clear_db_dags() - yield # Test runs here - clear_db_dags() - clear_db_runs() diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 1590fb7ee231c..0690a5f0182dc 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -42,8 +42,6 @@ write_default_airflow_configuration_if_needed, ) from airflow.providers_manager import ProvidersManager -from tests.test_utils.config import conf_vars -from tests.test_utils.reset_warning_registry import reset_warning_registry from tests.utils.test_config import ( remove_all_configurations, set_deprecated_options, @@ -51,6 +49,9 @@ use_config, ) +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.reset_warning_registry import reset_warning_registry + HOME_DIR = os.path.expanduser("~") # The conf has been updated with sql_alchemy_con and deactivate_stale_dags_interval to test the diff --git a/tests/core/test_core.py b/tests/core/test_core.py index d44235f955243..13b85a9ef1700 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -32,7 +32,8 @@ from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/core/test_example_dags_system.py b/tests/core/test_example_dags_system.py index 48a936f794f99..c60b7325b125d 100644 --- a/tests/core/test_example_dags_system.py +++ b/tests/core/test_example_dags_system.py @@ -19,7 +19,7 @@ import pytest -from tests.test_utils.system_tests_class import SystemTest +from dev.tests_common.test_utils.system_tests_class import SystemTest @pytest.mark.system("core") diff --git a/tests/core/test_impersonation_tests.py b/tests/core/test_impersonation_tests.py index 721d180ce7d3f..cf9359c863372 100644 --- a/tests/core/test_impersonation_tests.py +++ b/tests/core/test_impersonation_tests.py @@ -35,7 +35,8 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.test_utils import db + +from dev.tests_common.test_utils import db # The entire module into the quarantined mark, this might have unpredictable side effects to other tests # and should be moved into the isolated environment into the future. diff --git a/tests/core/test_logging_config.py b/tests/core/test_logging_config.py index c972532fe2ceb..a316130ecdabe 100644 --- a/tests/core/test_logging_config.py +++ b/tests/core/test_logging_config.py @@ -29,7 +29,8 @@ import pytest from airflow.configuration import conf -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars SETTINGS_FILE_VALID = """ LOGGING_CONFIG = { diff --git a/tests/core/test_otel_tracer.py b/tests/core/test_otel_tracer.py index db675dd4293ec..b9612c49dae92 100644 --- a/tests/core/test_otel_tracer.py +++ b/tests/core/test_otel_tracer.py @@ -26,7 +26,8 @@ from airflow.traces import TRACEPARENT, TRACESTATE, otel_tracer, utils from airflow.traces.tracer import Trace -from tests.test_utils.config import env_vars + +from dev.tests_common.test_utils.config import env_vars @pytest.fixture diff --git a/tests/core/test_sentry.py b/tests/core/test_sentry.py index b9a1ff7af8d7a..c67b49980aae2 100644 --- a/tests/core/test_sentry.py +++ b/tests/core/test_sentry.py @@ -30,7 +30,8 @@ from airflow.utils import timezone from airflow.utils.module_loading import import_string from airflow.utils.state import State -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars EXECUTION_DATE = timezone.utcnow() SCHEDULE_INTERVAL = datetime.timedelta(days=1) diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index cdcf52df16d84..619d64630029d 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -33,7 +33,8 @@ from airflow.exceptions import AirflowClusterPolicyViolation, AirflowConfigException from airflow.settings import _ENABLE_AIP_44, TracebackSession, is_usage_data_collection_enabled from airflow.utils.session import create_session -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars SETTINGS_FILE_POLICY = """ def test_policy(task_instance): diff --git a/tests/core/test_sqlalchemy_config.py b/tests/core/test_sqlalchemy_config.py index 154eb0a5f8931..5fed8745fd085 100644 --- a/tests/core/test_sqlalchemy_config.py +++ b/tests/core/test_sqlalchemy_config.py @@ -25,7 +25,8 @@ from airflow import settings from airflow.api_internal.internal_api_call import InternalApiConfig from airflow.exceptions import AirflowConfigException -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars SQL_ALCHEMY_CONNECT_ARGS = {"test": 43503, "dict": {"is": 1, "supported": "too"}} diff --git a/tests/core/test_stats.py b/tests/core/test_stats.py index ec7ff0cab08c5..9a218010a189e 100644 --- a/tests/core/test_stats.py +++ b/tests/core/test_stats.py @@ -36,7 +36,8 @@ PatternAllowListValidator, PatternBlockListValidator, ) -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars class CustomStatsd(statsd.StatsClient): diff --git a/tests/dag_processing/test_job_runner.py b/tests/dag_processing/test_job_runner.py index 1d3fefdf12d5f..81b94dd18b18a 100644 --- a/tests/dag_processing/test_job_runner.py +++ b/tests/dag_processing/test_job_runner.py @@ -60,9 +60,10 @@ from airflow.utils.session import create_session from tests.core.test_logging_config import SETTINGS_FILE_VALID, settings_context from tests.models import TEST_DAGS_FOLDER -from tests.test_utils.compat import ParseImportError -from tests.test_utils.config import conf_vars -from tests.test_utils.db import ( + +from dev.tests_common.test_utils.compat import ParseImportError +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import ( clear_db_callbacks, clear_db_dags, clear_db_import_errors, diff --git a/tests/dag_processing/test_processor.py b/tests/dag_processing/test_processor.py index d7b2b2116653e..c3a0f5a26e329 100644 --- a/tests/dag_processing/test_processor.py +++ b/tests/dag_processing/test_processor.py @@ -39,10 +39,11 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.types import DagRunType -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError -from tests.test_utils.config import conf_vars, env_vars -from tests.test_utils.db import ( + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError +from dev.tests_common.test_utils.config import conf_vars, env_vars +from dev.tests_common.test_utils.db import ( clear_db_dags, clear_db_import_errors, clear_db_jobs, @@ -50,7 +51,7 @@ clear_db_runs, clear_db_serialized_dags, ) -from tests.test_utils.mock_executor import MockExecutor +from dev.tests_common.test_utils.mock_executor import MockExecutor if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/dags/test_miscellaneous.py b/tests/dags/test_miscellaneous.py index 4a2c6b56a365d..b08e61e3bbddc 100644 --- a/tests/dags/test_miscellaneous.py +++ b/tests/dags/test_miscellaneous.py @@ -23,7 +23,8 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator -from tests.test_utils.compat import BashOperator + +from dev.tests_common.test_utils.compat import BashOperator args = { "owner": "airflow", diff --git a/tests/dags/test_sensor.py b/tests/dags/test_sensor.py index 07c9cc7efdffb..7c96160402663 100644 --- a/tests/dags/test_sensor.py +++ b/tests/dags/test_sensor.py @@ -21,7 +21,8 @@ from airflow.decorators import task from airflow.models.dag import DAG from airflow.utils import timezone -from tests.test_utils.compat import DateTimeSensor + +from dev.tests_common.test_utils.compat import DateTimeSensor with DAG( dag_id="test_sensor", start_date=datetime.datetime(2022, 1, 1), catchup=False, schedule="@once" diff --git a/tests/decorators/test_bash.py b/tests/decorators/test_bash.py index ba8948936eda1..da79fb4cca0fb 100644 --- a/tests/decorators/test_bash.py +++ b/tests/decorators/test_bash.py @@ -29,7 +29,8 @@ from airflow.models.renderedtifields import RenderedTaskInstanceFields from airflow.utils import timezone from airflow.utils.types import NOTSET -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields DEFAULT_DATE = timezone.datetime(2023, 1, 1) diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py index adbf96a0f41ba..d34fbbf552b30 100644 --- a/tests/decorators/test_python.py +++ b/tests/decorators/test_python.py @@ -40,7 +40,8 @@ from airflow.utils.types import DagRunType from airflow.utils.xcom import XCOM_RETURN_KEY from tests.operators.test_python import BasePythonTest -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/deprecations_ignore.yml b/tests/deprecations_ignore.yml index 8be939227d814..c3d5ddd8eafa8 100644 --- a/tests/deprecations_ignore.yml +++ b/tests/deprecations_ignore.yml @@ -51,116 +51,3 @@ - tests/www/views/test_views_rendered.py::test_rendered_task_detail_env_secret - tests/www/views/test_views_tasks.py::test_rendered_task_view - tests/www/views/test_views_tasks.py::test_views_get - - -# Providers -- tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py::TestAwsBaseAsyncHook::test_get_client_async -- tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_cluster_status -- tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_get_cluster_status -- tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_get_cluster_status_exception -- tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_pause_cluster -- tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_resume_cluster -- tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py::TestRedshiftAsyncHook::test_resume_cluster_exception -- tests/providers/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_exception -- tests/providers/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_resuming_status -- tests/providers/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_success -- tests/providers/google/common/auth_backend/test_google_openid.py::TestGoogleOpenID::test_success -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_api_resource_configs -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_api_resource_configs_duplication_warning -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_queries -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_query_cancel_completed -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_query_cancel_timeout -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_cancel_query_jobs_to_cancel -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_get_dataset_tables_list -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_invalid_schema_update_and_write_disposition -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_invalid_schema_update_options -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_invalid_source_format -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_extract -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_load_with_non_csv_as_src_fmt -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_schema_update_options -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_schema_update_options_incorrect -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect_default -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect_legacy_with_query_params -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_sql_dialect_legacy_with_query_params_fails -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_with_arg -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_query_without_sql_fails -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_run_table_delete -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookRunWithConfiguration::test_run_with_configuration_location -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_create_external_table_with_kms -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_run_copy_with_kms -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_run_load_with_kms -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithKMS::test_run_query_with_kms -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_create_external_table_description -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_create_external_table_labels -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_run_load_description -- tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryWithLabelsAndDescription::test_run_load_labels -- tests/providers/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_load_default -- tests/providers/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_load_with_arg -- tests/providers/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_query_default -- tests/providers/google/cloud/hooks/test_bigquery.py::TestClusteringInRunJob::test_run_query_with_arg -- tests/providers/google/cloud/hooks/test_bigquery.py::TestDatasetsOperations::test_patch_dataset -- tests/providers/google/cloud/hooks/test_bigquery.py::TestTableOperations::test_patch_table -- tests/providers/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_load_default -- tests/providers/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_load_with_arg -- tests/providers/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_query_with_arg -- tests/providers/google/cloud/hooks/test_bigquery.py::TestTimePartitioningInRunJob::test_run_with_auto_detect -- tests/providers/google/cloud/hooks/test_gcs.py::TestGCSHook::test_list__error_match_glob_and_invalid_delimiter -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_error_operation -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_life_science_client_creation -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_run_pipeline_immediately_complete -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithDefaultProjectIdFromConnection::test_waiting_operation -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_delegate_to_runtime_error -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_error_operation -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_life_science_client_creation -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_location_path -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_run_pipeline_immediately_complete -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithPassedProjectId::test_waiting_operation -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithoutProjectId::test_life_science_client_creation -- tests/providers/google/cloud/hooks/test_life_sciences.py::TestLifeSciencesHookWithoutProjectId::test_run_pipeline -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_cancel_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_create_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_delete_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_get_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithDefaultProjectIdHook::test_list_pipeline_jobs -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_cancel_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_create_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_delete_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_get_pipeline_job -- tests/providers/google/cloud/hooks/vertex_ai/test_custom_job.py::TestCustomJobWithoutDefaultProjectIdHook::test_list_pipeline_jobs -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcHadoopOperator::test_execute -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcHiveOperator::test_builder -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcHiveOperator::test_execute -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcPigOperator::test_builder -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcPigOperator::test_execute -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcPySparkOperator::test_execute -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcSparkOperator::test_execute -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcSparkSqlOperator::test_builder -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcSparkSqlOperator::test_execute -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataProcSparkSqlOperator::test_execute_override_project_id -- tests/providers/google/cloud/operators/test_dataproc.py::TestDataprocClusterScaleOperator::test_execute -- tests/providers/google/cloud/operators/test_dataproc.py::test_create_cluster_operator_extra_links -- tests/providers/google/cloud/operators/test_dataproc.py::test_scale_cluster_operator_extra_links -- tests/providers/google/cloud/operators/test_dataproc.py::test_submit_spark_job_operator_extra_links -- tests/providers/google/cloud/operators/test_gcs.py::TestGoogleCloudStorageListOperator::test_execute__delimiter -- tests/providers/google/cloud/operators/test_kubernetes_engine.py::TestGoogleCloudPlatformContainerOperator::test_create_execute_error_body -- tests/providers/google/cloud/operators/test_life_sciences.py::TestLifeSciencesRunPipelineOperator::test_executes -- tests/providers/google/cloud/operators/test_life_sciences.py::TestLifeSciencesRunPipelineOperator::test_executes_without_project_id -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_copy_files_into_a_folder -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_last_modified_time -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_more_than_1_wildcard -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_no_prefix -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_no_suffix -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_prefix_and_suffix -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_empty_destination_object -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_destination_object -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_destination_object_retained_prefix -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_replace_flag_false -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_with_replace_flag_false_with_destination_object -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_execute_wildcard_without_destination_object -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_executes_with_a_delimiter -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_executes_with_delimiter_and_destination_object -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_executes_with_different_delimiter_and_destination_object -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_wc_with_last_modified_time_with_all_true_cond -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_wc_with_last_modified_time_with_one_true_cond -- tests/providers/google/cloud/transfers/test_gcs_to_gcs.py::TestGoogleCloudStorageToCloudStorageOperator::test_wc_with_no_last_modified_time diff --git a/tests/executors/test_base_executor.py b/tests/executors/test_base_executor.py index 04a70c5e83689..da7422737ac4a 100644 --- a/tests/executors/test_base_executor.py +++ b/tests/executors/test_base_executor.py @@ -145,7 +145,7 @@ def test_gauge_executor_metrics_single_executor(mock_stats_gauge, mock_trigger_t @mock.patch("airflow.executors.sequential_executor.SequentialExecutor.sync") @mock.patch("airflow.executors.base_executor.BaseExecutor.trigger_tasks") @mock.patch("airflow.executors.base_executor.Stats.gauge") -@mock.patch("airflow.executors.executor_loader.ExecutorLoader.get_executor_names") +@mock.patch("airflow.executors.base_executor.ExecutorLoader.get_executor_names") def test_gauge_executor_metrics_with_multiple_executors( mock_get_executor_names, mock_stats_gauge, diff --git a/tests/executors/test_executor_loader.py b/tests/executors/test_executor_loader.py index 2192487a01cf8..d9bf81dd3116d 100644 --- a/tests/executors/test_executor_loader.py +++ b/tests/executors/test_executor_loader.py @@ -29,7 +29,8 @@ from airflow.executors.local_executor import LocalExecutor from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/tests/integration/cli/commands/test_celery_command.py b/tests/integration/cli/commands/test_celery_command.py index 61866184337d0..186addedb1164 100644 --- a/tests/integration/cli/commands/test_celery_command.py +++ b/tests/integration/cli/commands/test_celery_command.py @@ -25,7 +25,8 @@ from airflow.cli import cli_parser from airflow.cli.commands import celery_command from airflow.executors import executor_loader -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars @pytest.mark.integration("celery") diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 4ec1cc458c3a5..169d3e91356c9 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -45,7 +45,8 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.state import State, TaskInstanceState -from tests.test_utils import db + +from dev.tests_common.test_utils import db logger = logging.getLogger(__name__) diff --git a/tests/integration/security/test_kerberos.py b/tests/integration/security/test_kerberos.py index 033b455b56ea7..d16b8bc332c7d 100644 --- a/tests/integration/security/test_kerberos.py +++ b/tests/integration/security/test_kerberos.py @@ -26,7 +26,8 @@ from airflow.security import kerberos from airflow.security.kerberos import renew_from_kt -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars @pytest.mark.integration("kerberos") diff --git a/tests/jobs/test_backfill_job.py b/tests/jobs/test_backfill_job.py index 2a5a6689a98f1..f1bb6f17d05c4 100644 --- a/tests/jobs/test_backfill_job.py +++ b/tests/jobs/test_backfill_job.py @@ -59,16 +59,17 @@ from airflow.utils.types import DagRunType from tests.listeners import dag_listener from tests.models import TEST_DAGS_FOLDER -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import ( + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import ( clear_db_dags, clear_db_pools, clear_db_runs, clear_db_xcom, set_default_pool_slots, ) -from tests.test_utils.mock_executor import MockExecutor +from dev.tests_common.test_utils.mock_executor import MockExecutor if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/jobs/test_base_job.py b/tests/jobs/test_base_job.py index e9c9fe94ce737..20079f4f235e5 100644 --- a/tests/jobs/test_base_job.py +++ b/tests/jobs/test_base_job.py @@ -33,9 +33,10 @@ from airflow.utils.session import create_session from airflow.utils.state import State from tests.listeners import lifecycle_listener -from tests.test_utils.config import conf_vars from tests.utils.test_helpers import MockJobRunner, SchedulerJobRunner, TriggererJobRunner +from dev.tests_common.test_utils.config import conf_vars + if TYPE_CHECKING: from airflow.serialization.pydantic.job import JobPydantic diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py index 1a0e186264a72..577fddd84d2b7 100644 --- a/tests/jobs/test_local_task_job.py +++ b/tests/jobs/test_local_task_job.py @@ -52,11 +52,12 @@ from airflow.utils.state import State from airflow.utils.timeout import timeout from airflow.utils.types import DagRunType -from tests.test_utils import db -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.mock_executor import MockExecutor + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.mock_executor import MockExecutor if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index bc2be0a12c743..a5748ebaeef9c 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -72,10 +72,12 @@ from tests.listeners import dag_listener from tests.listeners.test_listeners import get_listener_manager from tests.models import TEST_DAGS_FOLDER -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars, env_vars -from tests.test_utils.db import ( +from tests.utils.test_timezone import UTC + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars, env_vars +from dev.tests_common.test_utils.db import ( clear_db_assets, clear_db_backfills, clear_db_dags, @@ -87,9 +89,8 @@ clear_db_sla_miss, set_default_pool_slots, ) -from tests.test_utils.mock_executor import MockExecutor -from tests.test_utils.mock_operators import CustomOperator -from tests.utils.test_timezone import UTC +from dev.tests_common.test_utils.mock_executor import MockExecutor +from dev.tests_common.test_utils.mock_operators import CustomOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/jobs/test_triggerer_job.py b/tests/jobs/test_triggerer_job.py index 84b422342e4ce..da8405e197317 100644 --- a/tests/jobs/test_triggerer_job.py +++ b/tests/jobs/test_triggerer_job.py @@ -47,7 +47,8 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunType from tests.core.test_logging_config import reset_logging -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/jobs/test_triggerer_job_logging.py b/tests/jobs/test_triggerer_job_logging.py index a039c43fb48ca..4f39047f050ee 100644 --- a/tests/jobs/test_triggerer_job_logging.py +++ b/tests/jobs/test_triggerer_job_logging.py @@ -30,7 +30,8 @@ from airflow.utils.log.file_task_handler import FileTaskHandler from airflow.utils.log.logging_mixin import RedirectStdHandler from airflow.utils.log.trigger_handler import DropTriggerLogsFilter, TriggererHandlerWrapper -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars def non_pytest_handlers(val): diff --git a/tests/lineage/test_hook.py b/tests/lineage/test_hook.py index c076b19aecedd..3fbbfaa021ffa 100644 --- a/tests/lineage/test_hook.py +++ b/tests/lineage/test_hook.py @@ -33,7 +33,8 @@ NoOpCollector, get_hook_lineage_collector, ) -from tests.test_utils.mock_plugins import mock_plugin_manager + +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager class TestHookLineageCollector: diff --git a/tests/lineage/test_lineage.py b/tests/lineage/test_lineage.py index c99c97ceb4e0c..ac42b2ee7f814 100644 --- a/tests/lineage/test_lineage.py +++ b/tests/lineage/test_lineage.py @@ -30,7 +30,8 @@ from airflow.utils import timezone from airflow.utils.context import Context from airflow.utils.types import DagRunType -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/listeners/class_listener.py b/tests/listeners/class_listener.py index ececa853213a5..2b9cefa178d7d 100644 --- a/tests/listeners/class_listener.py +++ b/tests/listeners/class_listener.py @@ -19,7 +19,8 @@ from airflow.listeners import hookimpl from airflow.utils.state import DagRunState, TaskInstanceState -from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS if AIRFLOW_V_2_10_PLUS: diff --git a/tests/listeners/test_dag_import_error_listener.py b/tests/listeners/test_dag_import_error_listener.py index 57fd4f79dd526..886044d27b44e 100644 --- a/tests/listeners/test_dag_import_error_listener.py +++ b/tests/listeners/test_dag_import_error_listener.py @@ -32,8 +32,9 @@ from airflow.models.errors import ParseImportError from airflow.utils import timezone from tests.listeners import dag_import_error_listener -from tests.test_utils.config import conf_vars, env_vars -from tests.test_utils.db import ( + +from dev.tests_common.test_utils.config import conf_vars, env_vars +from dev.tests_common.test_utils.db import ( clear_db_dags, clear_db_import_errors, clear_db_jobs, @@ -42,7 +43,7 @@ clear_db_serialized_dags, clear_db_sla_miss, ) -from tests.test_utils.mock_executor import MockExecutor +from dev.tests_common.test_utils.mock_executor import MockExecutor pytestmark = pytest.mark.db_test diff --git a/tests/models/test_backfill.py b/tests/models/test_backfill.py index c45625db335de..0f471fbd56546 100644 --- a/tests/models/test_backfill.py +++ b/tests/models/test_backfill.py @@ -33,7 +33,13 @@ ) from airflow.operators.python import PythonOperator from airflow.utils.state import DagRunState -from tests.test_utils.db import clear_db_backfills, clear_db_dags, clear_db_runs, clear_db_serialized_dags + +from dev.tests_common.test_utils.db import ( + clear_db_backfills, + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, +) pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag] diff --git a/tests/models/test_base.py b/tests/models/test_base.py index 3224616404d6a..27eeba2912f52 100644 --- a/tests/models/test_base.py +++ b/tests/models/test_base.py @@ -19,7 +19,8 @@ import pytest from airflow.models.base import get_id_collation_args -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index 3c5b7634d5a99..999529e14a997 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -50,7 +50,8 @@ from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from tests.models import DEFAULT_DATE -from tests.test_utils.mock_operators import DeprecatedOperator, MockOperator + +from dev.tests_common.test_utils.mock_operators import DeprecatedOperator, MockOperator if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/tests/models/test_cleartasks.py b/tests/models/test_cleartasks.py index 380c3c28c5143..810453053dd0c 100644 --- a/tests/models/test_cleartasks.py +++ b/tests/models/test_cleartasks.py @@ -35,8 +35,9 @@ from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.types import DagRunType from tests.models import DEFAULT_DATE -from tests.test_utils import db -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index c79ca24e03a25..997ef06329fde 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -102,13 +102,19 @@ StaticTestPriorityWeightStrategy, TestPriorityWeightStrategyPlugin, ) -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_assets, clear_db_dags, clear_db_runs, clear_db_serialized_dags -from tests.test_utils.mapping import expand_mapped_task -from tests.test_utils.mock_plugins import mock_plugin_manager -from tests.test_utils.timetables import cron_timetable, delta_timetable + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import ( + clear_db_assets, + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, +) +from dev.tests_common.test_utils.mapping import expand_mapped_task +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager +from dev.tests_common.test_utils.timetables import cron_timetable, delta_timetable if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -2296,14 +2302,14 @@ def test_next_dagrun_info_on_29_feb(self): ) def test_access_control_format(self, fab_version, perms, expected_exception, expected_perms): if expected_exception: - with patch("airflow.models.dag.FAB_VERSION", fab_version): + with patch("airflow.providers.fab.__version__", fab_version): with pytest.raises( expected_exception, match="Please upgrade the FAB provider to a version >= 1.3.0 to allow use the Dag Level Access Control new format.", ): DAG(dag_id="dag_test", schedule=None, access_control=perms) else: - with patch("airflow.models.dag.FAB_VERSION", fab_version): + with patch("airflow.providers.fab.__version__", fab_version): dag = DAG(dag_id="dag_test", schedule=None, access_control=perms) assert dag.access_control == expected_perms diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py index 0179fa865291d..0b477cc24c636 100644 --- a/tests/models/test_dagbag.py +++ b/tests/models/test_dagbag.py @@ -46,9 +46,10 @@ from airflow.www.security_appless import ApplessAirflowSecurityManager from tests import cluster_policies from tests.models import TEST_DAGS_FOLDER -from tests.test_utils import db -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/models/test_dagcode.py b/tests/models/test_dagcode.py index e566e9ceed080..0bc105cfaf42c 100644 --- a/tests/models/test_dagcode.py +++ b/tests/models/test_dagcode.py @@ -30,7 +30,8 @@ # To move it to a shared module. from airflow.utils.file import open_maybe_zipped from airflow.utils.session import create_session -from tests.test_utils.db import clear_db_dag_code + +from dev.tests_common.test_utils.db import clear_db_dag_code pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index dac2982b0ba34..46773232f77fe 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -47,10 +47,11 @@ from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from tests.models import DEFAULT_DATE as _DEFAULT_DATE -from tests.test_utils import db -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.mock_operators import MockOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dagwarning.py b/tests/models/test_dagwarning.py index d3ef85d351a3f..9b98e0c31d74d 100644 --- a/tests/models/test_dagwarning.py +++ b/tests/models/test_dagwarning.py @@ -25,7 +25,8 @@ from airflow.models import DagModel from airflow.models.dagwarning import DagWarning -from tests.test_utils.db import clear_db_dags + +from dev.tests_common.test_utils.db import clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_mappedoperator.py b/tests/models/test_mappedoperator.py index 0571e07e671f8..927dbfd73a563 100644 --- a/tests/models/test_mappedoperator.py +++ b/tests/models/test_mappedoperator.py @@ -42,8 +42,13 @@ from airflow.utils.trigger_rule import TriggerRule from airflow.utils.xcom import XCOM_RETURN_KEY from tests.models import DEFAULT_DATE -from tests.test_utils.mapping import expand_mapped_task -from tests.test_utils.mock_operators import MockOperator, MockOperatorWithNestedFields, NestedFields + +from dev.tests_common.test_utils.mapping import expand_mapped_task +from dev.tests_common.test_utils.mock_operators import ( + MockOperator, + MockOperatorWithNestedFields, + NestedFields, +) pytestmark = pytest.mark.db_test diff --git a/tests/models/test_param.py b/tests/models/test_param.py index 3d85a957ec5d9..2d324478deefe 100644 --- a/tests/models/test_param.py +++ b/tests/models/test_param.py @@ -26,7 +26,8 @@ from airflow.serialization.serialized_objects import BaseSerialization from airflow.utils import timezone from airflow.utils.types import DagRunType -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom class TestParam: diff --git a/tests/models/test_pool.py b/tests/models/test_pool.py index 3bd3b4f5ffb0f..22852261671ce 100644 --- a/tests/models/test_pool.py +++ b/tests/models/test_pool.py @@ -27,7 +27,13 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State -from tests.test_utils.db import clear_db_dags, clear_db_pools, clear_db_runs, set_default_pool_slots + +from dev.tests_common.test_utils.db import ( + clear_db_dags, + clear_db_pools, + clear_db_runs, + set_default_pool_slots, +) pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index 1de83954ee4bd..ea22d31871db9 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -34,8 +34,9 @@ from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.timezone import datetime -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields pytestmark = pytest.mark.db_test diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py index 93845e95832cf..5e6714feda373 100644 --- a/tests/models/test_serialized_dag.py +++ b/tests/models/test_serialized_dag.py @@ -36,8 +36,9 @@ from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 from airflow.utils.session import create_session -from tests.test_utils import db -from tests.test_utils.asserts import assert_queries_count + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.asserts import assert_queries_count pytestmark = pytest.mark.db_test diff --git a/tests/models/test_skipmixin.py b/tests/models/test_skipmixin.py index c24bfb7c86444..0f2406c5737c5 100644 --- a/tests/models/test_skipmixin.py +++ b/tests/models/test_skipmixin.py @@ -31,7 +31,8 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index c09d3575d1eca..03566dca30bc9 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -99,11 +99,12 @@ from airflow.utils.types import DagRunType from airflow.utils.xcom import XCOM_RETURN_KEY from tests.models import DEFAULT_DATE, TEST_DAGS_FOLDER -from tests.test_utils import db -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_connections, clear_db_runs -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_connections, clear_db_runs +from dev.tests_common.test_utils.mock_operators import MockOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_timestamp.py b/tests/models/test_timestamp.py index a2d4a92f16660..4888c2fdac268 100644 --- a/tests/models/test_timestamp.py +++ b/tests/models/test_timestamp.py @@ -25,7 +25,8 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import State -from tests.test_utils.db import clear_db_dags, clear_db_logs, clear_db_runs + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_logs, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/models/test_trigger.py b/tests/models/test_trigger.py index 407d6edd753a8..b44e4b9f6e881 100644 --- a/tests/models/test_trigger.py +++ b/tests/models/test_trigger.py @@ -42,7 +42,8 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.xcom import XCOM_RETURN_KEY -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/models/test_variable.py b/tests/models/test_variable.py index e9509b4569db4..dfe448092c842 100644 --- a/tests/models/test_variable.py +++ b/tests/models/test_variable.py @@ -27,8 +27,9 @@ from airflow.models import Variable, crypto, variable from airflow.secrets.cache import SecretCache from airflow.secrets.metastore import MetastoreBackend -from tests.test_utils import db -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils import db +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/models/test_xcom.py b/tests/models/test_xcom.py index 07533ec944be8..17ea5fa4ff114 100644 --- a/tests/models/test_xcom.py +++ b/tests/models/test_xcom.py @@ -35,8 +35,9 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.xcom import XCOM_RETURN_KEY -from tests.test_utils.config import conf_vars -from tests.www.test_utils import is_db_isolation_mode + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import is_db_isolation_mode pytestmark = pytest.mark.db_test diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index fcc2e546009c9..b161020d1fb91 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -22,8 +22,9 @@ from airflow.operators.python import PythonOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.types import NOTSET -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/operators/test_branch_operator.py b/tests/operators/test_branch_operator.py index 377473e0cc385..8943bf580e50a 100644 --- a/tests/operators/test_branch_operator.py +++ b/tests/operators/test_branch_operator.py @@ -28,7 +28,8 @@ from airflow.utils.state import State from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/operators/test_email.py b/tests/operators/test_email.py index 04c4cf7fd8f69..c86e0f94f6145 100644 --- a/tests/operators/test_email.py +++ b/tests/operators/test_email.py @@ -24,7 +24,8 @@ from airflow.operators.email import EmailOperator from airflow.utils import timezone -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/operators/test_generic_transfer.py b/tests/operators/test_generic_transfer.py index c877d7bed99cd..e1281ed4b2476 100644 --- a/tests/operators/test_generic_transfer.py +++ b/tests/operators/test_generic_transfer.py @@ -60,7 +60,7 @@ def teardown_method(self): ], ) def test_mysql_to_mysql(self, client): - from tests.providers.mysql.hooks.test_mysql import MySqlContext + from providers.tests.mysql.hooks.test_mysql import MySqlContext with MySqlContext(client): sql = "SELECT * FROM connection;" diff --git a/tests/operators/test_latest_only_operator.py b/tests/operators/test_latest_only_operator.py index 58a43ae7e66b6..78fabc4ca922f 100644 --- a/tests/operators/test_latest_only_operator.py +++ b/tests/operators/test_latest_only_operator.py @@ -30,8 +30,9 @@ from airflow.utils.state import State from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs, clear_db_xcom + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index c98dc7018a4a1..3dd8f506ee986 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -68,9 +68,10 @@ from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import NOTSET, DagRunType -from tests.test_utils import AIRFLOW_MAIN_FOLDER -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/plugins/test_plugin.py b/tests/plugins/test_plugin.py index 995b14e23e751..fd474766d41ea 100644 --- a/tests/plugins/test_plugin.py +++ b/tests/plugins/test_plugin.py @@ -35,7 +35,8 @@ from airflow.timetables.interval import CronDataIntervalTimetable from tests.listeners import empty_listener from tests.listeners.class_listener import ClassBasedListener -from tests.test_utils.mock_operators import ( + +from dev.tests_common.test_utils.mock_operators import ( AirflowLink, AirflowLink2, CustomBaseIndexOpLink, diff --git a/tests/plugins/test_plugins_manager.py b/tests/plugins/test_plugins_manager.py index 7e4bedbfb8c1b..1c54c3ebb195c 100644 --- a/tests/plugins/test_plugins_manager.py +++ b/tests/plugins/test_plugins_manager.py @@ -33,8 +33,9 @@ from airflow.plugins_manager import AirflowPlugin from airflow.utils.module_loading import qualname from airflow.www import app as application -from tests.test_utils.config import conf_vars -from tests.test_utils.mock_plugins import mock_plugin_manager + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager pytestmark = pytest.mark.db_test diff --git a/tests/secrets/test_cache.py b/tests/secrets/test_cache.py index 8d45080868691..40ab4aa290df5 100644 --- a/tests/secrets/test_cache.py +++ b/tests/secrets/test_cache.py @@ -22,7 +22,8 @@ import pytest from airflow.secrets.cache import SecretCache -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars def test_cache_disabled_by_default(): diff --git a/tests/security/test_kerberos.py b/tests/security/test_kerberos.py index b424edb820891..2b661fe9a1478 100644 --- a/tests/security/test_kerberos.py +++ b/tests/security/test_kerberos.py @@ -25,7 +25,8 @@ from airflow.security import kerberos from airflow.security.kerberos import get_kerberos_principle, renew_from_kt -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/sensors/test_base.py b/tests/sensors/test_base.py index 43c8fa5a64bd8..8a3b2b5c5c0fa 100644 --- a/tests/sensors/test_base.py +++ b/tests/sensors/test_base.py @@ -57,7 +57,8 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.timezone import datetime -from tests.test_utils import db + +from dev.tests_common.test_utils import db pytestmark = pytest.mark.db_test @@ -68,7 +69,7 @@ TEST_DAG_ID = "unit_test_dag" DUMMY_OP = "dummy_op" SENSOR_OP = "sensor_op" -DEV_NULL = "dev/null" +DEV_NULL = "/dev/null" @pytest.fixture diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index 9947a197a0335..f1739e6b84418 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -52,9 +52,10 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from tests.models import TEST_DAGS_FOLDER -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs +from dev.tests_common.test_utils.mock_operators import MockOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index f0f517042314a..35f5e196a2222 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -80,9 +80,14 @@ from airflow.utils.operator_resources import Resources from airflow.utils.task_group import TaskGroup from airflow.utils.xcom import XCOM_RETURN_KEY -from tests.test_utils.compat import BaseOperatorLink -from tests.test_utils.mock_operators import AirflowLink2, CustomOperator, GoogleLink, MockOperator -from tests.test_utils.timetables import CustomSerializationTimetable, cron_timetable, delta_timetable + +from dev.tests_common.test_utils.compat import BaseOperatorLink +from dev.tests_common.test_utils.mock_operators import AirflowLink2, CustomOperator, GoogleLink, MockOperator +from dev.tests_common.test_utils.timetables import ( + CustomSerializationTimetable, + cron_timetable, + delta_timetable, +) if TYPE_CHECKING: from airflow.utils.context import Context @@ -185,7 +190,9 @@ "max_retry_delay": 600.0, "downstream_task_ids": [], "_is_empty": False, - "_operator_extra_links": [{"tests.test_utils.mock_operators.CustomOpLink": {}}], + "_operator_extra_links": [ + {"dev.tests_common.test_utils.mock_operators.CustomOpLink": {}} + ], "ui_color": "#fff", "ui_fgcolor": "#000", "template_ext": [], @@ -193,7 +200,7 @@ "template_fields_renderers": {}, "_task_type": "CustomOperator", "_operator_name": "@custom", - "_task_module": "tests.test_utils.mock_operators", + "_task_module": "dev.tests_common.test_utils.mock_operators", "pool": "default_pool", "is_setup": False, "is_teardown": False, @@ -233,7 +240,7 @@ ) CUSTOM_TIMETABLE_SERIALIZED = { - "__type": "tests.test_utils.timetables.CustomSerializationTimetable", + "__type": "dev.tests_common.test_utils.timetables.CustomSerializationTimetable", "__var": {"value": "foo"}, } @@ -310,8 +317,8 @@ def get_excluded_patterns() -> Generator[str, None, None]: if python_version in provider_info.get("excluded-python-versions"): provider_path = provider.replace(".", "/") yield f"airflow/providers/{provider_path}/" - yield f"tests/providers/{provider_path}/" - yield f"tests/system/providers/{provider_path}/" + yield f"providers/tests/{provider_path}/" + yield f"providers/tests/system/{provider_path}/" def collect_dags(dag_folder=None): @@ -328,10 +335,10 @@ def collect_dags(dag_folder=None): else: patterns = [ "airflow/example_dags", - "airflow/providers/*/example_dags", # TODO: Remove once AIP-47 is completed - "airflow/providers/*/*/example_dags", # TODO: Remove once AIP-47 is completed - "tests/system/providers/*/", - "tests/system/providers/*/*/", + "providers/src/airflow/providers/*/example_dags", # TODO: Remove once AIP-47 is completed + "providers/src/airflow/providers/*/*/example_dags", # TODO: Remove once AIP-47 is completed + "providers/tests/system/*/", + "providers/tests/system/*/*/", ] excluded_patterns = [f"{ROOT_FOLDER}/{excluded_pattern}" for excluded_pattern in get_excluded_patterns()] for pattern in patterns: @@ -366,7 +373,7 @@ def timetable_plugin(monkeypatch): monkeypatch.setattr( plugins_manager, "timetable_classes", - {"tests.test_utils.timetables.CustomSerializationTimetable": CustomSerializationTimetable}, + {"dev.tests_common.test_utils.timetables.CustomSerializationTimetable": CustomSerializationTimetable}, ) @@ -462,7 +469,7 @@ def test_dag_serialization_unregistered_custom_timetable(self): message = ( "Failed to serialize DAG 'simple_dag': Timetable class " - "'tests.test_utils.timetables.CustomSerializationTimetable' " + "'dev.tests_common.test_utils.timetables.CustomSerializationTimetable' " "is not registered or " "you have a top level database access that disrupted the session. " "Please check the airflow best practices documentation." @@ -529,8 +536,8 @@ def test_deserialization_across_process(self): def test_roundtrip_provider_example_dags(self): dags = collect_dags( [ - "airflow/providers/*/example_dags", - "airflow/providers/*/*/example_dags", + "src/providers/airflow/providers/*/example_dags", + "src/providers/airflow/providers/*/*/example_dags", ] ) @@ -827,7 +834,7 @@ def test_deserialization_timetable_unregistered(self): SerializedDAG.from_dict(serialized) message = ( "Timetable class " - "'tests.test_utils.timetables.CustomSerializationTimetable' " + "'dev.tests_common.test_utils.timetables.CustomSerializationTimetable' " "is not registered or " "you have a top level database access that disrupted the session. " "Please check the airflow best practices documentation." @@ -979,15 +986,15 @@ def test_task_params_roundtrip(self, val, expected_val): [ pytest.param( "true", - [{"tests.test_utils.mock_operators.CustomOpLink": {}}], + [{"dev.tests_common.test_utils.mock_operators.CustomOpLink": {}}], {"Google Custom": "http://google.com/custom_base_link?search=true"}, id="non-indexed-link", ), pytest.param( ["echo", "true"], [ - {"tests.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 0}}, - {"tests.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 1}}, + {"dev.tests_common.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 0}}, + {"dev.tests_common.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 1}}, ], { "BigQuery Console #1": "https://console.cloud.google.com/bigquery?j=echo", @@ -1290,7 +1297,7 @@ def test_operator_deserialize_old_names(self): "template_fields": ["bash_command"], "template_fields_renderers": {}, "_task_type": "CustomOperator", - "_task_module": "tests.test_utils.mock_operators", + "_task_module": "dev.tests_common.test_utils.mock_operators", "pool": "default_pool", "ui_color": "#fff", "ui_fgcolor": "#000", @@ -2347,7 +2354,7 @@ def test_operator_expand_xcomarg_serde(): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "tests.test_utils.mock_operators", + "_task_module": "dev.tests_common.test_utils.mock_operators", "_task_type": "MockOperator", "downstream_task_ids": [], "expand_input": { @@ -2402,7 +2409,7 @@ def test_operator_expand_kwargs_literal_serde(strict): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "tests.test_utils.mock_operators", + "_task_module": "dev.tests_common.test_utils.mock_operators", "_task_type": "MockOperator", "downstream_task_ids": [], "expand_input": { @@ -2457,7 +2464,7 @@ def test_operator_expand_kwargs_xcomarg_serde(strict): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "tests.test_utils.mock_operators", + "_task_module": "dev.tests_common.test_utils.mock_operators", "_task_type": "MockOperator", "downstream_task_ids": [], "expand_input": { @@ -2820,7 +2827,7 @@ def operator_extra_links(self): "_disallow_kwargs_override": False, "_expand_input_attr": "expand_input", "downstream_task_ids": [], - "_operator_extra_links": [{"tests.test_utils.mock_operators.AirflowLink2": {}}], + "_operator_extra_links": [{"dev.tests_common.test_utils.mock_operators.AirflowLink2": {}}], "ui_color": "#fff", "ui_fgcolor": "#000", "template_ext": [], diff --git a/tests/serialization/test_pydantic_models.py b/tests/serialization/test_pydantic_models.py index 55c61ea220263..423dc41a38068 100644 --- a/tests/serialization/test_pydantic_models.py +++ b/tests/serialization/test_pydantic_models.py @@ -45,7 +45,8 @@ from airflow.utils.state import State from airflow.utils.types import AttributeRemoved, DagRunType from tests.models import DEFAULT_DATE -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/serialization/test_serde.py b/tests/serialization/test_serde.py index a36013d20cfa7..5c915839c6422 100644 --- a/tests/serialization/test_serde.py +++ b/tests/serialization/test_serde.py @@ -43,7 +43,8 @@ serialize, ) from airflow.utils.module_loading import import_string, iter_namespace, qualname -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars @pytest.fixture diff --git a/tests/serialization/test_serialized_objects.py b/tests/serialization/test_serialized_objects.py index 0bc8a67ef879a..67fdceeaf08d1 100644 --- a/tests/serialization/test_serialized_objects.py +++ b/tests/serialization/test_serialized_objects.py @@ -471,7 +471,8 @@ def test_all_pydantic_models_round_trip(): @pytest.mark.db_test def test_serialized_mapped_operator_unmap(dag_maker): from airflow.serialization.serialized_objects import SerializedDAG - from tests.test_utils.mock_operators import MockOperator + + from dev.tests_common.test_utils.mock_operators import MockOperator with dag_maker(dag_id="dag") as dag: MockOperator(task_id="task1", arg1="x") diff --git a/tests/system/core/example_external_task_child_deferrable.py b/tests/system/core/example_external_task_child_deferrable.py index 781ad4ea5ef1f..4e8eca3b15eb2 100644 --- a/tests/system/core/example_external_task_child_deferrable.py +++ b/tests/system/core/example_external_task_child_deferrable.py @@ -34,7 +34,7 @@ ) -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/core/example_external_task_parent_deferrable.py b/tests/system/core/example_external_task_parent_deferrable.py index 8896fdc1b9be4..1a64f7e98ae8a 100644 --- a/tests/system/core/example_external_task_parent_deferrable.py +++ b/tests/system/core/example_external_task_parent_deferrable.py @@ -57,14 +57,14 @@ start >> [trigger_child_task, external_task_sensor] >> end - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run +from dev.tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/example_empty.py b/tests/system/example_empty.py index 143c79fc2f990..de316a11eee77 100644 --- a/tests/system/example_empty.py +++ b/tests/system/example_empty.py @@ -35,14 +35,14 @@ chain(task) - from tests.system.utils.watcher import watcher + from dev.tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from tests.system.utils import get_test_run # noqa: E402 +from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/providers/microsoft/azure/__init__.py b/tests/system/providers/microsoft/azure/__init__.py deleted file mode 100644 index 217e5db960782..0000000000000 --- a/tests/system/providers/microsoft/azure/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/papermill/__init__.py b/tests/system/providers/papermill/__init__.py deleted file mode 100644 index 217e5db960782..0000000000000 --- a/tests/system/providers/papermill/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/task/task_runner/test_standard_task_runner.py b/tests/task/task_runner/test_standard_task_runner.py index 54b8a4d4173e7..55e3d34192d60 100644 --- a/tests/task/task_runner/test_standard_task_runner.py +++ b/tests/task/task_runner/test_standard_task_runner.py @@ -42,8 +42,9 @@ from airflow.utils.timeout import timeout from tests.listeners import xcom_listener from tests.listeners.file_write_listener import FileWriteListener -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/ti_deps/deps/test_pool_slots_available_dep.py b/tests/ti_deps/deps/test_pool_slots_available_dep.py index c81657bbb6488..d7fc33aa41bf8 100644 --- a/tests/ti_deps/deps/test_pool_slots_available_dep.py +++ b/tests/ti_deps/deps/test_pool_slots_available_dep.py @@ -26,7 +26,8 @@ from airflow.ti_deps.deps.pool_slots_available_dep import PoolSlotsAvailableDep from airflow.utils.session import create_session from airflow.utils.state import TaskInstanceState -from tests.test_utils import db + +from dev.tests_common.test_utils import db pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py b/tests/ti_deps/deps/test_prev_dagrun_dep.py index 4354a9237d5f9..638f4b69d7e92 100644 --- a/tests/ti_deps/deps/test_prev_dagrun_dep.py +++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py @@ -29,8 +29,9 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import convert_to_utc, datetime from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py index 568d6abf025c7..d137d43bbf38d 100644 --- a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py +++ b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py @@ -30,7 +30,8 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State -from tests.test_utils import db + +from dev.tests_common.test_utils import db pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/log/test_log_reader.py b/tests/utils/log/test_log_reader.py index 3216222909b7a..53e702efd8a8e 100644 --- a/tests/utils/log/test_log_reader.py +++ b/tests/utils/log/test_log_reader.py @@ -38,8 +38,9 @@ from airflow.utils.log.logging_mixin import ExternalLoggingMixin from airflow.utils.state import TaskInstanceState from airflow.utils.types import DagRunType -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/log/test_secrets_masker.py b/tests/utils/log/test_secrets_masker.py index 5781139677dfd..f3fed73675cce 100644 --- a/tests/utils/log/test_secrets_masker.py +++ b/tests/utils/log/test_secrets_masker.py @@ -38,7 +38,8 @@ should_hide_value_for_key, ) from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.enable_redact p = "password" diff --git a/tests/utils/log/test_task_context_logger.py b/tests/utils/log/test_task_context_logger.py index deca4e55484b4..3a08947ede091 100644 --- a/tests/utils/log/test_task_context_logger.py +++ b/tests/utils/log/test_task_context_logger.py @@ -24,8 +24,9 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.utils.log.task_context_logger import TaskContextLogger -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index ce77f80297fcf..0f3a56a5d1c8d 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -50,7 +50,8 @@ upgradedb, ) from airflow.utils.db_manager import RunDBManager -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py index 0a8cd9c90c962..3b5a3b2227231 100644 --- a/tests/utils/test_db_cleanup.py +++ b/tests/utils/test_db_cleanup.py @@ -48,7 +48,13 @@ run_cleanup, ) from airflow.utils.session import create_session -from tests.test_utils.db import clear_db_assets, clear_db_dags, clear_db_runs, drop_tables_with_prefix + +from dev.tests_common.test_utils.db import ( + clear_db_assets, + clear_db_dags, + clear_db_runs, + drop_tables_with_prefix, +) pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/test_db_manager.py b/tests/utils/test_db_manager.py index 1c8a6c6c7dfc8..975ea65499aae 100644 --- a/tests/utils/test_db_manager.py +++ b/tests/utils/test_db_manager.py @@ -25,7 +25,8 @@ from airflow.models import Base from airflow.utils.db import downgrade, initdb from airflow.utils.db_manager import BaseDBManager, RunDBManager -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test] diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index 0376848fce829..d1521ca6e0f69 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -29,8 +29,9 @@ from airflow.utils import dot_renderer, timezone from airflow.utils.state import State from airflow.utils.task_group import TaskGroup -from tests.test_utils.compat import BashOperator -from tests.test_utils.db import clear_db_dags + +from dev.tests_common.test_utils.compat import BashOperator +from dev.tests_common.test_utils.db import clear_db_dags START_DATE = timezone.utcnow() diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index bf5f3fc0a18fb..b47dcbc87585a 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -29,7 +29,8 @@ from airflow.configuration import conf from airflow.utils import email -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars EMAILS = ["test1@example.com", "test2@example.com"] diff --git a/tests/utils/test_file.py b/tests/utils/test_file.py index 50cc37ca57ce3..2ffb89ec34196 100644 --- a/tests/utils/test_file.py +++ b/tests/utils/test_file.py @@ -27,7 +27,8 @@ from airflow.utils import file as file_utils from airflow.utils.file import correct_maybe_zipped, find_path_from_directory, open_maybe_zipped from tests.models import TEST_DAGS_FOLDER -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars def might_contain_dag(file_path: str, zip_file: zipfile.ZipFile | None = None): diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index 478604186e3a4..0b0046b1ba082 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -37,8 +37,9 @@ validate_key, ) from airflow.utils.types import NOTSET -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs if TYPE_CHECKING: from airflow.jobs.job import Job diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py index 8de3b03a1af1b..4be3564567ebd 100644 --- a/tests/utils/test_log_handlers.py +++ b/tests/utils/test_log_handlers.py @@ -54,8 +54,9 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_net.py b/tests/utils/test_net.py index 50d44cf125b3c..60823911b8b03 100644 --- a/tests/utils/test_net.py +++ b/tests/utils/test_net.py @@ -24,7 +24,8 @@ from airflow.exceptions import AirflowConfigException from airflow.utils import net -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars def get_hostname(): diff --git a/tests/utils/test_serve_logs.py b/tests/utils/test_serve_logs.py index b6756502c10c2..8d41202be2545 100644 --- a/tests/utils/test_serve_logs.py +++ b/tests/utils/test_serve_logs.py @@ -28,7 +28,8 @@ from airflow.utils import timezone from airflow.utils.jwt_signer import JWTSigner from airflow.utils.serve_logs import create_app -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars if TYPE_CHECKING: from flask.testing import FlaskClient diff --git a/tests/utils/test_sqlalchemy.py b/tests/utils/test_sqlalchemy.py index 4352898090fec..42717a4fb045a 100644 --- a/tests/utils/test_sqlalchemy.py +++ b/tests/utils/test_sqlalchemy.py @@ -42,7 +42,8 @@ ) from airflow.utils.state import State from airflow.utils.timezone import utcnow -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_state.py b/tests/utils/test_state.py index 9356a9bbc54b5..6447477ef589b 100644 --- a/tests/utils/test_state.py +++ b/tests/utils/test_state.py @@ -26,7 +26,8 @@ from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType from tests.models import DEFAULT_DATE -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index a6008dc58c03f..6ea0fcdc36e13 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -39,7 +39,8 @@ from airflow.utils.dag_edges import dag_edges from airflow.utils.task_group import TASKGROUP_ARGS_EXPECTED_TYPES, TaskGroup, task_group_to_dict from tests.models import DEFAULT_DATE -from tests.test_utils.compat import BashOperator + +from dev.tests_common.test_utils.compat import BashOperator def make_task(name, type_="classic"): @@ -1414,7 +1415,7 @@ def test_task_group_edge_modifier_chain(): def test_mapped_task_group_id_prefix_task_id(): - from tests.test_utils.mock_operators import MockOperator + from dev.tests_common.test_utils.mock_operators import MockOperator with DAG(dag_id="d", schedule=None, start_date=DEFAULT_DATE) as dag: t1 = MockOperator.partial(task_id="t1").expand(arg1=[]) diff --git a/tests/utils/test_task_handler_with_custom_formatter.py b/tests/utils/test_task_handler_with_custom_formatter.py index a328253b7f83b..aee646d858b15 100644 --- a/tests/utils/test_task_handler_with_custom_formatter.py +++ b/tests/utils/test_task_handler_with_custom_formatter.py @@ -28,9 +28,10 @@ from airflow.utils.state import DagRunState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_types.py b/tests/utils/test_types.py index a338f6e65887f..cb5849b20599a 100644 --- a/tests/utils/test_types.py +++ b/tests/utils/test_types.py @@ -26,7 +26,8 @@ from airflow.utils.state import State from airflow.utils.types import DagRunType from tests.models import DEFAULT_DATE -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/test_app.py b/tests/www/test_app.py index 053fd1174dcae..673abd371dfb5 100644 --- a/tests/www/test_app.py +++ b/tests/www/test_app.py @@ -29,8 +29,9 @@ from airflow.exceptions import AirflowConfigException from airflow.www import app as application -from tests.test_utils.config import conf_vars -from tests.test_utils.decorators import dont_initialize_flask_app_submodules + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index 2ba1364328943..f8f0ae983382e 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -18,7 +18,6 @@ from __future__ import annotations import itertools -import os import re import time from datetime import datetime @@ -45,7 +44,8 @@ wrapped_markdown, ) from airflow.www.widgets import AirflowDateTimePickerROWidget, BS3TextAreaROWidget, BS3TextFieldROWidget -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -712,7 +712,3 @@ class TestForm(FlaskForm): assert 'readonly="true"' in html_output assert "form-control" in html_output - - -def is_db_isolation_mode(): - return os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true" diff --git a/tests/www/views/conftest.py b/tests/www/views/conftest.py index a4b9145797c9b..faa1cc23a8d36 100644 --- a/tests/www/views/conftest.py +++ b/tests/www/views/conftest.py @@ -27,10 +27,15 @@ from airflow import settings from airflow.models import DagBag from airflow.www.app import create_app -from tests.test_utils.api_connexion_utils import delete_user -from tests.test_utils.config import conf_vars -from tests.test_utils.decorators import dont_initialize_flask_app_submodules -from tests.test_utils.www import client_with_login, client_without_login, client_without_login_as_admin + +from dev.tests_common.test_utils.api_connexion_utils import delete_user +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from dev.tests_common.test_utils.www import ( + client_with_login, + client_without_login, + client_without_login_as_admin, +) @pytest.fixture(autouse=True, scope="module") diff --git a/tests/www/views/test_anonymous_as_admin_role.py b/tests/www/views/test_anonymous_as_admin_role.py index b7603d1eae5bb..700f03f8e63c8 100644 --- a/tests/www/views/test_anonymous_as_admin_role.py +++ b/tests/www/views/test_anonymous_as_admin_role.py @@ -23,7 +23,8 @@ from airflow.models import Pool from airflow.utils.session import create_session -from tests.test_utils.www import check_content_in_response + +from dev.tests_common.test_utils.www import check_content_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_session.py b/tests/www/views/test_session.py index 0ec219aaeb4b3..009917606bda3 100644 --- a/tests/www/views/test_session.py +++ b/tests/www/views/test_session.py @@ -22,8 +22,9 @@ from airflow.exceptions import AirflowConfigException from airflow.www import app -from tests.test_utils.config import conf_vars -from tests.test_utils.decorators import dont_initialize_flask_app_submodules + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index b51fd3ab1c0a5..bae022316c73d 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -42,10 +42,11 @@ get_task_stats_from_query, get_value_from_path, ) -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.mock_plugins import mock_plugin_manager -from tests.test_utils.www import check_content_in_response, check_content_not_in_response + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager +from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -329,7 +330,8 @@ def test_mark_task_instance_state(test_app): from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from airflow.www.views import Airflow - from tests.test_utils.db import clear_db_runs + + from dev.tests_common.test_utils.db import clear_db_runs clear_db_runs() start_date = datetime(2020, 1, 1) @@ -422,7 +424,8 @@ def test_mark_task_group_state(test_app): from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from airflow.www.views import Airflow - from tests.test_utils.db import clear_db_runs + + from dev.tests_common.test_utils.db import clear_db_runs clear_db_runs() start_date = datetime(2020, 1, 1) diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index 7b60a2c11ecad..23fb26c18cf98 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -30,11 +30,16 @@ from airflow.utils.state import State from airflow.utils.types import DagRunType from airflow.www.views import FILTER_STATUS_COOKIE -from tests.test_utils.api_connexion_utils import create_user_scope -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs -from tests.test_utils.permissions import _resource_name -from tests.test_utils.www import check_content_in_response, check_content_not_in_response, client_with_login + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs +from dev.tests_common.test_utils.permissions import _resource_name +from dev.tests_common.test_utils.www import ( + check_content_in_response, + check_content_not_in_response, + client_with_login, +) +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user_scope if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_base.py b/tests/www/views/test_views_base.py index a125ca2d72835..692f0301e2276 100644 --- a/tests/www/views/test_views_base.py +++ b/tests/www/views/test_views_base.py @@ -27,9 +27,10 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.www import app as application -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.config import conf_vars -from tests.test_utils.www import check_content_in_response, check_content_not_in_response + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_cluster_activity.py b/tests/www/views/test_views_cluster_activity.py index 543d563e8da03..6b83de3c7682a 100644 --- a/tests/www/views/test_views_cluster_activity.py +++ b/tests/www/views/test_views_cluster_activity.py @@ -26,7 +26,8 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType -from tests.test_utils.db import clear_db_runs + +from dev.tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_configuration.py b/tests/www/views/test_views_configuration.py index e874cf3267916..90ab9b7faa933 100644 --- a/tests/www/views/test_views_configuration.py +++ b/tests/www/views/test_views_configuration.py @@ -21,8 +21,9 @@ import pytest from airflow.configuration import conf -from tests.test_utils.config import conf_vars -from tests.test_utils.www import check_content_in_response, check_content_not_in_response + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 7530a1f677f9b..66e571780eb96 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -27,7 +27,12 @@ from airflow.models import Connection from airflow.utils.session import create_session from airflow.www.views import ConnectionFormWidget, ConnectionModelView -from tests.test_utils.www import _check_last_log, _check_last_log_masked_connection, check_content_in_response + +from dev.tests_common.test_utils.www import ( + _check_last_log, + _check_last_log_masked_connection, + check_content_in_response, +) pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_custom_user_views.py b/tests/www/views/test_views_custom_user_views.py index 84947a8e5f36f..49ecc2c4f30b2 100644 --- a/tests/www/views/test_views_custom_user_views.py +++ b/tests/www/views/test_views_custom_user_views.py @@ -27,11 +27,16 @@ from airflow import settings from airflow.security import permissions from airflow.www import app as application -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( - create_user as create_user, + +from dev.tests_common.test_utils.www import ( + check_content_in_response, + check_content_not_in_response, + client_with_login, +) +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( + create_user, delete_role, ) -from tests.test_utils.www import check_content_in_response, check_content_not_in_response, client_with_login pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_dagrun.py b/tests/www/views/test_views_dagrun.py index 9e45464c28aa1..3fb8eb02fb633 100644 --- a/tests/www/views/test_views_dagrun.py +++ b/tests/www/views/test_views_dagrun.py @@ -24,14 +24,19 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.www.views import DagRunModelView -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( +from tests.www.views.test_views_tasks import _get_appbuilder_pk_string + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.www import ( + check_content_in_response, + check_content_not_in_response, + client_with_login, +) +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, delete_user, ) -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.www import check_content_in_response, check_content_not_in_response, client_with_login -from tests.www.views.test_views_tasks import _get_appbuilder_pk_string if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_dataset.py b/tests/www/views/test_views_dataset.py index 3db16a996862e..899d64e9754ed 100644 --- a/tests/www/views/test_views_dataset.py +++ b/tests/www/views/test_views_dataset.py @@ -24,8 +24,9 @@ from airflow.assets import Asset from airflow.models.asset import AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.db import clear_db_assets + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.db import clear_db_assets pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py index 86bce2f3bbbc2..bdc402989fd0c 100644 --- a/tests/www/views/test_views_decorators.py +++ b/tests/www/views/test_views_decorators.py @@ -23,9 +23,14 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.db import clear_db_runs, clear_db_variables -from tests.test_utils.www import _check_last_log, _check_last_log_masked_variable, check_content_in_response + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_variables +from dev.tests_common.test_utils.www import ( + _check_last_log, + _check_last_log_masked_variable, + check_content_in_response, +) if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_extra_links.py b/tests/www/views/test_views_extra_links.py index 4fb0d2d40b017..fbc11bf51f337 100644 --- a/tests/www/views/test_views_extra_links.py +++ b/tests/www/views/test_views_extra_links.py @@ -28,9 +28,10 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink -from tests.test_utils.db import clear_db_runs -from tests.test_utils.mock_operators import AirflowLink, Dummy2TestOperator, Dummy3TestOperator + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink +from dev.tests_common.test_utils.db import clear_db_runs +from dev.tests_common.test_utils.mock_operators import AirflowLink, Dummy2TestOperator, Dummy3TestOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_grid.py b/tests/www/views/test_views_grid.py index 8d5e4e22e04b8..4f05f2a13456b 100644 --- a/tests/www/views/test_views_grid.py +++ b/tests/www/views/test_views_grid.py @@ -35,9 +35,10 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType from airflow.www.views import dag_to_grid -from tests.test_utils.asserts import assert_queries_count -from tests.test_utils.db import clear_db_assets, clear_db_runs -from tests.test_utils.mock_operators import MockOperator + +from dev.tests_common.test_utils.asserts import assert_queries_count +from dev.tests_common.test_utils.db import clear_db_assets, clear_db_runs +from dev.tests_common.test_utils.mock_operators import MockOperator pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_home.py b/tests/www/views/test_views_home.py index 6a5bf3c4ad531..4de032c58176e 100644 --- a/tests/www/views/test_views_home.py +++ b/tests/www/views/test_views_home.py @@ -27,10 +27,15 @@ from airflow.utils.state import State from airflow.www.utils import UIAlert from airflow.www.views import FILTER_LASTRUN_COOKIE, FILTER_STATUS_COOKIE, FILTER_TAGS_COOKIE -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user -from tests.test_utils.db import clear_db_dags, clear_db_import_errors, clear_db_serialized_dags -from tests.test_utils.permissions import _resource_name -from tests.test_utils.www import check_content_in_response, check_content_not_in_response, client_with_login + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_import_errors, clear_db_serialized_dags +from dev.tests_common.test_utils.permissions import _resource_name +from dev.tests_common.test_utils.www import ( + check_content_in_response, + check_content_not_in_response, + client_with_login, +) +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_log.py b/tests/www/views/test_views_log.py index 72e282c7b3f8c..c025ebcbbe8ed 100644 --- a/tests/www/views/test_views_log.py +++ b/tests/www/views/test_views_log.py @@ -40,11 +40,12 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType from airflow.www.app import create_app -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_dags, clear_db_runs -from tests.test_utils.decorators import dont_initialize_flask_app_submodules -from tests.test_utils.www import client_with_login + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from dev.tests_common.test_utils.www import client_with_login if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_mount.py b/tests/www/views/test_views_mount.py index f0c052294b60a..e58675439ec12 100644 --- a/tests/www/views/test_views_mount.py +++ b/tests/www/views/test_views_mount.py @@ -22,7 +22,8 @@ import werkzeug.wrappers from airflow.www.app import create_app -from tests.test_utils.config import conf_vars + +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_paused.py b/tests/www/views/test_views_paused.py index 46b0a3aa03f1a..551cdd180e078 100644 --- a/tests/www/views/test_views_paused.py +++ b/tests/www/views/test_views_paused.py @@ -19,7 +19,8 @@ import pytest from airflow.models.log import Log -from tests.test_utils.db import clear_db_dags + +from dev.tests_common.test_utils.db import clear_db_dags pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_pool.py b/tests/www/views/test_views_pool.py index 020c5353a5f34..9b9ddfaeeb79e 100644 --- a/tests/www/views/test_views_pool.py +++ b/tests/www/views/test_views_pool.py @@ -23,7 +23,8 @@ from airflow.models import Pool from airflow.utils.session import create_session -from tests.test_utils.www import check_content_in_response, check_content_not_in_response + +from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_rate_limit.py b/tests/www/views/test_views_rate_limit.py index 1f3eaf11d2ad0..d843c8ed54b09 100644 --- a/tests/www/views/test_views_rate_limit.py +++ b/tests/www/views/test_views_rate_limit.py @@ -20,9 +20,10 @@ import pytest from airflow.www.app import create_app -from tests.test_utils.config import conf_vars -from tests.test_utils.decorators import dont_initialize_flask_app_submodules -from tests.test_utils.www import client_with_login + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from dev.tests_common.test_utils.www import client_with_login pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 87e693d5400ab..6d96cdc4d625f 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -34,10 +34,15 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType -from tests.conftest import initial_db_init -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields -from tests.test_utils.www import check_content_in_response, check_content_not_in_response + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator +from dev.tests_common.test_utils.db import ( + clear_db_dags, + clear_db_runs, + clear_rendered_ti_fields, + initial_db_init, +) +from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_robots.py b/tests/www/views/test_views_robots.py index 03d8547c04d4b..41f963b3bf218 100644 --- a/tests/www/views/test_views_robots.py +++ b/tests/www/views/test_views_robots.py @@ -18,7 +18,7 @@ import pytest -from tests.test_utils.config import conf_vars +from dev.tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_task_norun.py b/tests/www/views/test_views_task_norun.py index 2a39b2a60134e..2ff6ff9d7cd67 100644 --- a/tests/www/views/test_views_task_norun.py +++ b/tests/www/views/test_views_task_norun.py @@ -22,7 +22,7 @@ import pytest -from tests.test_utils.db import clear_db_runs +from dev.tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 4dcb7252a3658..4e9de5c00fc9c 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -43,15 +43,20 @@ from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType from airflow.www.views import TaskInstanceModelView, _safe_parse_datetime -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( + +from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom +from dev.tests_common.test_utils.www import ( + check_content_in_response, + check_content_not_in_response, + client_with_login, +) +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, delete_user, ) -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator -from tests.test_utils.config import conf_vars -from tests.test_utils.db import clear_db_runs, clear_db_xcom -from tests.test_utils.www import check_content_in_response, check_content_not_in_response, client_with_login if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_trigger_dag.py b/tests/www/views/test_views_trigger_dag.py index 0c9384a195f5e..d2c3817ede6db 100644 --- a/tests/www/views/test_views_trigger_dag.py +++ b/tests/www/views/test_views_trigger_dag.py @@ -32,9 +32,10 @@ from airflow.utils.json import WebEncoder from airflow.utils.session import create_session from airflow.utils.types import DagRunType -from tests.test_utils.api_connexion_utils import create_test_client -from tests.test_utils.config import conf_vars -from tests.test_utils.www import check_content_in_response + +from dev.tests_common.test_utils.config import conf_vars +from dev.tests_common.test_utils.www import check_content_in_response +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_test_client pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_variable.py b/tests/www/views/test_views_variable.py index b7fa8b37c52c8..dae1f2e9b7315 100644 --- a/tests/www/views/test_views_variable.py +++ b/tests/www/views/test_views_variable.py @@ -25,13 +25,14 @@ from airflow.models import Variable from airflow.security import permissions from airflow.utils.session import create_session -from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user -from tests.test_utils.www import ( + +from dev.tests_common.test_utils.www import ( _check_last_log, check_content_in_response, check_content_not_in_response, client_with_login, ) +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user pytestmark = pytest.mark.db_test VARIABLE = { From 875b700a017eb73c448d3447fff0c8b987d3e9bc Mon Sep 17 00:00:00 2001 From: sonu4578 Date: Wed, 9 Oct 2024 13:03:40 -0700 Subject: [PATCH 040/199] Improving validation of task retries to handle None values (#42532) * Improving validation of task retries to handle None values * Updated the validation check to use the "or" logical operator Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> * Added a comment for the new validation check * Update baseoperator.py * Update taskinstance.py --------- Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> --- airflow/models/baseoperator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 9e0c8e1e69b61..7b5c8e7c4f8c4 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -127,7 +127,9 @@ def parse_retries(retries: Any) -> int | None: - if retries is None or type(retries) == int: # noqa: E721 + if retries is None: + return 0 + elif type(retries) == int: # noqa: E721 return retries try: parsed_retries = int(retries) From 154d70ceb85bc69171a972253912a9f0488d9cc4 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 9 Oct 2024 21:31:26 +0100 Subject: [PATCH 041/199] Fix deprecated stage names for Pre-commit (#42872) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before ``` ❯ pre-commit install [WARNING] top-level `default_stages` uses deprecated stage names (commit, push) which will be removed in a future version. run: `pre-commit migrate-config` to automatically fix this. pre-commit installed at .git/hooks/pre-commit ``` --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 748a85e860bb7..394e11ac20502 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. --- -default_stages: [commit, push] +default_stages: [pre-commit, pre-push] default_language_version: python: python3 node: 22.2.0 From d75a543922a8a1994f889914c44508895b6739dc Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Thu, 10 Oct 2024 04:32:28 +0800 Subject: [PATCH 042/199] Fix ui lint pre-commit interactive mode (#42854) --- scripts/ci/pre_commit/compile_ui_assets.py | 5 ++++- scripts/ci/pre_commit/compile_ui_assets_dev.py | 2 +- scripts/ci/pre_commit/lint_ui.py | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/ci/pre_commit/compile_ui_assets.py b/scripts/ci/pre_commit/compile_ui_assets.py index 9189981069764..b4c38e3d7cc60 100755 --- a/scripts/ci/pre_commit/compile_ui_assets.py +++ b/scripts/ci/pre_commit/compile_ui_assets.py @@ -68,7 +68,10 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) -> shutil.rmtree(dist_directory, ignore_errors=True) env = os.environ.copy() env["FORCE_COLOR"] = "true" - subprocess.check_call(["pnpm", "install", "--frozen-lockfile"], cwd=os.fspath(ui_directory)) + subprocess.check_call( + ["pnpm", "install", "--frozen-lockfile", "--config.confirmModulesPurge=false"], + cwd=os.fspath(ui_directory), + ) subprocess.check_call(["pnpm", "run", "build"], cwd=os.fspath(ui_directory), env=env) new_hash = get_directory_hash(ui_directory, skip_path_regexp=r".*node_modules.*") UI_HASH_FILE.write_text(new_hash) diff --git a/scripts/ci/pre_commit/compile_ui_assets_dev.py b/scripts/ci/pre_commit/compile_ui_assets_dev.py index 02b2e580946af..d820db8701eba 100755 --- a/scripts/ci/pre_commit/compile_ui_assets_dev.py +++ b/scripts/ci/pre_commit/compile_ui_assets_dev.py @@ -49,7 +49,7 @@ UI_ASSET_OUT_FILE.unlink(missing_ok=True) with open(UI_ASSET_OUT_DEV_MODE_FILE, "w") as f: subprocess.run( - ["pnpm", "install", "--frozen-lockfile"], + ["pnpm", "install", "--frozen-lockfile", "--config.confirmModulesPurge=false"], cwd=os.fspath(ui_directory), check=True, stdout=f, diff --git a/scripts/ci/pre_commit/lint_ui.py b/scripts/ci/pre_commit/lint_ui.py index 9604db170ae91..bac91b6dfc20f 100755 --- a/scripts/ci/pre_commit/lint_ui.py +++ b/scripts/ci/pre_commit/lint_ui.py @@ -29,7 +29,9 @@ if __name__ == "__main__": dir = Path("airflow") / "ui" subprocess.check_call(["pnpm", "config", "set", "store-dir", ".pnpm-store"], cwd=dir) - subprocess.check_call(["pnpm", "install", "--frozen-lockfile"], cwd=dir) + subprocess.check_call( + ["pnpm", "install", "--frozen-lockfile", "--config.confirmModulesPurge=false"], cwd=dir + ) subprocess.check_call(["pnpm", "codegen"], cwd=dir) subprocess.check_call(["pnpm", "format"], cwd=dir) subprocess.check_call(["pnpm", "lint:fix"], cwd=dir) From 50f1b4243b9538b2d4d9d2cbf1489d671a5cd1b5 Mon Sep 17 00:00:00 2001 From: GPK Date: Wed, 9 Oct 2024 21:54:19 +0100 Subject: [PATCH 043/199] update k8s tests urllib3 retry config status_forcelist and allowed_methods (#42871) --- kubernetes_tests/test_base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py index 0f5b673db2f93..ac311daa52f8c 100644 --- a/kubernetes_tests/test_base.py +++ b/kubernetes_tests/test_base.py @@ -123,7 +123,12 @@ def _delete_airflow_pod(name=""): def _get_session_with_retries(self): session = requests.Session() session.auth = ("admin", "admin") - retries = Retry(total=3, backoff_factor=1) + retries = Retry( + total=3, + backoff_factor=1, + status_forcelist=[404], + allowed_methods=Retry.DEFAULT_ALLOWED_METHODS | frozenset(["PATCH", "POST"]), + ) session.mount("http://", HTTPAdapter(max_retries=retries)) session.mount("https://", HTTPAdapter(max_retries=retries)) return session From 17c6382222587a518c3b820c64c67aeaa127afd2 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 9 Oct 2024 21:59:36 +0100 Subject: [PATCH 044/199] Bump `blacken-docs` pre-commit (#42880) Bump `blacken-docs` pre-commit from 1.18.0 -> 1.19.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 394e11ac20502..d492747856a92 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -227,7 +227,7 @@ repos: additional_dependencies: ["libcst>=1.1.0"] files: ^(providers/src/)?airflow/.*/(sensors|operators)/.*\.py$ - repo: https://github.com/asottile/blacken-docs - rev: 1.18.0 + rev: 1.19.0 hooks: - id: blacken-docs name: Run black on docs From 98d354b57f8fbc13245f187bbdf0dfe5bcaf0b2d Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 9 Oct 2024 23:02:42 +0100 Subject: [PATCH 045/199] Exclude "not-ready" providers when building docs (#42873) This fixes the issue we saw in https://apache-airflow.slack.com/archives/C03G9H97MM2/p1728492002477169 This PR excludes the providers that aren't ready yet for building docs. --- .../authoring-and-scheduling/{assets.rst => datasets.rst} | 0 docs/apache-airflow/authoring-and-scheduling/index.rst | 2 +- docs/apache-airflow/templates-ref.rst | 6 +++--- docs/exts/provider_yaml_utils.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename docs/apache-airflow/authoring-and-scheduling/{assets.rst => datasets.rst} (100%) diff --git a/docs/apache-airflow/authoring-and-scheduling/assets.rst b/docs/apache-airflow/authoring-and-scheduling/datasets.rst similarity index 100% rename from docs/apache-airflow/authoring-and-scheduling/assets.rst rename to docs/apache-airflow/authoring-and-scheduling/datasets.rst diff --git a/docs/apache-airflow/authoring-and-scheduling/index.rst b/docs/apache-airflow/authoring-and-scheduling/index.rst index 5ec94d6ca7301..28cc46ab0617a 100644 --- a/docs/apache-airflow/authoring-and-scheduling/index.rst +++ b/docs/apache-airflow/authoring-and-scheduling/index.rst @@ -41,5 +41,5 @@ It's recommended that you first review the pages in :doc:`core concepts timetable diff --git a/docs/apache-airflow/templates-ref.rst b/docs/apache-airflow/templates-ref.rst index 5524c82f8cc95..e3de362191340 100644 --- a/docs/apache-airflow/templates-ref.rst +++ b/docs/apache-airflow/templates-ref.rst @@ -62,10 +62,10 @@ Variable Type Description ``{{ prev_end_date_success }}`` `pendulum.DateTime`_ End date from prior successful :class:`~airflow.models.dagrun.DagRun` (if available). | ``None`` ``{{ inlets }}`` list List of inlets declared on the task. -``{{ inlet_events }}`` dict[str, ...] Access past events of inlet assets. See :doc:`Assets `. Added in version 2.10. +``{{ inlet_events }}`` dict[str, ...] Access past events of inlet assets. See :doc:`Assets `. Added in version 2.10. ``{{ outlets }}`` list List of outlets declared on the task. ``{{ outlet_events }}`` dict[str, ...] | Accessors to attach information to asset events that will be emitted by the current task. - | See :doc:`Assets `. Added in version 2.10. + | See :doc:`Assets `. Added in version 2.10. ``{{ dag }}`` DAG The currently running :class:`~airflow.models.dag.DAG`. You can read more about DAGs in :doc:`DAGs `. ``{{ task }}`` BaseOperator | The currently running :class:`~airflow.models.baseoperator.BaseOperator`. You can read more about Tasks in :doc:`core-concepts/operators` ``{{ macros }}`` | A reference to the macros package. See Macros_ below. @@ -90,7 +90,7 @@ Variable Type Description | Added in version 2.5. ``{{ triggering_asset_events }}`` dict[str, | If in a Asset Scheduled DAG, a map of Asset URI to a list of triggering :class:`~airflow.models.asset.AssetEvent` list[AssetEvent]] | (there may be more than one, if there are multiple Assets with different frequencies). - | Read more here :doc:`Assets `. + | Read more here :doc:`Assets `. | Added in version 2.4. =========================================== ===================== =================================================================== diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index 6e5d3a835e1d0..ad99da3c10eaf 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -70,7 +70,7 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: except jsonschema.ValidationError as ex: msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" raise RuntimeError(msg) - if provider["state"] == "suspended" and not include_suspended: + if provider["state"] in ["suspended", "not-ready"] and not include_suspended: continue provider_yaml_dir = os.path.dirname(provider_yaml_path) provider["python-module"] = _filepath_to_module(provider_yaml_dir) From c3a8b33481ec97de0d9dfffc426f530c4e69fd7e Mon Sep 17 00:00:00 2001 From: GPK Date: Thu, 10 Oct 2024 06:50:43 +0100 Subject: [PATCH 046/199] Move Hooks to Standard provider (#42794) --- airflow/providers_manager.py | 4 +- airflow/sensors/filesystem.py | 2 +- .../logging-monitoring/errors.rst | 2 +- .../operators-and-hooks-ref.rst | 4 +- .../providers/standard/hooks/__init__.py | 16 ++++++++ .../providers/standard}/hooks/filesystem.py | 0 .../standard}/hooks/package_index.py | 0 .../providers/standard}/hooks/subprocess.py | 4 +- .../providers/standard/operators/bash.py | 2 +- .../airflow/providers/standard/provider.yaml | 7 ++++ providers/tests/standard/hooks/__init__.py | 16 ++++++++ .../tests/standard/hooks/test_filesystem.py | 39 +++++++++++++++++++ .../standard}/hooks/test_package_index.py | 6 +-- .../tests/standard}/hooks/test_subprocess.py | 6 +-- tests/sensors/test_filesystem.py | 2 +- 15 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 providers/src/airflow/providers/standard/hooks/__init__.py rename {airflow => providers/src/airflow/providers/standard}/hooks/filesystem.py (100%) rename {airflow => providers/src/airflow/providers/standard}/hooks/package_index.py (100%) rename {airflow => providers/src/airflow/providers/standard}/hooks/subprocess.py (96%) create mode 100644 providers/tests/standard/hooks/__init__.py create mode 100644 providers/tests/standard/hooks/test_filesystem.py rename {tests => providers/tests/standard}/hooks/test_package_index.py (93%) rename {tests => providers/tests/standard}/hooks/test_subprocess.py (95%) diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index 1b1ca469f21d5..573d256d6e59a 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -36,8 +36,8 @@ from packaging.utils import canonicalize_name from airflow.exceptions import AirflowOptionalProviderFeatureException -from airflow.hooks.filesystem import FSHook -from airflow.hooks.package_index import PackageIndexHook +from airflow.providers.standard.hooks.filesystem import FSHook +from airflow.providers.standard.hooks.package_index import PackageIndexHook from airflow.typing_compat import ParamSpec from airflow.utils import yaml from airflow.utils.entry_points import entry_points_with_dist diff --git a/airflow/sensors/filesystem.py b/airflow/sensors/filesystem.py index 5d32ab07ad4e7..4496f5d6abfa4 100644 --- a/airflow/sensors/filesystem.py +++ b/airflow/sensors/filesystem.py @@ -25,7 +25,7 @@ from airflow.configuration import conf from airflow.exceptions import AirflowException -from airflow.hooks.filesystem import FSHook +from airflow.providers.standard.hooks.filesystem import FSHook from airflow.sensors.base import BaseSensorOperator from airflow.triggers.base import StartTriggerArgs from airflow.triggers.file import FileTrigger diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst index cb09843422321..0ad3fa8c5127a 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst @@ -96,7 +96,7 @@ Impact of Sentry on Environment variables passed to Subprocess Hook When Sentry is enabled, by default it changes the standard library to pass all environment variables to subprocesses opened by Airflow. This changes the default behaviour of -:class:`airflow.hooks.subprocess.SubprocessHook` - always all environment variables are passed to the +:class:`airflow.providers.standard.hooks.subprocess.SubprocessHook` - always all environment variables are passed to the subprocess executed with specific set of environment variables. In this case not only the specified environment variables are passed but also all existing environment variables are passed with ``SUBPROCESS_`` prefix added. This happens also for all other subprocesses. diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst index c82a4f3a66d73..655551705e41e 100644 --- a/docs/apache-airflow/operators-and-hooks-ref.rst +++ b/docs/apache-airflow/operators-and-hooks-ref.rst @@ -106,8 +106,8 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Hooks - Guides - * - :mod:`airflow.hooks.filesystem` + * - :mod:`airflow.providers.standard.hooks.filesystem` - - * - :mod:`airflow.hooks.subprocess` + * - :mod:`airflow.providers.standard.hooks.subprocess` - diff --git a/providers/src/airflow/providers/standard/hooks/__init__.py b/providers/src/airflow/providers/standard/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/src/airflow/providers/standard/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/hooks/filesystem.py b/providers/src/airflow/providers/standard/hooks/filesystem.py similarity index 100% rename from airflow/hooks/filesystem.py rename to providers/src/airflow/providers/standard/hooks/filesystem.py diff --git a/airflow/hooks/package_index.py b/providers/src/airflow/providers/standard/hooks/package_index.py similarity index 100% rename from airflow/hooks/package_index.py rename to providers/src/airflow/providers/standard/hooks/package_index.py diff --git a/airflow/hooks/subprocess.py b/providers/src/airflow/providers/standard/hooks/subprocess.py similarity index 96% rename from airflow/hooks/subprocess.py rename to providers/src/airflow/providers/standard/hooks/subprocess.py index bc20b5c20b4c5..9e578a7d8034b 100644 --- a/airflow/hooks/subprocess.py +++ b/providers/src/airflow/providers/standard/hooks/subprocess.py @@ -52,8 +52,8 @@ def run_command( :param env: Optional dict containing environment variables to be made available to the shell environment in which ``command`` will be executed. If omitted, ``os.environ`` will be used. Note, that in case you have Sentry configured, original variables from the environment - will also be passed to the subprocess with ``SUBPROCESS_`` prefix. See - :doc:`/administration-and-deployment/logging-monitoring/errors` for details. + will also be passed to the subprocess with ``SUBPROCESS_`` prefix. See: + https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/errors.html for details. :param output_encoding: encoding to use for decoding stdout :param cwd: Working directory to run the command in. If None (default), the command is run in a temporary directory. diff --git a/providers/src/airflow/providers/standard/operators/bash.py b/providers/src/airflow/providers/standard/operators/bash.py index 2ec0341a0d1e2..bf4a943df6e08 100644 --- a/providers/src/airflow/providers/standard/operators/bash.py +++ b/providers/src/airflow/providers/standard/operators/bash.py @@ -24,8 +24,8 @@ from typing import TYPE_CHECKING, Any, Callable, Container, Sequence, cast from airflow.exceptions import AirflowException, AirflowSkipException -from airflow.hooks.subprocess import SubprocessHook from airflow.models.baseoperator import BaseOperator +from airflow.providers.standard.hooks.subprocess import SubprocessHook from airflow.utils.operator_helpers import context_to_airflow_vars from airflow.utils.types import ArgNotSet diff --git a/providers/src/airflow/providers/standard/provider.yaml b/providers/src/airflow/providers/standard/provider.yaml index 2d4c4f29bef5c..b3111d62b1a66 100644 --- a/providers/src/airflow/providers/standard/provider.yaml +++ b/providers/src/airflow/providers/standard/provider.yaml @@ -52,3 +52,10 @@ sensors: - airflow.providers.standard.sensors.time - airflow.providers.standard.sensors.weekday - airflow.providers.standard.sensors.bash + +hooks: + - integration-name: Standard + python-modules: + - airflow.providers.standard.hooks.filesystem + - airflow.providers.standard.hooks.package_index + - airflow.providers.standard.hooks.subprocess diff --git a/providers/tests/standard/hooks/__init__.py b/providers/tests/standard/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/tests/standard/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/tests/standard/hooks/test_filesystem.py b/providers/tests/standard/hooks/test_filesystem.py new file mode 100644 index 0000000000000..bbcd22dc94219 --- /dev/null +++ b/providers/tests/standard/hooks/test_filesystem.py @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.standard.hooks.filesystem import FSHook + +pytestmark = pytest.mark.db_test + + +class TestFSHook: + def test_get_ui_field_behaviour(self): + fs_hook = FSHook() + assert fs_hook.get_ui_field_behaviour() == { + "hidden_fields": ["host", "schema", "port", "login", "password", "extra"], + "relabeling": {}, + "placeholders": {}, + } + + def test_get_path(self): + fs_hook = FSHook(fs_conn_id="fs_default") + + assert fs_hook.get_path() == "/" diff --git a/tests/hooks/test_package_index.py b/providers/tests/standard/hooks/test_package_index.py similarity index 93% rename from tests/hooks/test_package_index.py rename to providers/tests/standard/hooks/test_package_index.py index 9da429c5a09cf..6a90db0715d81 100644 --- a/tests/hooks/test_package_index.py +++ b/providers/tests/standard/hooks/test_package_index.py @@ -21,8 +21,8 @@ import pytest -from airflow.hooks.package_index import PackageIndexHook from airflow.models.connection import Connection +from airflow.providers.standard.hooks.package_index import PackageIndexHook class MockConnection(Connection): @@ -73,7 +73,7 @@ def mock_get_connection(monkeypatch: pytest.MonkeyPatch, request: pytest.Fixture password: str | None = testdata.get("password", None) expected_result: str | None = testdata.get("expected_result", None) monkeypatch.setattr( - "airflow.hooks.package_index.PackageIndexHook.get_connection", + "airflow.providers.standard.hooks.package_index.PackageIndexHook.get_connection", lambda *_: MockConnection(host, login, password), ) return expected_result @@ -104,7 +104,7 @@ class MockProc: return MockProc() - monkeypatch.setattr("airflow.hooks.package_index.subprocess.run", mock_run) + monkeypatch.setattr("airflow.providers.standard.hooks.package_index.subprocess.run", mock_run) hook_instance = PackageIndexHook() if mock_get_connection: diff --git a/tests/hooks/test_subprocess.py b/providers/tests/standard/hooks/test_subprocess.py similarity index 95% rename from tests/hooks/test_subprocess.py rename to providers/tests/standard/hooks/test_subprocess.py index 0f625be816887..2b2e9473359e5 100644 --- a/tests/hooks/test_subprocess.py +++ b/providers/tests/standard/hooks/test_subprocess.py @@ -26,7 +26,7 @@ import pytest -from airflow.hooks.subprocess import SubprocessHook +from airflow.providers.standard.hooks.subprocess import SubprocessHook OS_ENV_KEY = "SUBPROCESS_ENV_TEST" OS_ENV_VAL = "this-is-from-os-environ" @@ -81,11 +81,11 @@ def test_return_value(self, val, expected): @mock.patch.dict("os.environ", clear=True) @mock.patch( - "airflow.hooks.subprocess.TemporaryDirectory", + "airflow.providers.standard.hooks.subprocess.TemporaryDirectory", return_value=MagicMock(__enter__=MagicMock(return_value="/tmp/airflowtmpcatcat")), ) @mock.patch( - "airflow.hooks.subprocess.Popen", + "airflow.providers.standard.hooks.subprocess.Popen", return_value=MagicMock(stdout=MagicMock(readline=MagicMock(side_effect=StopIteration), returncode=0)), ) def test_should_exec_subprocess(self, mock_popen, mock_temporary_directory): diff --git a/tests/sensors/test_filesystem.py b/tests/sensors/test_filesystem.py index 1fb123cfe7248..641f2f218f2db 100644 --- a/tests/sensors/test_filesystem.py +++ b/tests/sensors/test_filesystem.py @@ -40,7 +40,7 @@ @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode class TestFileSensor: def setup_method(self): - from airflow.hooks.filesystem import FSHook + from airflow.providers.standard.hooks.filesystem import FSHook hook = FSHook() args = {"owner": "airflow", "start_date": DEFAULT_DATE} From 40a863a1c2ff58352af56f87ecf6a91738cc2bf3 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Thu, 10 Oct 2024 15:15:41 +0800 Subject: [PATCH 047/199] Fix dag warning documentation (#42858) --- airflow/api_connexion/openapi/v1.yaml | 2 +- airflow/www/static/js/types/api-generated.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 3e120ce2f5118..305653e781797 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -3463,7 +3463,7 @@ components: allOf: - type: object properties: - import_errors: + dag_warnings: type: array items: $ref: "#/components/schemas/DagWarning" diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 87f89d19cd714..6aeb8721e4280 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1263,7 +1263,7 @@ export interface components { }; /** @description Collection of DAG warnings. */ DagWarningCollection: { - import_errors?: components["schemas"]["DagWarning"][]; + dag_warnings?: components["schemas"]["DagWarning"][]; } & components["schemas"]["CollectionInfo"]; SetDagRunNote: { /** @description Custom notes left by users for this Dag Run. */ From a9ab2c6643ce019bc3a0cb223fa64118be7914f1 Mon Sep 17 00:00:00 2001 From: Elad Kalif <45845474+eladkal@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:22:24 +0700 Subject: [PATCH 048/199] Fix issue generation for provider after folder restructure (#42883) --- .../src/airflow_breeze/commands/release_management_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index d286e7ed4e660..b0a1ca2877332 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -137,6 +137,7 @@ run_with_pool, ) from airflow_breeze.utils.path_utils import ( + AIRFLOW_PROVIDERS_SRC, AIRFLOW_SOURCES_ROOT, CONSTRAINTS_CACHE_DIR, DIST_DIR, @@ -2125,7 +2126,7 @@ class ProviderPRInfo(NamedTuple): pull_request_list = [pull_requests[pr] for pr in provider_prs[provider_id] if pr in pull_requests] provider_yaml_dict = yaml.safe_load( ( - AIRFLOW_SOURCES_ROOT + AIRFLOW_PROVIDERS_SRC / "airflow" / "providers" / provider_id.replace(".", os.sep) From fe86dfa07122167ccad54ea849d6e62020f0cab6 Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:17:49 +0530 Subject: [PATCH 049/199] AIP-84 Migrate the public endpoint Get DAG to FastAPI (#42848) * Migrate the public endpoint Get DAG to FastAPI * Use proper name for test function --- airflow/api_fastapi/openapi/v1-generated.yaml | 62 +++++++++++++++++-- airflow/api_fastapi/serializers/dags.py | 40 +++++++----- airflow/api_fastapi/views/public/dags.py | 20 ++++++ airflow/ui/openapi-gen/queries/common.ts | 16 +++++ airflow/ui/openapi-gen/queries/prefetch.ts | 20 ++++++ airflow/ui/openapi-gen/queries/queries.ts | 26 ++++++++ airflow/ui/openapi-gen/queries/suspense.ts | 26 ++++++++ .../ui/openapi-gen/requests/services.gen.ts | 45 +++++++++++--- airflow/ui/openapi-gen/requests/types.gen.ts | 49 ++++++++++++--- tests/api_fastapi/views/public/test_dags.py | 48 ++++++++++++++ 10 files changed, 313 insertions(+), 39 deletions(-) diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index 463cc1e92f4d6..fb19c1abd1c12 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -291,13 +291,13 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /public/dags/{dag_id}/details: + /public/dags/{dag_id}: get: tags: - DAG - summary: Get Dag Details - description: Get details of DAG. - operationId: get_dag_details + summary: Get Dag + description: Get basic information about a DAG. + operationId: get_dag parameters: - name: dag_id in: path @@ -311,7 +311,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/DAGDetailsResponse' + $ref: '#/components/schemas/DAGResponse' '400': content: application/json: @@ -342,7 +342,6 @@ paths: schema: $ref: '#/components/schemas/HTTPExceptionResponse' description: Unprocessable Entity - /public/dags/{dag_id}: patch: tags: - DAG @@ -409,6 +408,57 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/dags/{dag_id}/details: + get: + tags: + - DAG + summary: Get Dag Details + description: Get details of DAG. + operationId: get_dag_details + parameters: + - name: dag_id + in: path + required: true + schema: + type: string + title: Dag Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/DAGDetailsResponse' + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Bad Request + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unprocessable Entity /public/connections/{connection_id}: delete: tags: diff --git a/airflow/api_fastapi/serializers/dags.py b/airflow/api_fastapi/serializers/dags.py index 17677054c4c1c..9879badf25048 100644 --- a/airflow/api_fastapi/serializers/dags.py +++ b/airflow/api_fastapi/serializers/dags.py @@ -24,9 +24,9 @@ from itsdangerous import URLSafeSerializer from pendulum.tz.timezone import FixedTimezone, Timezone from pydantic import ( - AliasChoices, + AliasGenerator, BaseModel, - Field, + ConfigDict, computed_field, field_validator, ) @@ -77,6 +77,14 @@ def get_owners(cls, v: Any) -> list[str] | None: return v.split(",") return v + @field_validator("timetable_summary", mode="before") + @classmethod + def get_timetable_summary(cls, tts: str | None) -> str | None: + """Validate the string representation of timetable_summary.""" + if tts is None or tts == "None": + return None + return str(tts) + # Mypy issue https://github.com/python/mypy/issues/1362 @computed_field # type: ignore[misc] @property @@ -103,9 +111,7 @@ class DAGDetailsResponse(DAGResponse): """Specific serializer for DAG Details responses.""" catchup: bool - dag_run_timeout: timedelta | None = Field( - validation_alias=AliasChoices("dag_run_timeout", "dagrun_timeout") - ) + dag_run_timeout: timedelta | None dataset_expression: dict | None doc_md: str | None start_date: datetime | None @@ -114,11 +120,19 @@ class DAGDetailsResponse(DAGResponse): orientation: str params: abc.MutableMapping | None render_template_as_native_obj: bool - template_search_path: Iterable[str] | None = Field( - validation_alias=AliasChoices("template_search_path", "template_searchpath") - ) + template_search_path: Iterable[str] | None timezone: str | None - last_parsed: datetime | None = Field(validation_alias=AliasChoices("last_parsed", "last_loaded")) + last_parsed: datetime | None + + model_config = ConfigDict( + alias_generator=AliasGenerator( + validation_alias=lambda field_name: { + "dag_run_timeout": "dagrun_timeout", + "last_parsed": "last_loaded", + "template_search_path": "template_searchpath", + }.get(field_name, field_name), + ) + ) @field_validator("timezone", mode="before") @classmethod @@ -128,14 +142,6 @@ def get_timezone(cls, tz: Timezone | FixedTimezone) -> str | None: return None return str(tz) - @field_validator("timetable_summary", mode="before") - @classmethod - def get_timetable_summary(cls, tts: str | None) -> str | None: - """Validate the string representation of timetable_summary.""" - if tts is None or tts == "None": - return None - return str(tts) - @field_validator("params", mode="before") @classmethod def get_params(cls, params: abc.MutableMapping | None) -> dict | None: diff --git a/airflow/api_fastapi/views/public/dags.py b/airflow/api_fastapi/views/public/dags.py index f0df86b787a20..ca0f44162eb2d 100644 --- a/airflow/api_fastapi/views/public/dags.py +++ b/airflow/api_fastapi/views/public/dags.py @@ -92,6 +92,26 @@ async def get_dags( ) +@dags_router.get("/{dag_id}", responses=create_openapi_http_exception_doc([400, 401, 403, 404, 422])) +async def get_dag( + dag_id: str, session: Annotated[Session, Depends(get_session)], request: Request +) -> DAGResponse: + """Get basic information about a DAG.""" + dag: DAG = request.app.state.dag_bag.get_dag(dag_id) + if not dag: + raise HTTPException(404, f"Dag with id {dag_id} was not found") + + dag_model: DagModel = session.get(DagModel, dag_id) + if not dag_model: + raise HTTPException(404, f"Unable to obtain dag with id {dag_id} from session") + + for key, value in dag.__dict__.items(): + if not key.startswith("_") and not hasattr(dag_model, key): + setattr(dag_model, key, value) + + return DAGResponse.model_validate(dag_model, from_attributes=True) + + @dags_router.get("/{dag_id}/details", responses=create_openapi_http_exception_doc([400, 401, 403, 404, 422])) async def get_dag_details( dag_id: str, session: Annotated[Session, Depends(get_session)], request: Request diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 72fd0ef9ccde1..a4d65c69003f1 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -98,6 +98,22 @@ export const UseDagServiceGetDagsKeyFn = ( }, ]), ]; +export type DagServiceGetDagDefaultResponse = Awaited< + ReturnType +>; +export type DagServiceGetDagQueryResult< + TData = DagServiceGetDagDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useDagServiceGetDagKey = "DagServiceGetDag"; +export const UseDagServiceGetDagKeyFn = ( + { + dagId, + }: { + dagId: string; + }, + queryKey?: Array, +) => [useDagServiceGetDagKey, ...(queryKey ?? [{ dagId }])]; export type DagServiceGetDagDetailsDefaultResponse = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index a114b9dc92c6c..8bd691ca33be4 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -126,6 +126,26 @@ export const prefetchUseDagServiceGetDags = ( tags, }), }); +/** + * Get Dag + * Get basic information about a DAG. + * @param data The data for the request. + * @param data.dagId + * @returns DAGResponse Successful Response + * @throws ApiError + */ +export const prefetchUseDagServiceGetDag = ( + queryClient: QueryClient, + { + dagId, + }: { + dagId: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseDagServiceGetDagKeyFn({ dagId }), + queryFn: () => DagService.getDag({ dagId }), + }); /** * Get Dag Details * Get details of DAG. diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index a3ce02257160a..51b8f4fb051d7 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -153,6 +153,32 @@ export const useDagServiceGetDags = < }) as TData, ...options, }); +/** + * Get Dag + * Get basic information about a DAG. + * @param data The data for the request. + * @param data.dagId + * @returns DAGResponse Successful Response + * @throws ApiError + */ +export const useDagServiceGetDag = < + TData = Common.DagServiceGetDagDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + dagId, + }: { + dagId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseDagServiceGetDagKeyFn({ dagId }, queryKey), + queryFn: () => DagService.getDag({ dagId }) as TData, + ...options, + }); /** * Get Dag Details * Get details of DAG. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index fbef843c6e0ab..b437007468f20 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -148,6 +148,32 @@ export const useDagServiceGetDagsSuspense = < }) as TData, ...options, }); +/** + * Get Dag + * Get basic information about a DAG. + * @param data The data for the request. + * @param data.dagId + * @returns DAGResponse Successful Response + * @throws ApiError + */ +export const useDagServiceGetDagSuspense = < + TData = Common.DagServiceGetDagDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + dagId, + }: { + dagId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseDagServiceGetDagKeyFn({ dagId }, queryKey), + queryFn: () => DagService.getDag({ dagId }) as TData, + ...options, + }); /** * Get Dag Details * Get details of DAG. diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 7f61fd32f3493..24fbb9c29c46a 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -11,10 +11,12 @@ import type { GetDagsResponse, PatchDagsData, PatchDagsResponse, - GetDagDetailsData, - GetDagDetailsResponse, + GetDagData, + GetDagResponse, PatchDagData, PatchDagResponse, + GetDagDetailsData, + GetDagDetailsResponse, DeleteConnectionData, DeleteConnectionResponse, GetConnectionData, @@ -166,19 +168,17 @@ export class DagService { } /** - * Get Dag Details - * Get details of DAG. + * Get Dag + * Get basic information about a DAG. * @param data The data for the request. * @param data.dagId - * @returns DAGDetailsResponse Successful Response + * @returns DAGResponse Successful Response * @throws ApiError */ - public static getDagDetails( - data: GetDagDetailsData, - ): CancelablePromise { + public static getDag(data: GetDagData): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/public/dags/{dag_id}/details", + url: "/public/dags/{dag_id}", path: { dag_id: data.dagId, }, @@ -225,6 +225,33 @@ export class DagService { }, }); } + + /** + * Get Dag Details + * Get details of DAG. + * @param data The data for the request. + * @param data.dagId + * @returns DAGDetailsResponse Successful Response + * @throws ApiError + */ + public static getDagDetails( + data: GetDagDetailsData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/dags/{dag_id}/details", + path: { + dag_id: data.dagId, + }, + errors: { + 400: "Bad Request", + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Unprocessable Entity", + }, + }); + } } export class ConnectionService { diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 7b5fc54065a01..368c981b9da1e 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -260,11 +260,11 @@ export type PatchDagsData = { export type PatchDagsResponse = DAGCollectionResponse; -export type GetDagDetailsData = { +export type GetDagData = { dagId: string; }; -export type GetDagDetailsResponse = DAGDetailsResponse; +export type GetDagResponse = DAGResponse; export type PatchDagData = { dagId: string; @@ -274,6 +274,12 @@ export type PatchDagData = { export type PatchDagResponse = DAGResponse; +export type GetDagDetailsData = { + dagId: string; +}; + +export type GetDagDetailsResponse = DAGDetailsResponse; + export type DeleteConnectionData = { connectionId: string; }; @@ -379,14 +385,14 @@ export type $OpenApiTs = { }; }; }; - "/public/dags/{dag_id}/details": { + "/public/dags/{dag_id}": { get: { - req: GetDagDetailsData; + req: GetDagData; res: { /** * Successful Response */ - 200: DAGDetailsResponse; + 200: DAGResponse; /** * Bad Request */ @@ -409,8 +415,6 @@ export type $OpenApiTs = { 422: HTTPExceptionResponse; }; }; - }; - "/public/dags/{dag_id}": { patch: { req: PatchDagData; res: { @@ -441,6 +445,37 @@ export type $OpenApiTs = { }; }; }; + "/public/dags/{dag_id}/details": { + get: { + req: GetDagDetailsData; + res: { + /** + * Successful Response + */ + 200: DAGDetailsResponse; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Unprocessable Entity + */ + 422: HTTPExceptionResponse; + }; + }; + }; "/public/connections/{connection_id}": { delete: { req: DeleteConnectionData; diff --git a/tests/api_fastapi/views/public/test_dags.py b/tests/api_fastapi/views/public/test_dags.py index 7ac93a2f2e07d..5512f7bb13849 100644 --- a/tests/api_fastapi/views/public/test_dags.py +++ b/tests/api_fastapi/views/public/test_dags.py @@ -303,3 +303,51 @@ def test_dag_details(test_client, query_params, dag_id, expected_status_code, da "timezone": UTC_JSON_REPR, } assert res_json == expected + + +@pytest.mark.parametrize( + "query_params, dag_id, expected_status_code, dag_display_name", + [ + ({}, "fake_dag_id", 404, "fake_dag"), + ({}, DAG2_ID, 200, DAG2_DISPLAY_NAME), + ], +) +def test_get_dag(test_client, query_params, dag_id, expected_status_code, dag_display_name): + response = test_client.get(f"/public/dags/{dag_id}", params=query_params) + assert response.status_code == expected_status_code + if expected_status_code != 200: + return + + # Match expected and actual responses below. + res_json = response.json() + last_parsed_time = res_json["last_parsed_time"] + file_token = res_json["file_token"] + expected = { + "dag_id": dag_id, + "dag_display_name": dag_display_name, + "description": None, + "fileloc": "/opt/airflow/tests/api_fastapi/views/public/test_dags.py", + "file_token": file_token, + "is_paused": False, + "is_active": True, + "owners": ["airflow"], + "timetable_summary": None, + "tags": [], + "next_dagrun": None, + "has_task_concurrency_limits": True, + "next_dagrun_data_interval_start": None, + "next_dagrun_data_interval_end": None, + "max_active_runs": 16, + "max_consecutive_failed_dag_runs": 0, + "next_dagrun_create_after": None, + "last_expired": None, + "max_active_tasks": 16, + "last_pickled": None, + "default_view": "grid", + "last_parsed_time": last_parsed_time, + "scheduler_lock": None, + "timetable_description": "Never, external triggers only", + "has_import_errors": False, + "pickle_id": None, + } + assert res_json == expected From 78b5a0109ba056b3035dfc3a5c8d6974e76d237b Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Thu, 10 Oct 2024 13:33:45 +0530 Subject: [PATCH 050/199] AIP-84 Migrate GET Dag Run endpoint to FastAPI (#42725) * get dag_run init * add serializer * Merge branch 'main' of https://github.com/apache/airflow into kalyan/AIP-84/get_dag_run * add types * add test * working tests * add note to DagRunResponse * add note * add test to test non Null note * Update airflow/api_fastapi/views/public/dag_run.py Co-authored-by: Pierre Jeambrun * Update airflow/api_fastapi/views/public/dag_run.py Co-authored-by: Pierre Jeambrun * Merge branch 'main' of https://github.com/apache/airflow into kalyan/AIP-84/get_dag_run * add 404 test --------- Co-authored-by: Pierre Jeambrun --- .../endpoints/dag_run_endpoint.py | 2 + airflow/api_fastapi/openapi/v1-generated.yaml | 153 +++++++++++++++++ airflow/api_fastapi/serializers/dag_run.py | 44 +++++ airflow/api_fastapi/views/public/__init__.py | 2 + airflow/api_fastapi/views/public/dag_run.py | 44 +++++ airflow/ui/openapi-gen/queries/common.ts | 19 ++ airflow/ui/openapi-gen/queries/prefetch.ts | 23 +++ airflow/ui/openapi-gen/queries/queries.ts | 32 ++++ airflow/ui/openapi-gen/queries/suspense.ts | 32 ++++ .../ui/openapi-gen/requests/schemas.gen.ts | 162 ++++++++++++++++++ .../ui/openapi-gen/requests/services.gen.ts | 31 ++++ airflow/ui/openapi-gen/requests/types.gen.ts | 78 +++++++++ .../api_fastapi/views/public/test_dag_run.py | 137 +++++++++++++++ 13 files changed, 759 insertions(+) create mode 100644 airflow/api_fastapi/serializers/dag_run.py create mode 100644 airflow/api_fastapi/views/public/dag_run.py create mode 100644 tests/api_fastapi/views/public/test_dag_run.py diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py b/airflow/api_connexion/endpoints/dag_run_endpoint.py index 44891c0ef2c84..a862b7c969503 100644 --- a/airflow/api_connexion/endpoints/dag_run_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py @@ -63,6 +63,7 @@ from airflow.models import DagModel, DagRun from airflow.timetables.base import DataInterval from airflow.utils.airflow_flask_app import get_airflow_app +from airflow.utils.api_migration import mark_fastapi_migration_done from airflow.utils.db import get_query_count from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.state import DagRunState @@ -90,6 +91,7 @@ def delete_dag_run(*, dag_id: str, dag_run_id: str, session: Session = NEW_SESSI return NoContent, HTTPStatus.NO_CONTENT +@mark_fastapi_migration_done @security.requires_access_dag("GET", DagAccessEntity.RUN) @provide_session def get_dag_run( diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index fb19c1abd1c12..7debfbb1008af 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -629,6 +629,56 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/dags/{dag_id}/dagRuns/{dag_run_id}: + get: + tags: + - DagRun + summary: Get Dag Run + operationId: get_dag_run + parameters: + - name: dag_id + in: path + required: true + schema: + type: string + title: Dag Id + - name: dag_run_id + in: path + required: true + schema: + type: string + title: Dag Run Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/DAGRunResponse' + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' components: schemas: ConnectionResponse: @@ -1097,6 +1147,87 @@ components: - file_token title: DAGResponse description: DAG serializer for responses. + DAGRunResponse: + properties: + run_id: + anyOf: + - type: string + - type: 'null' + title: Run Id + dag_id: + type: string + title: Dag Id + logical_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Logical Date + start_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Start Date + end_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: End Date + data_interval_start: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Data Interval Start + data_interval_end: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Data Interval End + last_scheduling_decision: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Last Scheduling Decision + run_type: + $ref: '#/components/schemas/DagRunType' + state: + $ref: '#/components/schemas/DagRunState' + external_trigger: + type: boolean + title: External Trigger + triggered_by: + $ref: '#/components/schemas/DagRunTriggeredByType' + conf: + type: object + title: Conf + note: + anyOf: + - type: string + - type: 'null' + title: Note + type: object + required: + - run_id + - dag_id + - logical_date + - start_date + - end_date + - data_interval_start + - data_interval_end + - last_scheduling_decision + - run_type + - state + - external_trigger + - triggered_by + - conf + - note + title: DAGRunResponse + description: DAG Run serializer for responses. DAGRunStates: properties: queued: @@ -1157,6 +1288,28 @@ components: so please ensure that their values always match the ones with the same name in TaskInstanceState.' + DagRunTriggeredByType: + type: string + enum: + - cli + - operator + - rest_api + - ui + - test + - timetable + - dataset + - backfill + title: DagRunTriggeredByType + description: Class with TriggeredBy types for DagRun. + DagRunType: + type: string + enum: + - backfill + - scheduled + - manual + - dataset_triggered + title: DagRunType + description: Class with DagRun types. DagTagPydantic: properties: name: diff --git a/airflow/api_fastapi/serializers/dag_run.py b/airflow/api_fastapi/serializers/dag_run.py new file mode 100644 index 0000000000000..4622fac645c07 --- /dev/null +++ b/airflow/api_fastapi/serializers/dag_run.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, Field + +from airflow.utils.state import DagRunState +from airflow.utils.types import DagRunTriggeredByType, DagRunType + + +class DAGRunResponse(BaseModel): + """DAG Run serializer for responses.""" + + dag_run_id: str | None = Field(alias="run_id") + dag_id: str + logical_date: datetime | None + start_date: datetime | None + end_date: datetime | None + data_interval_start: datetime | None + data_interval_end: datetime | None + last_scheduling_decision: datetime | None + run_type: DagRunType + state: DagRunState + external_trigger: bool + triggered_by: DagRunTriggeredByType + conf: dict + note: str | None diff --git a/airflow/api_fastapi/views/public/__init__.py b/airflow/api_fastapi/views/public/__init__.py index 4e02d9ab43bcf..9d90a0966802c 100644 --- a/airflow/api_fastapi/views/public/__init__.py +++ b/airflow/api_fastapi/views/public/__init__.py @@ -18,6 +18,7 @@ from __future__ import annotations from airflow.api_fastapi.views.public.connections import connections_router +from airflow.api_fastapi.views.public.dag_run import dag_run_router from airflow.api_fastapi.views.public.dags import dags_router from airflow.api_fastapi.views.public.variables import variables_router from airflow.api_fastapi.views.router import AirflowRouter @@ -28,3 +29,4 @@ public_router.include_router(dags_router) public_router.include_router(connections_router) public_router.include_router(variables_router) +public_router.include_router(dag_run_router) diff --git a/airflow/api_fastapi/views/public/dag_run.py b/airflow/api_fastapi/views/public/dag_run.py new file mode 100644 index 0000000000000..d39fb6f2f331c --- /dev/null +++ b/airflow/api_fastapi/views/public/dag_run.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from fastapi import Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session +from typing_extensions import Annotated + +from airflow.api_fastapi.db.common import get_session +from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.serializers.dag_run import DAGRunResponse +from airflow.api_fastapi.views.router import AirflowRouter +from airflow.models import DagRun + +dag_run_router = AirflowRouter(tags=["DagRun"], prefix="/dags/{dag_id}/dagRuns") + + +@dag_run_router.get("/{dag_run_id}", responses=create_openapi_http_exception_doc([401, 403, 404])) +async def get_dag_run( + dag_id: str, dag_run_id: str, session: Annotated[Session, Depends(get_session)] +) -> DAGRunResponse: + dag_run = session.scalar(select(DagRun).filter_by(dag_id=dag_id, run_id=dag_run_id)) + if dag_run is None: + raise HTTPException( + 404, f"The DagRun with dag_id: `{dag_id}` and run_id: `{dag_run_id}` was not found" + ) + + return DAGRunResponse.model_validate(dag_run, from_attributes=True) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index a4d65c69003f1..aaff196c0791d 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -4,6 +4,7 @@ import { UseQueryResult } from "@tanstack/react-query"; import { AssetService, ConnectionService, + DagRunService, DagService, DashboardService, VariableService, @@ -166,6 +167,24 @@ export const UseVariableServiceGetVariableKeyFn = ( }, queryKey?: Array, ) => [useVariableServiceGetVariableKey, ...(queryKey ?? [{ variableKey }])]; +export type DagRunServiceGetDagRunDefaultResponse = Awaited< + ReturnType +>; +export type DagRunServiceGetDagRunQueryResult< + TData = DagRunServiceGetDagRunDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useDagRunServiceGetDagRunKey = "DagRunServiceGetDagRun"; +export const UseDagRunServiceGetDagRunKeyFn = ( + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, + queryKey?: Array, +) => [useDagRunServiceGetDagRunKey, ...(queryKey ?? [{ dagId, dagRunId }])]; export type DagServicePatchDagsMutationResult = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index 8bd691ca33be4..3e194302f4be0 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -4,6 +4,7 @@ import { type QueryClient } from "@tanstack/react-query"; import { AssetService, ConnectionService, + DagRunService, DagService, DashboardService, VariableService, @@ -206,3 +207,25 @@ export const prefetchUseVariableServiceGetVariable = ( queryKey: Common.UseVariableServiceGetVariableKeyFn({ variableKey }), queryFn: () => VariableService.getVariable({ variableKey }), }); +/** + * Get Dag Run + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns DAGRunResponse Successful Response + * @throws ApiError + */ +export const prefetchUseDagRunServiceGetDagRun = ( + queryClient: QueryClient, + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseDagRunServiceGetDagRunKeyFn({ dagId, dagRunId }), + queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }), + }); diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 51b8f4fb051d7..19bb17b342a84 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -9,6 +9,7 @@ import { import { AssetService, ConnectionService, + DagRunService, DagService, DashboardService, VariableService, @@ -263,6 +264,37 @@ export const useVariableServiceGetVariable = < queryFn: () => VariableService.getVariable({ variableKey }) as TData, ...options, }); +/** + * Get Dag Run + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns DAGRunResponse Successful Response + * @throws ApiError + */ +export const useDagRunServiceGetDagRun = < + TData = Common.DagRunServiceGetDagRunDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseDagRunServiceGetDagRunKeyFn( + { dagId, dagRunId }, + queryKey, + ), + queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, + ...options, + }); /** * Patch Dags * Patch multiple DAGs. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index b437007468f20..79ad479f0a42f 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -4,6 +4,7 @@ import { UseQueryOptions, useSuspenseQuery } from "@tanstack/react-query"; import { AssetService, ConnectionService, + DagRunService, DagService, DashboardService, VariableService, @@ -258,3 +259,34 @@ export const useVariableServiceGetVariableSuspense = < queryFn: () => VariableService.getVariable({ variableKey }) as TData, ...options, }); +/** + * Get Dag Run + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns DAGRunResponse Successful Response + * @throws ApiError + */ +export const useDagRunServiceGetDagRunSuspense = < + TData = Common.DagRunServiceGetDagRunDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseDagRunServiceGetDagRunKeyFn( + { dagId, dagRunId }, + queryKey, + ), + queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, + ...options, + }); diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 8f76ebd13c40f..18df5284651b7 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -784,6 +784,145 @@ export const $DAGResponse = { description: "DAG serializer for responses.", } as const; +export const $DAGRunResponse = { + properties: { + run_id: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Run Id", + }, + dag_id: { + type: "string", + title: "Dag Id", + }, + logical_date: { + anyOf: [ + { + type: "string", + format: "date-time", + }, + { + type: "null", + }, + ], + title: "Logical Date", + }, + start_date: { + anyOf: [ + { + type: "string", + format: "date-time", + }, + { + type: "null", + }, + ], + title: "Start Date", + }, + end_date: { + anyOf: [ + { + type: "string", + format: "date-time", + }, + { + type: "null", + }, + ], + title: "End Date", + }, + data_interval_start: { + anyOf: [ + { + type: "string", + format: "date-time", + }, + { + type: "null", + }, + ], + title: "Data Interval Start", + }, + data_interval_end: { + anyOf: [ + { + type: "string", + format: "date-time", + }, + { + type: "null", + }, + ], + title: "Data Interval End", + }, + last_scheduling_decision: { + anyOf: [ + { + type: "string", + format: "date-time", + }, + { + type: "null", + }, + ], + title: "Last Scheduling Decision", + }, + run_type: { + $ref: "#/components/schemas/DagRunType", + }, + state: { + $ref: "#/components/schemas/DagRunState", + }, + external_trigger: { + type: "boolean", + title: "External Trigger", + }, + triggered_by: { + $ref: "#/components/schemas/DagRunTriggeredByType", + }, + conf: { + type: "object", + title: "Conf", + }, + note: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Note", + }, + }, + type: "object", + required: [ + "run_id", + "dag_id", + "logical_date", + "start_date", + "end_date", + "data_interval_start", + "data_interval_end", + "last_scheduling_decision", + "run_type", + "state", + "external_trigger", + "triggered_by", + "conf", + "note", + ], + title: "DAGRunResponse", + description: "DAG Run serializer for responses.", +} as const; + export const $DAGRunStates = { properties: { queued: { @@ -845,6 +984,29 @@ so please ensure that their values always match the ones with the same name in TaskInstanceState.`, } as const; +export const $DagRunTriggeredByType = { + type: "string", + enum: [ + "cli", + "operator", + "rest_api", + "ui", + "test", + "timetable", + "dataset", + "backfill", + ], + title: "DagRunTriggeredByType", + description: "Class with TriggeredBy types for DagRun.", +} as const; + +export const $DagRunType = { + type: "string", + enum: ["backfill", "scheduled", "manual", "dataset_triggered"], + title: "DagRunType", + description: "Class with DagRun types.", +} as const; + export const $DagTagPydantic = { properties: { name: { diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 24fbb9c29c46a..9a126aef25fbc 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -25,6 +25,8 @@ import type { DeleteVariableResponse, GetVariableData, GetVariableResponse, + GetDagRunData, + GetDagRunResponse, } from "./types.gen"; export class AssetService { @@ -361,3 +363,32 @@ export class VariableService { }); } } + +export class DagRunService { + /** + * Get Dag Run + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns DAGRunResponse Successful Response + * @throws ApiError + */ + public static getDagRun( + data: GetDagRunData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/dags/{dag_id}/dagRuns/{dag_run_id}", + path: { + dag_id: data.dagId, + dag_run_id: data.dagRunId, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Validation Error", + }, + }); + } +} diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 368c981b9da1e..45bfa51aec9c4 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -120,6 +120,28 @@ export type DAGResponse = { readonly file_token: string; }; +/** + * DAG Run serializer for responses. + */ +export type DAGRunResponse = { + run_id: string | null; + dag_id: string; + logical_date: string | null; + start_date: string | null; + end_date: string | null; + data_interval_start: string | null; + data_interval_end: string | null; + last_scheduling_decision: string | null; + run_type: DagRunType; + state: DagRunState; + external_trigger: boolean; + triggered_by: DagRunTriggeredByType; + conf: { + [key: string]: unknown; + }; + note: string | null; +}; + /** * DAG Run States for responses. */ @@ -149,6 +171,28 @@ export type DAGRunTypes = { */ export type DagRunState = "queued" | "running" | "success" | "failed"; +/** + * Class with TriggeredBy types for DagRun. + */ +export type DagRunTriggeredByType = + | "cli" + | "operator" + | "rest_api" + | "ui" + | "test" + | "timetable" + | "dataset" + | "backfill"; + +/** + * Class with DagRun types. + */ +export type DagRunType = + | "backfill" + | "scheduled" + | "manual" + | "dataset_triggered"; + /** * Serializable representation of the DagTag ORM SqlAlchemyModel used by internal API. */ @@ -304,6 +348,13 @@ export type GetVariableData = { export type GetVariableResponse = VariableResponse; +export type GetDagRunData = { + dagId: string; + dagRunId: string; +}; + +export type GetDagRunResponse = DAGRunResponse; + export type $OpenApiTs = { "/ui/next_run_assets/{dag_id}": { get: { @@ -580,4 +631,31 @@ export type $OpenApiTs = { }; }; }; + "/public/dags/{dag_id}/dagRuns/{dag_run_id}": { + get: { + req: GetDagRunData; + res: { + /** + * Successful Response + */ + 200: DAGRunResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; }; diff --git a/tests/api_fastapi/views/public/test_dag_run.py b/tests/api_fastapi/views/public/test_dag_run.py new file mode 100644 index 0000000000000..dab81907068e7 --- /dev/null +++ b/tests/api_fastapi/views/public/test_dag_run.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from airflow.operators.empty import EmptyOperator +from airflow.utils.session import provide_session +from airflow.utils.state import DagRunState +from airflow.utils.types import DagRunTriggeredByType, DagRunType +from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags + +pytestmark = pytest.mark.db_test + +DAG1_ID = "test_dag1" +DAG2_ID = "test_dag2" +DAG1_RUN1_ID = "dag_run_1" +DAG1_RUN2_ID = "dag_run_2" +DAG2_RUN1_ID = "dag_run_3" +DAG2_RUN2_ID = "dag_run_4" +DAG1_RUN1_STATE = DagRunState.SUCCESS +DAG1_RUN2_STATE = DagRunState.FAILED +DAG2_RUN1_STATE = DagRunState.SUCCESS +DAG2_RUN2_STATE = DagRunState.SUCCESS +DAG1_RUN1_RUN_TYPE = DagRunType.MANUAL +DAG1_RUN2_RUN_TYPE = DagRunType.SCHEDULED +DAG2_RUN1_RUN_TYPE = DagRunType.BACKFILL_JOB +DAG2_RUN2_RUN_TYPE = DagRunType.DATASET_TRIGGERED +DAG1_RUN1_TRIGGERED_BY = DagRunTriggeredByType.UI +DAG1_RUN2_TRIGGERED_BY = DagRunTriggeredByType.DATASET +DAG2_RUN1_TRIGGERED_BY = DagRunTriggeredByType.CLI +DAG2_RUN2_TRIGGERED_BY = DagRunTriggeredByType.REST_API +START_DATE = datetime(2024, 6, 15, 0, 0, tzinfo=timezone.utc) +EXECUTION_DATE = datetime(2024, 6, 16, 0, 0, tzinfo=timezone.utc) +DAG1_NOTE = "test_note" + + +@pytest.fixture(autouse=True) +@provide_session +def setup(dag_maker, session=None): + clear_db_runs() + clear_db_dags() + clear_db_serialized_dags() + + with dag_maker( + DAG1_ID, + schedule="@daily", + start_date=START_DATE, + ): + EmptyOperator(task_id="task_1") + dag1 = dag_maker.create_dagrun( + run_id=DAG1_RUN1_ID, + state=DAG1_RUN1_STATE, + run_type=DAG1_RUN1_RUN_TYPE, + triggered_by=DAG1_RUN1_TRIGGERED_BY, + ) + dag1.note = (DAG1_NOTE, 1) + + dag_maker.create_dagrun( + run_id=DAG1_RUN2_ID, + state=DAG1_RUN2_STATE, + run_type=DAG1_RUN2_RUN_TYPE, + triggered_by=DAG1_RUN2_TRIGGERED_BY, + execution_date=EXECUTION_DATE, + ) + + with dag_maker( + DAG2_ID, + schedule=None, + start_date=START_DATE, + ): + EmptyOperator(task_id="task_2") + dag_maker.create_dagrun( + run_id=DAG2_RUN1_ID, + state=DAG2_RUN1_STATE, + run_type=DAG2_RUN1_RUN_TYPE, + triggered_by=DAG2_RUN1_TRIGGERED_BY, + execution_date=EXECUTION_DATE, + ) + dag_maker.create_dagrun( + run_id=DAG2_RUN2_ID, + state=DAG2_RUN2_STATE, + run_type=DAG2_RUN2_RUN_TYPE, + triggered_by=DAG2_RUN2_TRIGGERED_BY, + execution_date=EXECUTION_DATE, + ) + + dag_maker.dagbag.sync_to_db() + dag_maker.dag_model + dag_maker.dag_model.has_task_concurrency_limits = True + session.merge(dag_maker.dag_model) + session.commit() + + +@pytest.mark.parametrize( + "dag_id, run_id, state, run_type, triggered_by, dag_run_note", + [ + (DAG1_ID, DAG1_RUN1_ID, DAG1_RUN1_STATE, DAG1_RUN1_RUN_TYPE, DAG1_RUN1_TRIGGERED_BY, DAG1_NOTE), + (DAG1_ID, DAG1_RUN2_ID, DAG1_RUN2_STATE, DAG1_RUN2_RUN_TYPE, DAG1_RUN2_TRIGGERED_BY, None), + (DAG2_ID, DAG2_RUN1_ID, DAG2_RUN1_STATE, DAG2_RUN1_RUN_TYPE, DAG2_RUN1_TRIGGERED_BY, None), + (DAG2_ID, DAG2_RUN2_ID, DAG2_RUN2_STATE, DAG2_RUN2_RUN_TYPE, DAG2_RUN2_TRIGGERED_BY, None), + ], +) +def test_get_dag_run(test_client, dag_id, run_id, state, run_type, triggered_by, dag_run_note): + response = test_client.get(f"/public/dags/{dag_id}/dagRuns/{run_id}") + assert response.status_code == 200 + body = response.json() + assert body["dag_id"] == dag_id + assert body["run_id"] == run_id + assert body["state"] == state + assert body["run_type"] == run_type + assert body["triggered_by"] == triggered_by.value + assert body["note"] == dag_run_note + + +def test_get_dag_run_not_found(test_client): + response = test_client.get(f"/public/dags/{DAG1_ID}/dagRuns/invalid") + assert response.status_code == 404 + body = response.json() + assert body["detail"] == "The DagRun with dag_id: `test_dag1` and run_id: `invalid` was not found" From 92f135ce82ee36ed68350b75000a605a679c63b6 Mon Sep 17 00:00:00 2001 From: majorosdonat Date: Thu, 10 Oct 2024 11:34:15 +0200 Subject: [PATCH 051/199] add majorosdonat as bosch user (#42890) Co-authored-by: Majoros Donat (XC-DX/EET2-Bp) --- INTHEWILD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INTHEWILD.md b/INTHEWILD.md index c65597acfb60f..310d018b98329 100644 --- a/INTHEWILD.md +++ b/INTHEWILD.md @@ -101,7 +101,7 @@ Currently, **officially** using Airflow: 1. [Bombora Inc](https://bombora.com/) [[@jeffkpayne](https://github.com/jeffkpayne), [@pakelley](https://github.com/pakelley), [@dNavalta](https://github.com/dNavalta), [@austynh](https://github.com/austynh), [@TheOriginalAlex](https://github.com/TheOriginalAlex)] 1. [Bonial International GmbH](https://www.bonial.com/) 1. [Bonnier Broadcasting](http://www.bonnierbroadcasting.com) [[@wileeam](https://github.com/wileeam)] -1. [Bosch (Robert Bosch GmbH)](https://www.bosch.com/stories/topics/automated-driving/) [[@jscheffl](https://github.com/jscheffl), [@clellmann](https://github.com/clellmann), [@wolfdn](https://github.com/wolfdn), [@AutomationDev85](https://github.com/AutomationDev85)] +1. [Bosch (Robert Bosch GmbH)](https://www.bosch.com/stories/topics/automated-driving/) [[@jscheffl](https://github.com/jscheffl), [@clellmann](https://github.com/clellmann), [@wolfdn](https://github.com/wolfdn), [@AutomationDev85](https://github.com/AutomationDev85), [@majorosdonat](https://github.com/majorosdonat)] 1. [BounceX](http://www.bouncex.com) [[@JoshFerge](https://github.com/JoshFerge), [@hudsonrio](https://github.com/hudsonrio), [@ronniekritou](https://github.com/ronniekritou)] 1. [Braintree](https://www.braintreepayments.com) [[@coopergillan](https://github.com/coopergillan), [@curiousjazz77](https://github.com/curiousjazz77), [@raymondberg](https://github.com/raymondberg)] 1. [Branch](https://branch.io) [[@sdebarshi](https://github.com/sdebarshi), [@dmitrig01](https://github.com/dmitrig01)] From be261dfd63257947ffa50ee36bba05958ea6ea05 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Thu, 10 Oct 2024 09:06:54 -0400 Subject: [PATCH 052/199] Move the session auth backend to FAB auth manager (#42878) --- .../auth_manager/api/auth/backend/session.py | 47 ++++++++++++ .../api/auth/backend/test_session.py | 73 +++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 providers/src/airflow/providers/fab/auth_manager/api/auth/backend/session.py create mode 100644 providers/tests/fab/auth_manager/api/auth/backend/test_session.py diff --git a/providers/src/airflow/providers/fab/auth_manager/api/auth/backend/session.py b/providers/src/airflow/providers/fab/auth_manager/api/auth/backend/session.py new file mode 100644 index 0000000000000..d51f7bf1cf4c9 --- /dev/null +++ b/providers/src/airflow/providers/fab/auth_manager/api/auth/backend/session.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Session authentication backend.""" + +from __future__ import annotations + +from functools import wraps +from typing import Any, Callable, TypeVar, cast + +from flask import Response + +from airflow.www.extensions.init_auth_manager import get_auth_manager + +CLIENT_AUTH: tuple[str, str] | Any | None = None + + +def init_app(_): + """Initialize authentication backend.""" + + +T = TypeVar("T", bound=Callable) + + +def requires_authentication(function: T): + """Decorate functions that require authentication.""" + + @wraps(function) + def decorated(*args, **kwargs): + if not get_auth_manager().is_logged_in(): + return Response("Unauthorized", 401, {}) + return function(*args, **kwargs) + + return cast(T, decorated) diff --git a/providers/tests/fab/auth_manager/api/auth/backend/test_session.py b/providers/tests/fab/auth_manager/api/auth/backend/test_session.py new file mode 100644 index 0000000000000..513e8ec2aa293 --- /dev/null +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_session.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest.mock import Mock, patch + +import pytest +from flask import Response + +from airflow.providers.fab.auth_manager.api.auth.backend.session import requires_authentication +from airflow.www import app as application + +from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + +pytestmark = [ + pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), +] + + +@pytest.fixture +def app(): + return application.create_app(testing=True) + + +mock_call = Mock() + + +@requires_authentication +def function_decorated(): + mock_call() + + +@pytest.mark.db_test +class TestSessionAuth: + def setup_method(self) -> None: + mock_call.reset_mock() + + @patch("airflow.providers.fab.auth_manager.api.auth.backend.session.get_auth_manager") + def test_requires_authentication_when_not_authenticated(self, mock_get_auth_manager, app): + auth_manager = Mock() + auth_manager.is_logged_in.return_value = False + mock_get_auth_manager.return_value = auth_manager + with app.test_request_context() as mock_context: + mock_context.request.authorization = None + result = function_decorated() + + assert type(result) is Response + assert result.status_code == 401 + + @patch("airflow.providers.fab.auth_manager.api.auth.backend.session.get_auth_manager") + def test_requires_authentication_when_authenticated(self, mock_get_auth_manager, app): + auth_manager = Mock() + auth_manager.is_logged_in.return_value = True + mock_get_auth_manager.return_value = auth_manager + with app.test_request_context() as mock_context: + mock_context.request.authorization = None + function_decorated() + + mock_call.assert_called_once() From 9373ec498dabf2e5324a9f0a4a10cd8a7752d4d4 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Thu, 10 Oct 2024 09:08:42 -0400 Subject: [PATCH 053/199] Move user and roles schemas to fab provider (#42869) --- .../role_and_permission_endpoint.py | 6 +++--- .../auth_manager/api_endpoints/user_endpoint.py | 6 +++--- .../fab/auth_manager/schemas/__init__.py | 16 ++++++++++++++++ .../schemas/role_and_permission_schema.py | 0 .../fab/auth_manager}/schemas/user_schema.py | 2 +- .../tests/fab/auth_manager/schemas/__init__.py | 17 +++++++++++++++++ .../test_role_and_permission_schema.py | 2 +- .../test_user_schema.py | 5 ++++- 8 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 providers/src/airflow/providers/fab/auth_manager/schemas/__init__.py rename {airflow/api_connexion => providers/src/airflow/providers/fab/auth_manager}/schemas/role_and_permission_schema.py (100%) rename {airflow/api_connexion => providers/src/airflow/providers/fab/auth_manager}/schemas/user_schema.py (96%) create mode 100644 providers/tests/fab/auth_manager/schemas/__init__.py rename providers/tests/fab/auth_manager/{api_endpoints => schemas}/test_role_and_permission_schema.py (97%) rename providers/tests/fab/auth_manager/{api_endpoints => schemas}/test_user_schema.py (97%) diff --git a/providers/src/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py b/providers/src/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py index ed42f91163982..121a88be28587 100644 --- a/providers/src/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py +++ b/providers/src/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py @@ -26,15 +26,15 @@ from airflow.api_connexion.exceptions import AlreadyExists, BadRequest, NotFound from airflow.api_connexion.parameters import check_limit, format_parameters -from airflow.api_connexion.schemas.role_and_permission_schema import ( +from airflow.api_connexion.security import requires_access_custom_view +from airflow.providers.fab.auth_manager.models import Action, Role +from airflow.providers.fab.auth_manager.schemas.role_and_permission_schema import ( ActionCollection, RoleCollection, action_collection_schema, role_collection_schema, role_schema, ) -from airflow.api_connexion.security import requires_access_custom_view -from airflow.providers.fab.auth_manager.models import Action, Role from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride from airflow.security import permissions from airflow.www.extensions.init_auth_manager import get_auth_manager diff --git a/providers/src/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py b/providers/src/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py index 665b7f52d896f..43464a23d365e 100644 --- a/providers/src/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py +++ b/providers/src/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py @@ -27,14 +27,14 @@ from airflow.api_connexion.exceptions import AlreadyExists, BadRequest, NotFound, Unknown from airflow.api_connexion.parameters import check_limit, format_parameters -from airflow.api_connexion.schemas.user_schema import ( +from airflow.api_connexion.security import requires_access_custom_view +from airflow.providers.fab.auth_manager.models import User +from airflow.providers.fab.auth_manager.schemas.user_schema import ( UserCollection, user_collection_item_schema, user_collection_schema, user_schema, ) -from airflow.api_connexion.security import requires_access_custom_view -from airflow.providers.fab.auth_manager.models import User from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride from airflow.security import permissions from airflow.www.extensions.init_auth_manager import get_auth_manager diff --git a/providers/src/airflow/providers/fab/auth_manager/schemas/__init__.py b/providers/src/airflow/providers/fab/auth_manager/schemas/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/src/airflow/providers/fab/auth_manager/schemas/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/api_connexion/schemas/role_and_permission_schema.py b/providers/src/airflow/providers/fab/auth_manager/schemas/role_and_permission_schema.py similarity index 100% rename from airflow/api_connexion/schemas/role_and_permission_schema.py rename to providers/src/airflow/providers/fab/auth_manager/schemas/role_and_permission_schema.py diff --git a/airflow/api_connexion/schemas/user_schema.py b/providers/src/airflow/providers/fab/auth_manager/schemas/user_schema.py similarity index 96% rename from airflow/api_connexion/schemas/user_schema.py rename to providers/src/airflow/providers/fab/auth_manager/schemas/user_schema.py index 167dd160d1baf..4155667d56766 100644 --- a/airflow/api_connexion/schemas/user_schema.py +++ b/providers/src/airflow/providers/fab/auth_manager/schemas/user_schema.py @@ -22,8 +22,8 @@ from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field from airflow.api_connexion.parameters import validate_istimezone -from airflow.api_connexion.schemas.role_and_permission_schema import RoleSchema from airflow.providers.fab.auth_manager.models import User +from airflow.providers.fab.auth_manager.schemas.role_and_permission_schema import RoleSchema class UserCollectionItemSchema(SQLAlchemySchema): diff --git a/providers/tests/fab/auth_manager/schemas/__init__.py b/providers/tests/fab/auth_manager/schemas/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/tests/fab/auth_manager/schemas/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py b/providers/tests/fab/auth_manager/schemas/test_role_and_permission_schema.py similarity index 97% rename from providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py rename to providers/tests/fab/auth_manager/schemas/test_role_and_permission_schema.py index f5ec73a3e9893..8e44c35546332 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_schema.py +++ b/providers/tests/fab/auth_manager/schemas/test_role_and_permission_schema.py @@ -18,7 +18,7 @@ import pytest -from airflow.api_connexion.schemas.role_and_permission_schema import ( +from airflow.providers.fab.auth_manager.schemas.role_and_permission_schema import ( RoleCollection, role_collection_schema, role_schema, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_user_schema.py b/providers/tests/fab/auth_manager/schemas/test_user_schema.py similarity index 97% rename from providers/tests/fab/auth_manager/api_endpoints/test_user_schema.py rename to providers/tests/fab/auth_manager/schemas/test_user_schema.py index b37f27abe0397..648372d3baebd 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_user_schema.py +++ b/providers/tests/fab/auth_manager/schemas/test_user_schema.py @@ -24,8 +24,11 @@ from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role with ignore_provider_compatibility_error("2.9.0+", __file__): - from airflow.api_connexion.schemas.user_schema import user_collection_item_schema, user_schema from airflow.providers.fab.auth_manager.models import User + from airflow.providers.fab.auth_manager.schemas.user_schema import ( + user_collection_item_schema, + user_schema, + ) TEST_EMAIL = "test@example.org" From c3d2b9fc423a7e8e62a674c7d4657cfd3762d481 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:16:01 -0700 Subject: [PATCH 054/199] Add support for run conf to backfill (#42865) This actually does a little bit more. It changes the backfill create endpoint to take a json payload instead of just query params. This is just easier because we can use the backfill schema as the schema of the request body. One thing that is maybe weird is I add a decorator to translate the request body to the kwargs in the endpoint function. The main motivator here was for compatibility with the requires_access_dag decorator, which doesn't check request body. --- .../endpoints/backfill_endpoint.py | 39 +++++++++--- airflow/api_connexion/openapi/v1.yaml | 59 +++---------------- .../api_connexion/schemas/backfill_schema.py | 17 +++--- airflow/models/backfill.py | 7 ++- .../api_endpoints/test_backfill_endpoint.py | 20 +++---- .../endpoints/test_backfill_endpoint.py | 23 ++++---- tests/models/test_backfill.py | 4 +- 7 files changed, 74 insertions(+), 95 deletions(-) diff --git a/airflow/api_connexion/endpoints/backfill_endpoint.py b/airflow/api_connexion/endpoints/backfill_endpoint.py index a0e728c5bc464..94d6ad21f05f1 100644 --- a/airflow/api_connexion/endpoints/backfill_endpoint.py +++ b/airflow/api_connexion/endpoints/backfill_endpoint.py @@ -19,14 +19,14 @@ import logging from functools import wraps -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING -import pendulum -from pendulum import DateTime +from flask import request +from marshmallow import ValidationError from sqlalchemy import select from airflow.api_connexion import security -from airflow.api_connexion.exceptions import Conflict, NotFound +from airflow.api_connexion.exceptions import BadRequest, Conflict, NotFound from airflow.api_connexion.schemas.backfill_schema import ( BackfillCollection, backfill_collection_schema, @@ -42,6 +42,8 @@ from airflow.www.decorators import action_logging if TYPE_CHECKING: + from datetime import datetime + from sqlalchemy.orm import Session from airflow.api_connexion.types import APIResponse @@ -119,12 +121,33 @@ def get_backfill(*, backfill_id: int, session: Session = NEW_SESSION, **kwargs): raise NotFound("Backfill not found") +def backfill_obj_to_kwargs(f): + """ + Convert the request body (containing backfill object json) to kwargs. + + The main point here is to be compatible with the ``requires_access_dag`` decorator, + which takes dag_id kwarg and doesn't support json request body. + """ + + @wraps(f) + def inner(): + body = request.json + try: + obj = backfill_schema.load(body) + except ValidationError as err: + raise BadRequest(detail=str(err.messages)) + return f(**obj) + + return inner + + +@backfill_obj_to_kwargs @security.requires_access_dag("PUT") @action_logging def create_backfill( dag_id: str, - from_date: str, - to_date: str, + from_date: datetime, + to_date: datetime, max_active_runs: int = 10, reverse: bool = False, dag_run_conf: dict | None = None, @@ -132,8 +155,8 @@ def create_backfill( try: backfill_obj = _create_backfill( dag_id=dag_id, - from_date=cast(DateTime, pendulum.parse(from_date)), - to_date=cast(DateTime, pendulum.parse(to_date)), + from_date=from_date, + to_date=to_date, max_active_runs=max_active_runs, reverse=reverse, dag_run_conf=dag_run_conf, diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 305653e781797..b39d1cd955dd9 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -277,57 +277,12 @@ paths: x-openapi-router-controller: airflow.api_connexion.endpoints.backfill_endpoint operationId: create_backfill tags: [Backfill] - parameters: - - name: dag_id - in: query - schema: - type: string - required: true - description: | - Create dag runs for this dag. - - - name: from_date - in: query - schema: - type: string - format: date-time - required: true - description: | - Create dag runs with logical dates from this date onward, including this date. - - - name: to_date - in: query - schema: - type: string - format: date-time - required: true - description: | - Create dag runs for logical dates up to but not including this date. - - - name: max_active_runs - in: query - schema: - type: integer - required: false - description: | - Maximum number of active DAG runs for the the backfill. - - - name: reverse - in: query - schema: - type: boolean - required: false - description: | - If true, run the dag runs in descending order of logical date. - - - name: config - in: query - schema: - # todo: AIP-78 make this object - type: string - required: false - description: | - If true, run the dag runs in descending order of logical date. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/Backfill" responses: "200": description: Success. @@ -2920,7 +2875,7 @@ components: nullable: true description: To date of the backfill (exclusive). dag_run_conf: - type: string + type: object nullable: true description: Dag run conf to be forwarded to the dag runs. is_paused: diff --git a/airflow/api_connexion/schemas/backfill_schema.py b/airflow/api_connexion/schemas/backfill_schema.py index 7f83d76df6dfc..db496bf1ac5d5 100644 --- a/airflow/api_connexion/schemas/backfill_schema.py +++ b/airflow/api_connexion/schemas/backfill_schema.py @@ -34,15 +34,16 @@ class Meta: model = Backfill id = auto_field(dump_only=True) - dag_id = auto_field(dump_only=True) - from_date = auto_field(dump_only=True) - to_date = auto_field(dump_only=True) + dag_id = auto_field() + from_date = auto_field() + to_date = auto_field() dag_run_conf = fields.Dict(allow_none=True) - is_paused = auto_field(dump_only=True) - max_active_runs = auto_field(dump_only=True) - created_at = auto_field(dump_only=True) - completed_at = auto_field(dump_only=True) - updated_at = auto_field(dump_only=True) + reverse = fields.Boolean() + is_paused = auto_field() + max_active_runs = auto_field() + created_at = auto_field() + completed_at = auto_field() + updated_at = auto_field() class BackfillDagRunSchema(SQLAlchemySchema): diff --git a/airflow/models/backfill.py b/airflow/models/backfill.py index 37683ee6f1e55..aa9cb695b7579 100644 --- a/airflow/models/backfill.py +++ b/airflow/models/backfill.py @@ -41,7 +41,8 @@ from airflow.utils.types import DagRunTriggeredByType, DagRunType if TYPE_CHECKING: - from pendulum import DateTime + from datetime import datetime + log = logging.getLogger(__name__) @@ -121,8 +122,8 @@ def validate_sort_ordinal(self, key, val): def _create_backfill( *, dag_id: str, - from_date: DateTime, - to_date: DateTime, + from_date: datetime, + to_date: datetime, max_active_runs: int, reverse: bool, dag_run_conf: dict | None, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py index 9d9a79af51135..f8015ff907ba6 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py @@ -19,7 +19,6 @@ import os from datetime import datetime from unittest import mock -from urllib.parse import urlencode import pendulum import pytest @@ -197,21 +196,20 @@ def test_create_backfill(self, session, dag_maker): to_date = pendulum.parse("2024-02-01") to_date_iso = to_date.isoformat() max_active_runs = 5 - query = urlencode( - query={ - "dag_id": dag.dag_id, - "from_date": f"{from_date_iso}", - "to_date": f"{to_date_iso}", - "max_active_runs": max_active_runs, - "reverse": False, - } - ) + data = { + "dag_id": dag.dag_id, + "from_date": f"{from_date_iso}", + "to_date": f"{to_date_iso}", + "max_active_runs": max_active_runs, + "reverse": False, + } kwargs = {} kwargs.update(environ_overrides={"REMOTE_USER": "test_granular_permissions"}) response = self.client.post( - f"/api/v1/backfills?{query}", + "/api/v1/backfills", **kwargs, + json=data, ) assert response.status_code == 200 assert response.json == { diff --git a/tests/api_connexion/endpoints/test_backfill_endpoint.py b/tests/api_connexion/endpoints/test_backfill_endpoint.py index 67ec6316e2ffd..5bb959cabaf56 100644 --- a/tests/api_connexion/endpoints/test_backfill_endpoint.py +++ b/tests/api_connexion/endpoints/test_backfill_endpoint.py @@ -19,7 +19,6 @@ import os from datetime import datetime from unittest import mock -from urllib.parse import urlencode import pendulum import pytest @@ -272,21 +271,21 @@ def test_create_backfill(self, user, expected, session, dag_maker): to_date = pendulum.parse("2024-02-01") to_date_iso = to_date.isoformat() max_active_runs = 5 - query = urlencode( - query={ - "dag_id": dag.dag_id, - "from_date": f"{from_date_iso}", - "to_date": f"{to_date_iso}", - "max_active_runs": max_active_runs, - "reverse": False, - } - ) + data = { + "dag_id": dag.dag_id, + "from_date": f"{from_date_iso}", + "to_date": f"{to_date_iso}", + "max_active_runs": max_active_runs, + "reverse": False, + "dag_run_conf": {"param1": "val1", "param2": True}, + } kwargs = {} if user: kwargs.update(environ_overrides={"REMOTE_USER": user}) response = self.client.post( - f"/api/v1/backfills?{query}", + "/api/v1/backfills", + json=data, **kwargs, ) assert response.status_code == expected @@ -295,7 +294,7 @@ def test_create_backfill(self, user, expected, session, dag_maker): "completed_at": mock.ANY, "created_at": mock.ANY, "dag_id": "TEST_DAG_1", - "dag_run_conf": None, + "dag_run_conf": {"param1": "val1", "param2": True}, "from_date": from_date_iso, "id": mock.ANY, "is_paused": False, diff --git a/tests/models/test_backfill.py b/tests/models/test_backfill.py index 0f471fbd56546..06c41cadd8cc3 100644 --- a/tests/models/test_backfill.py +++ b/tests/models/test_backfill.py @@ -93,13 +93,14 @@ def test_create_backfill_simple(reverse, dag_maker, session): """ with dag_maker(schedule="@daily") as dag: PythonOperator(task_id="hi", python_callable=print) + expected_run_conf = {"param1": "valABC"} b = _create_backfill( dag_id=dag.dag_id, from_date=pendulum.parse("2021-01-01"), to_date=pendulum.parse("2021-01-05"), max_active_runs=2, reverse=reverse, - dag_run_conf={}, + dag_run_conf=expected_run_conf, ) query = ( select(DagRun) @@ -114,6 +115,7 @@ def test_create_backfill_simple(reverse, dag_maker, session): expected_dates = list(reversed(expected_dates)) assert dates == expected_dates assert all(x.state == DagRunState.QUEUED for x in dag_runs) + assert all(x.conf == expected_run_conf for x in dag_runs) def test_params_stored_correctly(dag_maker, session): From 2447b978e2cb71d4fa12b5d5a115b8f2d4e1fb39 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Thu, 10 Oct 2024 23:34:59 +0800 Subject: [PATCH 055/199] Fix main (#42903) --- tests/api_fastapi/views/public/test_dag_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/api_fastapi/views/public/test_dag_run.py b/tests/api_fastapi/views/public/test_dag_run.py index dab81907068e7..176ae07d3fadb 100644 --- a/tests/api_fastapi/views/public/test_dag_run.py +++ b/tests/api_fastapi/views/public/test_dag_run.py @@ -25,7 +25,8 @@ from airflow.utils.session import provide_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunTriggeredByType, DagRunType -from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags + +from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = pytest.mark.db_test From 6b9921dd1c30788554e4f941b2d32d9d1a76471e Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:18:31 -0700 Subject: [PATCH 056/199] Fix main: js/types/api-generated.ts (#42906) --- airflow/www/static/js/types/api-generated.ts | 25 ++++++-------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 6aeb8721e4280..15391c2942432 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -919,7 +919,7 @@ export interface components { /** @description To date of the backfill (exclusive). */ to_date?: string | null; /** @description Dag run conf to be forwarded to the dag runs. */ - dag_run_conf?: string | null; + dag_run_conf?: { [key: string]: unknown } | null; /** @description is_paused */ is_paused?: boolean | null; /** @description max_active_runs */ @@ -2728,22 +2728,6 @@ export interface operations { }; }; create_backfill: { - parameters: { - query: { - /** Create dag runs for this dag. */ - dag_id: string; - /** Create dag runs with logical dates from this date onward, including this date. */ - from_date: string; - /** Create dag runs for logical dates up to but not including this date. */ - to_date: string; - /** Maximum number of active DAG runs for the the backfill. */ - max_active_runs?: number; - /** If true, run the dag runs in descending order of logical date. */ - reverse?: boolean; - /** If true, run the dag runs in descending order of logical date. */ - config?: string; - }; - }; responses: { /** Success. */ 200: { @@ -2755,6 +2739,11 @@ export interface operations { 401: components["responses"]["Unauthenticated"]; 403: components["responses"]["PermissionDenied"]; }; + requestBody: { + content: { + "application/json": components["schemas"]["Backfill"]; + }; + }; }; get_backfill: { parameters: { @@ -5531,7 +5520,7 @@ export type ListBackfillsVariables = CamelCasedPropertiesDeep< operations["list_backfills"]["parameters"]["query"] >; export type CreateBackfillVariables = CamelCasedPropertiesDeep< - operations["create_backfill"]["parameters"]["query"] + operations["create_backfill"]["requestBody"]["content"]["application/json"] >; export type GetBackfillVariables = CamelCasedPropertiesDeep< operations["get_backfill"]["parameters"]["path"] From ba28e36024589276ba1d19e803ef62686b526962 Mon Sep 17 00:00:00 2001 From: GPK Date: Thu, 10 Oct 2024 21:55:58 +0100 Subject: [PATCH 057/199] mark TestGKEStartKueueInsideClusterOperator tests with flaky decorator (#42916) --- .../tests/google/cloud/operators/test_kubernetes_engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/providers/tests/google/cloud/operators/test_kubernetes_engine.py b/providers/tests/google/cloud/operators/test_kubernetes_engine.py index d11a5af90a1cf..f0f42745c1c4d 100644 --- a/providers/tests/google/cloud/operators/test_kubernetes_engine.py +++ b/providers/tests/google/cloud/operators/test_kubernetes_engine.py @@ -545,6 +545,7 @@ def setup_test(self): self.gke_op._cluster_url = CLUSTER_URL self.gke_op._ssl_ca_cert = SSL_CA_CERT + @pytest.mark.flaky(reruns=5) @pytest.mark.db_test @mock.patch.dict(os.environ, {}) @mock.patch(TEMP_FILE) @@ -560,6 +561,7 @@ def test_execute(self, mock_pod_hook, mock_deployment, mock_hook, fetch_cluster_ fetch_cluster_info_mock.assert_called_once() + @pytest.mark.flaky(reruns=5) @mock.patch.dict(os.environ, {}) @mock.patch(TEMP_FILE) @mock.patch(f"{GKE_CLUSTER_AUTH_DETAILS_PATH}.fetch_cluster_info") @@ -578,6 +580,7 @@ def test_execute_autoscaled_cluster( assert "Kueue installed successfully!" in caplog.text + @pytest.mark.flaky(reruns=5) @mock.patch.dict(os.environ, {}) @mock.patch(TEMP_FILE) @mock.patch(f"{GKE_CLUSTER_AUTH_DETAILS_PATH}.fetch_cluster_info") From 108f992af3cb0fafc94b9808317c496b62ce818c Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:39:33 -0700 Subject: [PATCH 058/199] Remove dag.run() method (#42761) This method uses Backfill internally. Before we can remove BackfillJobRunner, we need to remove DAG.run. But before we can remove DAG.run, we need to update some old tests that use it. So this is the first step towards removing BackflilJobRunner. There were some very old tests that came from airflow github issue 1225. These appeared to test the scheduler but really they tested the backfill job runner. Just to be cautious, I kept most of them rather than remove (which probably would have been fine since they essentially tested code that we'll be removing). As appropriate I either changed them to run on dag.test or scheduler. The ones dealing with ignore first depends on past will have to be added back when that functionality is implemented in new backfill. --- airflow/executors/executor_loader.py | 2 + airflow/models/dag.py | 83 +------- .../contributors_quick_start_pycharm.rst | 27 +-- .../contributors_quick_start_vscode.rst | 3 +- .../images/pycharm_add_configuration.png | Bin 97805 -> 0 bytes .../images/pycharm_add_env_variable.png | Bin 86158 -> 0 bytes contributing-docs/testing/dag_testing.rst | 15 +- dev/tests_common/test_utils/system_tests.py | 7 +- .../test_utils/system_tests_class.py | 26 --- docs/apache-airflow/core-concepts/debug.rst | 13 +- .../example_dags/example_display_video.py | 4 +- .../cloud/operators/test_dataprep_system.py | 8 +- .../cloud/operators/test_datastore_system.py | 4 +- .../test_facebook_ads_to_gcs_system.py | 6 +- .../test_salesforce_to_gcs_system.py | 7 +- .../operators/test_display_video_system.py | 15 +- .../google/cloud/dataprep/example_dataprep.py | 1 + tests/cli/commands/test_dag_command.py | 2 +- tests/core/test_example_dags_system.py | 118 ++++++++++- tests/dags/test_future_start_date.py | 41 ++++ tests/dags/test_issue_1225.py | 149 ------------- tests/jobs/test_backfill_job.py | 30 --- tests/jobs/test_scheduler_job.py | 195 +++--------------- tests/models/test_dag.py | 19 -- tests/models/test_xcom_arg.py | 7 +- 25 files changed, 240 insertions(+), 542 deletions(-) delete mode 100644 contributing-docs/quick-start-ide/images/pycharm_add_configuration.png delete mode 100644 contributing-docs/quick-start-ide/images/pycharm_add_env_variable.png create mode 100644 tests/dags/test_future_start_date.py delete mode 100644 tests/dags/test_issue_1225.py diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py index 1eeee1ff68a9f..ec79860918b80 100644 --- a/airflow/executors/executor_loader.py +++ b/airflow/executors/executor_loader.py @@ -173,6 +173,8 @@ def set_default_executor(cls, executor: BaseExecutor) -> None: This is used in rare cases such as dag.run which allows, as a user convenience, to provide the executor by cli/argument instead of Airflow configuration + + todo: given comments above, is this needed anymore since DAG.run is removed? """ exec_class_name = executor.__class__.__qualname__ exec_name = ExecutorName(f"{executor.__module__}.{exec_class_name}") diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 2dc425daa0549..5cc5cf4431407 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -28,7 +28,6 @@ import sys import time import traceback -import warnings import weakref from collections import abc, defaultdict, deque from contextlib import ExitStack @@ -88,13 +87,11 @@ DuplicateTaskIdFound, FailStopDagInvalidTriggerRule, ParamValidationError, - RemovedInAirflow3Warning, TaskDeferred, TaskNotFound, UnknownExecutorException, ) from airflow.executors.executor_loader import ExecutorLoader -from airflow.jobs.job import run_job from airflow.models.abstractoperator import AbstractOperator, TaskStateChangeCallback from airflow.models.asset import ( AssetDagRunQueue, @@ -2296,84 +2293,8 @@ def _remove_task(self, task_id: str) -> None: self.task_count = len(self.task_dict) - def run( - self, - start_date=None, - end_date=None, - mark_success=False, - local=False, - donot_pickle=airflow_conf.getboolean("core", "donot_pickle"), - ignore_task_deps=False, - ignore_first_depends_on_past=True, - pool=None, - delay_on_limit_secs=1.0, - verbose=False, - conf=None, - rerun_failed_tasks=False, - run_backwards=False, - run_at_least_once=False, - continue_on_failures=False, - disable_retry=False, - ): - """ - Run the DAG. - - :param start_date: the start date of the range to run - :param end_date: the end date of the range to run - :param mark_success: True to mark jobs as succeeded without running them - :param local: True to run the tasks using the LocalExecutor - :param donot_pickle: True to avoid pickling DAG object and send to workers - :param ignore_task_deps: True to skip upstream tasks - :param ignore_first_depends_on_past: True to ignore depends_on_past - dependencies for the first set of tasks only - :param pool: Resource pool to use - :param delay_on_limit_secs: Time in seconds to wait before next attempt to run - dag run when max_active_runs limit has been reached - :param verbose: Make logging output more verbose - :param conf: user defined dictionary passed from CLI - :param rerun_failed_tasks: - :param run_backwards: - :param run_at_least_once: If true, always run the DAG at least once even - if no logical run exists within the time range. - """ - warnings.warn( - "`DAG.run()` is deprecated and will be removed in Airflow 3.0. Consider " - "using `DAG.test()` instead, or trigger your dag via API.", - RemovedInAirflow3Warning, - stacklevel=2, - ) - - from airflow.executors.executor_loader import ExecutorLoader - from airflow.jobs.backfill_job_runner import BackfillJobRunner - - if local: - from airflow.executors.local_executor import LocalExecutor - - ExecutorLoader.set_default_executor(LocalExecutor()) - - from airflow.jobs.job import Job - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=self, - start_date=start_date, - end_date=end_date, - mark_success=mark_success, - donot_pickle=donot_pickle, - ignore_task_deps=ignore_task_deps, - ignore_first_depends_on_past=ignore_first_depends_on_past, - pool=pool, - delay_on_limit_secs=delay_on_limit_secs, - verbose=verbose, - conf=conf, - rerun_failed_tasks=rerun_failed_tasks, - run_backwards=run_backwards, - run_at_least_once=run_at_least_once, - continue_on_failures=continue_on_failures, - disable_retry=disable_retry, - ) - run_job(job=job, execute_callable=job_runner._execute) + def run(self, *args, **kwargs): + """Leaving this here to be removed in other PR for simpler review.""" def cli(self): """Exposes a CLI specific to this DAG.""" diff --git a/contributing-docs/quick-start-ide/contributors_quick_start_pycharm.rst b/contributing-docs/quick-start-ide/contributors_quick_start_pycharm.rst index d830496b27206..4a3319ae97dd1 100644 --- a/contributing-docs/quick-start-ide/contributors_quick_start_pycharm.rst +++ b/contributing-docs/quick-start-ide/contributors_quick_start_pycharm.rst @@ -78,35 +78,14 @@ It requires "airflow-env" virtual environment configured locally. - Copy any example DAG present in the ``/airflow/example_dags`` directory to ``/files/dags/``. -- Add a ``__main__`` block at the end of your DAG file to make it runnable. It will run a ``back_fill`` job: +- Add a ``__main__`` block at the end of your DAG file to make it runnable: .. code-block:: python if __name__ == "__main__": - dag.clear() - dag.run() + dag.test() -- Add ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` to Environment variable of Run Configuration. - - - Click on Add configuration - - .. raw:: html - -
- Add Configuration pycharm -
- - - Add Script Path and Environment Variable to new Python configuration - - .. raw:: html - -
- Add environment variable pycharm -
- -- Now Debug an example dag and view the entries in tables such as ``dag_run, xcom`` etc in MySQL Workbench. +- Run the file. Creating a branch ################# diff --git a/contributing-docs/quick-start-ide/contributors_quick_start_vscode.rst b/contributing-docs/quick-start-ide/contributors_quick_start_vscode.rst index 88ff1fdd84e52..61fdf501063db 100644 --- a/contributing-docs/quick-start-ide/contributors_quick_start_vscode.rst +++ b/contributing-docs/quick-start-ide/contributors_quick_start_vscode.rst @@ -72,8 +72,7 @@ Setting up debugging if __name__ == "__main__": - dag.clear() - dag.run() + dag.test() - Add ``"AIRFLOW__CORE__EXECUTOR": "DebugExecutor"`` to the ``"env"`` field of Debug configuration. diff --git a/contributing-docs/quick-start-ide/images/pycharm_add_configuration.png b/contributing-docs/quick-start-ide/images/pycharm_add_configuration.png deleted file mode 100644 index 525b73e6141abc16831d4228d60d504e16188696..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 97805 zcmbq)byQnT*Kcr#Qmjy{xVyW%y9Fs$+}#U>;uLpxcb6i?-7Qd@;O=s1pXYtQ@2>mj zot2!G>@#O(XZD`oY)hz;f+Pwe9wGn$K#`UbQvm?ryZ`_Q6?n)uin9#K1^|G_u?~`IP98RK#(V| zQ_?R2Zs=525NCYn`KNovjG#!+6AL6p8t$xE8SkiPEMbplREcS)(bc3d z`;2gCg{t~|O%Y$Z(=Gh1Tj*!0ivjgJ_D2uD-F?Z6OZRnU-yu`4z3<_IPb5Q(TRWG} z_mANt=3@7IIAs04y5oSHySOX;wl%Voo!wdwY8Ui_( z!I$QAV2>*PQ{9rw-Hc{-N(!hm!PlU!hl?!lo{kaKc)k>&EPa4^l~s@Z_R}B`5PR~G z)8qZqxkoQi7Z!E_;@6A(mb#zldsbo8;IhWyK&K6XjVH?-Oy@32V)1re_+ac}YK~ID z226Hy`A|Mr0G?Q;>=*`gtkfEGiWN0I+!+63n}@`VlU?F!En2TU;8Ot>%tuD4`>TRh}`ku$rx(jP<*G5Z-_*C<96=_Q! zxRLeQlCA7t4NZ2n%`#Y4W|xY_YN)c*lgb;NuT8Cv+DC#ZTvu8j8~GIw;R?B~2Q4HO z1zr0K=51T9hix?Nx?Be+E$!6&9-p9Qr$0745%1iBDJ>&l=c1`&XF|Emh!d?K9bBN( zyYcH5RV$W$IF06-ri!o#hZ$qII`;WjXU6XoDjs#lJnA;Xfc=n~p1NX?P(iOyEdb}P ztsA&vn}wEZ;+q4J2@5*s^@Ij*+vwfr~S1-pm25?7T{t>Uqx*>SM}7N_+Vi zmLC}IugGsJmRsX%Rj5%{Q+QV_xigl5p9<;S)wVKS4|%>No3id<$F_(xZ97IL-ojT9 z87q61S0YW!cX~m&*K+1j6m@#0sa?hq>K&%nGj z5%H+S$8*huweN<))}|-U@4S@)Ylh|PBuHuVP3~m00&0@ud5sF^0=NZb+++lAM+JD4 zp_w+pW^Eb53FX_t2<}a&wcy4=8+$-I*F&f+jrW||P-I&=7cGSi5lZh2^Gc(LsGbhH z0c(e9u`r>oe)+;KU-lE-#3Y06M?T?leBk!DtK%K@dE#kKDjLDfF`Au31^^)T+bw=t z+s(4DYk#Y3F6#r_ea1%P>|VwQhGpAeMNtzXh1C{w)0p$ZhoTDCiJCT4)|0kdjNp0r z?0k1w`vN$2Jxy@apb5z_u~RV=wjzD9r(gl^k8@|?MBnFBD~g^Ep3FKalV6g55X}{C zg52P-pl#4^Y>nf`sTw?sa~&6!E3ynkkh7R&O0r`%=tDV+#4YASop2vmZ7ykf*XMbq zT?ua|>Ygd1pC97$l`24L9}QgNP={wpBi&7_Z*Ct(_9Ln31@TunwgFZNBC6bhytlSA zZK%8|P6u1oGkTLIV@$ngU#pZ5d&!rkQI>#i_vM1zIwENGz=gzkWKfvKJU>SQ#h@$P!009}-wV&(yI!gx%1!YrHQXNuZHy8mrjM z%?p`NRt5&nQcDQRfl0Dk60~)sw zLwAft0aOJ}*sr$C$wK}6m~wPhjcG%A7~Gezm>e!96?)x=(>Q0uP4g)=rdROd>K4PE z#pG^3Iao1M^p>qBNGG$pyrD}z3PNZ+8DGv(r6YU_Fvz^>2>DKUFC#s!uX4UZ#lZf_ z5$z<}Rz?hG5=AE)z1D6c-7&vVqE~}1NYFF__>gU^wx~DTj`vG+N*q|H6clDOg#C`? zs{8Dw{x;ix>9fAK*lg`pdgy6s4>X&itz@>?_MnSf{)}hA3bHETNh*lId@Q(Ik%}(8 z)WO!$E~Au(irPJpM<<_vXAhNI=in#2As-Dtdt0B9RL2U=hwlQpOx27hl1#Y8ZQ|@E zC4yh;%kDVDe?q)SqVg}ni{~N6SR+QWwAXDkzT5sf1P;s{1VfTWfsYp1ZOF2Xyi-@2 z#0Y-&8~Om)rt#QvNiN;>4LCyOAzCWC*=T^cWR(A^4SKgLeF#tvuGTTm6i4#W+M}a0895@PkYG z&ISBT=1aVw>FX036pD;uN{DRj)e@R=c{wJY#k-_$59u^)NhvcFVxPd=wNu3+-@Tfl zJfh|w{BiW7efd-D`LpNk2=JrhdYo=w+&xE8ipDsozMPG~z>1w3I!;gs6f}k(k{zUc zLDwtdI;;$^lVXJW)@h53CE!&9aja!MCZaI)_3#CbQdl%D6t^vnej6Umf(lP*3N>2@ z9dYyK4sNCy zyBHk~cnZnI1A6j-5YMwts)EjRld1u*FvNkn(byNq zzHKP)?)iGPVnc(9mpf`!rw_eg-SXOJw6M24cSZAie&uwzMbhkJ>t z+V)0%Ck30RZV_UV#2!*lD15`!!q)L))>!cj#}21|hb!2e}Ld4a&XdRk^h$=b)>CPk#1=)LDU- zvy22B;e+lHOj{PgTQFc0F};nhodH1v-(ZHkH-w037f_0R3hJ$XzLFQ9l1bQ&S zXHb1ZQ_@r7GEW97U(mO@9R$}vUaK8YPuDD}V^Ih~b9AH$-;q*|O5b^lGm7l@!+(Cy zdd;mY*W=_3>K#~Uqnfx7xZT@Vb)|?zWi9vjZtY<;*SAaU|9jk9;@q;wm?_+bbt;GeQeG3$_ z066v&w~BwEU>4lk#C}@s&!6;@b-Oq7s{>%Dn8M0f&Y#_%m^8AFT%z6F8g6e+Q_TDX z6S^E5q>O6F-i&($dns)v000T&&l>`en*QO92JdiuNb^K<`qBJiJr{&Q*g5GwzF zQKUXGZAJXO`?5ay^>Sr{JHbu7;Ru5dL=P{NQY~eR3@scl9{&G`}vj zIUTTEJ~%T_#V0dfl^q`eSxEokXJ^Cn(E5A6KjD>Z|JLBaSTWFy zzAJl#h-l6Ii5bl5<@>Jd7ZMkyF5W*JBC2T1gZ%U}k{PJzu%P^-D0=(VhCs^N@&!NK zD%?QEuZ~$po1Sf?30n4DfvNl z(cNxxa~V%~Bvn)^HUp?uP?e2H-B|Befm^087#gNfVB`6)Qu>~&{GZ=%tY0{r*`dzx zuJe&Z-!*`$O$ZZLb*>9(VC^gYR!g#j#GiG1aAaD!O1nr9ZcI!otIcnNIsbtc{-yEx z`>%DE=}KL9;^AK!q80w*e$FS0JKihG5HF3Ti}-&f@Jh|7YN@zN71w;6CM1>BtcAcZs)%v~VJ7NW{R`*_;aeq_6n@P$58Sd*D zy9LCG;PT{;gK=)J&<<1KWkrmuJ}<5DDy;RlV9tC+UBWm9uV$~E6G`^@?(kjbIZE*G zJ``<}_$*I^{#6AdUktXFdQXBVPFBA89**YhBwb~V>YlA_M3URdul#LI=4P`>{t%Q) zR#ghXerZry2%~_)$T)3fF${TSfXkzQw!m7U3a4L;){ul($rV`8ij;L)w$BpQru_jL zu3^TUecDGzlVY9Emxhg}LhGm6+-w6PKJU_c^NBQ!TO&O@H8n`vC($^4YTBXaUgvh58=;%w%-2Rusn6W+(2S!qX7Aj;w$lo<+`4%-fs_K0GQa> zf%RXM>%v=K+aHe2@cihUJ8tvOs#r?^Os!ek=5C-W96@M*9SDN!_T{d?)V}xj_eg&R znKTbk*G*;I{Gh#*YBd8RjZWD};Hh*orMbL6ygkSp9p`8je=Rq5uP36d9{2`2@z>Vr z=^1Zr`GZe#L;gq>!Gq6bN^!g+xsNfS@;kT3{b0dyiiO(pql!gh8ASyg2mmlSRYOLZ zyQB&RAe2;4fHfWD`%#zPV`j{Gsjh3mhX3|=gVaioO47K8r^i5{DgwKZ{W2VyqcRQ| zkjJ$urZiO0kFQp1sq)pHi0~8XO=<*=;?{I{-+_eLF6?ju7;WNc&4q@fL85n3Dvrk2 z<#aGg^}NB3@cv0!&lCL@GY2Q9Qpr$6i-RF(cf6m29(GtAQfJ2|4$%@?#rC?5et>0eQMCHhh-*#s1Zm=_bk2vl7!-e{FaXn59Y2LeCEH#Qs-hpEQlTt@PzDd-Vfp=naaXzwcV(KZdV_EAf zF_@h^EsnO^J>)P4 z8_H6Azvy|<^C9Cnury&T5+V@syj5-~0VlBKjs0w4<#j*+q!uDT9$Yj9DL18r2i}nZ z-*>v)ckeM0@>QK(%@OCbm)HBI&lUJ$wbq}R_LV(MeTM`rfx3UY;c|AU0`pWX%<-z` z9>4irXuM@#0%pb(E#)AR&*S6!KWw>^yG;~<9 zK1C53k$(@TrBBlDsfku=QWYFG?ZZ#)?N63aw5lBN(=)YH0AhE6-p8eqc)>RG%(40t-rPC?YRRLZu!ah1 zsQrstM?8R5AqVy_G*Cm0vWCzInc+T@x;Jer%uLD(WcfZF zr($UuEOhYUBMru8pX{T;YUoH+4#MVE1WR{A9jDpkX3rBzEAF!vNbAmVU}ZapO0&Ym zO3OSaO4kbs7WfV$Ll`0fhsZ#RwPJa>4Bb6Z=+DVidi!*LX|vK8Dn=4ZMk$y4`{bmu z`S86+_~mY$uMI<*aHkiJgp{9XuvI$xA4n5mbsjt9eSPxT5Y|bF@r_F`nXD&4tfZ^T z4cNAHI9MmBZB9La25M>Ri|_Mt?Ptg;MkG}ErL*Sbnpda zs7wufdDscZ>4`@Dsd6IK<<=~NSKQOan+Wo8qwOv(uk?CXvUOzX$ikp6b*b# zn^T@E2B3c*y5B08kbs~=g)OW1B`3e2LSeE1YiEv)%4e2>g?5Qd&LxUQI7vf&ZZaEL zMUxSdFsh{#aPvdzL%QPDFL2&Vx6?-CQ$CZS6Tb7W$4#ueJjIAcGbfX%ULZ65BM zi`QFRAH`>9!mMS6YAP}udz)g8DeNqsj#+YY3q=4!hfMBDUdL|h<;G{DxNl+YjgGuM z3^7am>(YqBkgB$^<6q{(=#OwnK?t~3bV6#%8sZqp>guMD{)t7g@*)eESZ_0{0!6<8 z4jLlfNy!iRvihQqMs#}ZtQNrikaTmB^To@jZkaT@$qO>)KsKm!H+7W+hKGx9n$62J z!n#nzFF$f+MEcz@9fV3~v?g23jf-Ravv?vXYl-LgPZ1Fjl!}>0t1Wd5;MG`2o3Z#c zvvkl=g931*{X-%O;4?F5O!M`)*IKUN#3>lp{4cgogmW<#`9orb*ao&Wm_j7zVtRIx z@e7H+Shc^nxu&76SJnALyQHZ*BK5RdAuSca^6yWVTiTg?5^ z7*hiUc#&=%p!`Ta-rrq}03u5PbwAp$bnLsJ(l&>OV`BjPvBNE30|D9Lc~KcrCDS)4 z97Vn1f*c`09``YD$Qo=|1W-+v7M55z9>;^S5jJ^#9$N+=#Pop-!r4Q*7Lxy~78ROB)z0n*}?# zVL_?EcnF*A3n>jhRbf*poKscGf>AUx-7i0j)s*-d*kO>TxXjyMTMhW&5#CRj27EVN z>BjnZaZj~tO+q1Mo!r!-B1+!FTL6+94!|objKQW01(^^+De~@IB~r8ytp8f3II!!h zLHBoay@6IV$8uZ|!}Z>!FaV^$L0R(e#>h2RFAsq3OcQG2yX~v-!$q zQL9Ajfr*W_k$Rc~ZI|O`HHj>K3}DWb@3fFPfy+quiK6Rj*TqD|OMI$hMA)yX48}KdA0FFL%KdlkASR6tXQwsu7lO-T4D~g z*w`_WdM_udY0d!gv)0vV_kB=ncyzeLrK5LH;^W?Ta`M>$Rhv z`)@3*@;_m#&=p?E3}=tJIXAHHzAs2ADOpka<}i}6t265$NCHLI_9l_)r-=SwJK^!I&x%B?<6~tnQ&AU{mgCef3&xtQcV?SW zny+_aQ?)H07rJ=nSz3{iYs4to7yB62;F#vJCMT)veWfBYw^4#!bYp@v$G)i0~yb>rIy9T zdvns@o*or+z*)20N7clTxUh)1F_*LFf8AUnbV`FI7jE!YL6_X2ew?Qc^9vfy8FzO0 z>B*{dkz5H2ZAG@kjIo*~QoFbCAE`bx71PAM5X?UD4nB){j|BhaYEp84`f87vxhLAz zyvAq$H5$aVGJ8PaC?$JqI+BY!R`$WJ>$|C=BZ z7Vr=1{ptU=+y6@tj!#H1qW!Ukbd7hoEC1$z zjn$@&S#oB5LQ`zvt6AbR@RM)i}HmX>DBx9Ui7FSt*Y?ru4ilVrvq6(cHdO ztk?ZlTOQJx)?cEx9~89TTVHZm^#Asl$O_qZ@xuAI6lY#WdN!+THG#g(dfBH-c71ek{1V#BGr9}oVsgoIGf#1H5T$&A zZF)n8Y3!r~xaLv_VJ2RHZW*sE2vNn?6B-@m9;o$g*Um!uV|>5F)Ove;RaxL3{i z2RQy}oz5n(A6M7{{<#|d}ilXj1it#v`rM_Mu$aUc*L#-+(qf48IOsh^nlT4w* z@toU+KQ9ZT?m90~gVmIMb7@(cwp~=?-hzsZ`Ib62M*htDIsaa>)JIc~m# zQlE*>uw}{8Ek1vtExtk)^HWkKL|4_z<=R<;FN*{PFd@r87*993THus=En-rKuZ-Zq z_gHBG)U?HU64n4_c5qZIn8>D2WxF*+R(6BxDtsBn*Zax$~Ck|>tB~}E|UrKF` zXl_C%AOir(qvtNVZutWB_;Xg4z+vt94{5|2hnru<@A!(C{);g&){zw%c}Qre^K;+^ ze$gICbK!>sg8^4prujKmhZ7E$N2qVMecD)V-bU3)kvWx_B~DG$a9d(32dYkH>As*~ z{77&0XM(Q_K0@XVVbw@ErgnCD`UWj`5w)lXr+xAEn4v%B*HS?eBkoTrVXXt-F*xrI zljYi#)NhOS@2fib@}?+dcEHHb=v{O-qf6XAQ`aQYHquFY2D5W>!%~vE1C#S!`_pw* zmJp?O-c}W+Jm=n~&v`*Vkd`gezi=JBlG!3z$F#V;AF zwL}*lpr^;y_=g^^wZhZ2oeO~fP^n^BD2wQV;=zG+X_b}9g2Q&LD#BbO3c@!uanlfh zAn9OuxL@7v*ZLmnFi7D$K1q4hklu|8N`uOTso2f*m5)U?_UsD_bDU0BY1NGSbTu9Y z-wd9uU(0g>j%^}c8eAf*pEbo1bn#QUUTzimI;s@rIIGXb(NNnty@=JsZShwICVmB1 z4b055Hy(0r?u~%MRzFKfFpe}If~?279tWWP;5r!;C;HavUY7?c>@cT;S{SXqPbcgo zuyt!FTihTzOiy;}rz!Xn-kNoW8GzdFWyFvO*PE}+nw7N~KZoNs+8iecaG?*KVB*$^ zevYGgxtdEZl$$Osmfw1~6TT(TU{s%s4*|Y>D)v@+=c00AJrK{Q8 zmrGeLXq>xuxP2suupZ^LTe}ONZYoibf-%e1`B_pK*KfN@QfqU2dL9B<)VNYhf55=4$PUSiJI6C^FDn&1VXwG1ymDlef@+m$`Bs zoUj_jpdkVJs5fp|mh>UkN>3$~@fxYV3`Xfaeu_)2S^1jxNK1+@arDt8ymBCz?!>=UhIbf`I$)_#B2fADlO6b3z%Y zb36)y71hME1fBUu%n#a2Y`D~Z{OpgPx|lTKE9W`CDe(H1(U~R)iU+0<+)s?V7)w%v zkTdr3zh5`9cxpcYPvr+yog}N8%>^B%u!($V1O88EYlR+x4>uFMn= z`xLHWH8)*pv;OE&%cOR)o@f19`5JEYFjhcW)}fu7+|gw~=K%cR`rrk}0r&~srcyVA z`(BnflIIGSNod?B8lmE(8^p%A%I8(qZV&?tNN~A-5*^+M2|$(OCTX}h*za$~bP)`^ zs3wL{>>)ywqM<+V<(L}}D);#CqGioLv?8=7hR|!W9n8MKqnOfEUA0j8l6ty6baHlG zMR4%KyP76-rt9^raZ^s+8IW0i*<{-v=B<}3WV(*%Gqle%tQhXU39C$Ya&O_5I~0G3 zqxo10<-f1}QX}e68`i-3RP6DK@x@xZXl4f&m;7f<^=#ZC-R2!XEo%PCc&o3g1uqsL zndzrx^=Wh&M+ln!ApxU{6qO0pyw62wgA}7y%o|~YG%rt{FT;Jz;AFY|h?`53 zkwl=C!tVQPd?*9;Wz}JEHl>cjxlSqJ^v(u2_S@E34*j@N@r{EkHE#PyL`|cG_3?E& zT)cAeg8n{@>Wmfc+SD~_;^!dhj)n!Qhb!?NLfljD?Ow#ok6s!(@`WYwjOCWEiQ(@qX&`L`CS zG|{{AHOlKoZ`@8Ou3ev2F{8rG^L8QX4WB4C*Jyi63|=`-8&lP215eU&VaPK76L#a3q-W#q* zKoeN7xnB?K_oyNz0vY0KxuNBi*jwwHlS!^x;qI$rE7zs4p(elOED(P6Qew=z zT|nIhbzY~}q7fdKFl-H;Vy)0*D9o?{gr!d8Y4&a^w3ZzvCm)CuOj1#BHwK1Lks>jq zh~6XOhW*_O^mOG%%bvHXce+6`r&uBLXtU(0z&kH`>7BK=L0$3rR^`pUs!81CVn;Yz zGR5duUNbgP+(4q{qDwAFhT+u7589iYUe|3IWMD z9SArXO0sO4sWni2Vxv=SljU64eJ;kgU^5xJu$>#getNI5T+J_@!Z{y-=3K+}+v5D{ zxU)qGf(-=_qFp7i2So9s-bBJ@IeDdt+cSi2)AnY5EoMUHAWieM3|5-O@1_4D> z)P+h`F`-$WU0qE}>__WEtKFeO@d4Ok@*9je30PF*lW5IJT(r}kLM1_;dGR9Jt5Izc z_9wrMSlw6}Ix25Lh9M^z&pS0)v*!3ofIsotdXuncS4nBfd&mGGbPPoeM$QTt__4J^ z1)D6x3+H2aM&6`s4@qNMEY4d~w5f->#LkuM7F`AbPj{1Amz$Cnr`PeAq7XHnyJlwD zOnw<`6Clr{61Aa_Cc_L%ahKEB*vvvW%mk4*lE;ZY$7rEjSOi94xp2YROj2x#*KH<> zvz_^%^#&Pvb1-bFv$Ah^A>hA8*ubIY4wFGt1{GT6e_8A?zAX9#@5P04)t-C)- zsCl|dmCOIeepJ9CpjpuC%CdH4&HUKBDBH3m|C6Z3>^xNA635ojvkfsuW`);fG!;79 zF6kJdeusHPR5SXq<1J|q|12Q-jN{{WvRSs#YJNJ|S@u<6^*VlbQi-x;dNVdUP0h20 zh@%RUC*yc|O7&Q2@f+>Z)&TC0_mBL7O}}l(!KLl(jv~5}Y1BSaRAyAQkwBh%H#1T5 zB*9EKyqCvOdHzZ#P3!3Y)B=oKm5~{2OzXtX&Thu48ToEJCcGAV6ojnjM(gJX!3M>? zPs^qqrnOMAC@+EkO^zI<4*O#B*$@u7-U=p34AiY-ct9@Ws>F3?f(N7No zSB(KEKYl237ZH`Zu3!z41Lz)6?6n(qXA2|=(Hxgf zI&F_A;zikfd*Zo^b&jtJnir5-YqPUce_L8@aQqX~dqpd)sHj1Y!Pj(^mR^L1_i5a2 ziJ_E=CH|75Bm`x#zhi?iG{NcuT3xrWD;rS}%Rqi=UBqCFI!0rROOdBwZ`O+)9vZe- zd*F0!+YlwN>_vTQKiPM>&Ln+2ke2fujEbLYG$5LJl5O)7c%H2HLi?B)<>%J)m9|YY z%Xb(&KcUzF6Zw;gsLeC#@bEDj@h}hLmtbTGJ0X|f(4s&_M~8P?a~jRuLh>Y-KR3kl zszz{upkoaF1G>qbGRs1z^hGMV`wp|M@v)Z}Mn zJ?v>Wluyx;dW%2luZQG7l-L&i!%$XpV^aN`>u6&wiD?Te;g<%}<50d8Zd&{lA-vmz!$y@>4rtUWZbi0+3HtwbgEnK z>{*Tqdfap1q#6$jk%{x>&j`btK8W$w(bVLDf~zk3VWYtV(-Jp!u6w$J3(Juk=Wx{d z*7DSardu&Y?f}<*$I5^N_BYRbs=-pwCH-PCr>U##TU1yz)62YINUz0;U5HoB4o_>W z*DZri+bgPHch;4x&*!A=+m8or*8P!`atq`yj={Y#0=y>mu7{bOsovN&ceO0tuz(}4Y80@c!3>;`vfj^bD(*Ap$2guzzg4^ZBBZ+X5r?zO+1)esaEu}MK` z|6s`ojW~RfGMZa)( z`H@%jO)*zHpIZ0&%uvt}44^g%oP{vC>-h)g*M7jpc;%62)b8BdEu-MMtl6O{e7M5JuBO@!6Bf#57hR)BVCK!| zw4lxEO!dUJIjdzEhXqLGar7^lFEA>Usc~2cnv9KO#T{$?w|Fdv5nfqCBe>%1wdkpb zEXm<~<|O#VaNy^>5WElUeotVxYh9VIucreIk(=#5l~aO&ykyyCKP%GFAFP=n{ChW~ z+S;x!rn0uPBMBeYON3ZUXE$|1RTV1N(#lP3`(}toR7j&wdC0GT1lA*W5j{_2M84j^ z>JQriO?(H6Ty27v5>~@GK=YK|RMM_jY>YT#s4fp_QEpsq`};8$zPl=DwL+*|lA zC)i#S$$XYMw66-jF*_#mVuhrM~^i04cC^Tv3`4(|72Tcc=WP(h96zN+L38 zaLQX<#7SWA>x-ag5cWJG6^vR?Wkn@A)W7Ck19pzm;Ep4fc5VdHv__kT__TeC-YVlx zooK-kfwl3Rofo7699hN_uDNng=TxP)YBV+FwCQes>6RiL-hq{Q`N#~r!_md_y`FMU zW=&^Q;Zx<|mJaETwWW1(;t%;5=dw}k*6r>aYEb&vl{=s4O6Fmf*%5)|Df5;d^Yc>B zS;^#5m`zwJ?fP=^)oSW0&0a-elf8ej4*X-6(89pBD~ z$K8(u=HE!xtS4r3Ys+rWLaLPB2#k86`$ z|I+#JEfl3q%lkNWecE#RvC(4nxqMmGJY$0-cpd7j(#pcp@;(MS+xaGXym4%4wYrWoGD5@PV>4>FJc?44@XS?Oo1dSUc*_7#9`DF-g7wrnD5kENbEaFq zO}`uYra9Kxo-NBbj?1@a68bnQcyk15K08z}Q&ayI5&!KoV-#X<(^y9B`C8lZOs?PQ zKK;$X9BqJ*5soWlKz3;<4L5g7dwV+rNKajz-0of^r#e5MjFpx3)29vaOp#)ifc)E9 zjgHb$Q{P7<*)8-R6Mza9M)?&N^-0a?_1L_u87WE{#7z;W&nFoGA2<`ky zZb*rlS$PvGtI%e;@d`G`8C8hQT>Zc?#|T#uP_OHBKmQDBb;-%e@$m4#``g|=ynTJE zEb=!c8~LsK&9gCzTn0A6K++~-sWx0%*4V~qOZ)> zg$6QdkaC6F$nC@zs|nt5HIJ^Nx5}knB|~8^t0F>(mK%(`por9>6ZYx$-UEw8?0;1J zd0~>9vMFn@u&-}@g^}@^;HSF%c|#3v@XgbScW1!F@2XvHFsqCz;Ws@2U$^1*&crcV zrFXsuBAH7Il4x|-s-8}96BRwD`Gs_tUJmg%jpmmSNE3o)^XoldSRV~|J;M_c6BCn? z4#8mX;bEMKtW|k=xwVnekL2XLH`xKQsfMSgRkXFWD@8H|d>$OgWo2a#UrtX>)*{0A zeV?@YJC2tdtc-qtH9w>r>r&aNdph`$Qh9Jagdszmy=eDBoHD9$G90UO-o64;V%lRf zA+vA?ruxk%Gy|8Fh3xnITfaY$J5!a8O@7<+;lA$>=e?b`{Z`bPRjPhZI1XbJA^)%m z%UjLY$;O53A*$a>tA{%TS~NbVYJ24GT}nbC~n{YGFi zg8~px;Nes@HQgyuElQT#ZFbli9UTQ^+kQz&OLIGz9U2|Ae%lpA3K=Y%oSLetRx$x7 zGh=TO^Y!9Q8{56S@9gZfS+2K`4hOZlRWX40OouUH10evdt*sCc5T2f%lF@`HC@3y& zZb{GJ@cPU0{rk#uAn17P%43-is_e$6&@P&$R5lffh|XbV)4txv&xZjL&`SJy;VZ&3 z50CbT{V#`j+D$|)yvlM&-$?L;Fq`HT(|R}Tom*`ce$-DyQa#@6f;(2U4-OB~8c@=! zA7m_TK#Tl*qy8fs`&W0Y8Ib-LaR#^!>=m6zB`|=gEz{!$ZhZR3mL_si` zy2QpZRG_1evZf|2DS-`vsP1Ww>_n%hQ(s`9i;JhtL^dG%Vd+I%6l!G!@6ht_!_q@l z*n1UBn6GCZQK|TExuhM%&ZmtZ;wd?YlrAnDa|`UXhg^&I?IUmfYHG(5gnabC9D<)>#Qi7ipXpCdI;T*XJ&cY|}gESM++V@_J6}6^&q*xC)w+yB5xZ^!@@$jU$$$VarkSWbS!hX3cdMo9+ z2e!X~!XDo5nT^*xNuj5QrzhWI`fHaf`sM44hCK%a`Z~#cjy5XW1$Z|rQ6SK`EsE5) zGkxuDK;eCnO5V9l`1w`~ow3qfhlC>+K5Et00nTUKg!|^j+y1)vQjPVyafuP<&6eRN zMlN&8vxXH$FFMG7HltD*x;PQv*>{4wYr9Ly<@ul!!H0uq+3w-RUZ+Q{kCs$rUvwX$ zKp$x>r-7%A$a{Y44-xy#?t80lZ|ZMYd!TBQ8*x}RaHnNiM&op63$;fS?iGhojBLO= z+m+FfQ;3*WFuK!*qU9~1e#c>o_;rPK3>RAXUH-1-o)zNFjfTcsI_G7fzOTiNL*b%n zM%B1Zy&0B+pf!30ixl!*m)=}J1GUL>=$L>do2McxGM#jg= zL`_UgY-}pJy8M8xxPUjK#u+?a(E6H|l5+p}xG+C2AtAw0^D{jiKeQ|j2C}^d5+PlHH?z*HHoXBQ(#+{7M_zqr0V)cyTjaO;1uyZ*G#OR7@{#bY2 zAMm44Cj)Y&u4)y%PrzFPs<1ryF}l>E^k%5n#39?KJFW=)nYe)m)>$c*X?Tf_o`I86 zaD7q8;Im$-4)%$tSv1>BKC8&B^)j19u36egrz1QGan1iHQ3rtlkZWVM!g1b07pj&A zX~j(aM)<|Kdri>Ul?&RpOX96jtvUj6)VPUUkBR0jyIiR~9}e|uZ2-`)CwCMH+c7l~ z`6Do{Z3duQw}*PkrmtxkQ9MMmkr*bxtPjPDcEi&s96L>JHd;*iTF)mxro4{{1uF=> zomMY?6}28CX|%N+gml3?yH@Rwkpl9?fEn(O(;c5*mt@USSVr{Ims`DV&U(GOHu83n zXG=U?oey7dF|G*g810Q9ff?^y&-H6g>)-w)pl(%1qIhJ$PeS6mcMa<9moH1Znt@92 zu$GyOACb#qt{Ye>!{TeR-*5haZ{mU-$bh1Y$z~#1`5YrX(#dQX&&S(&|NEO4_~QU~ ze0nOc=P&-p(D25)ebxJO@LTW$*4>!?i7=CbQ#;@FNV$8Y2M^M)xkCdDY22?>^6gwI zPXZI>nLmvdsa6s#@;3R!5^s;P=@xDiTBy=BEFly|>7LzHIpf0IMq3+jy2ekBO?t2h zY{l~=$RU8wW2cSdnUV^SOCJ^nkJmk!AfzfgsHyW<-YcM^e2vjjpZFPd5JGaohkbQv z4CT+nb0M5im=*nuQ?CX1>k;NKD%kWHglx@vF`SqmCGzJU+tH%p`!YL~T&CTgsrBd7 z1Q<`SK2tn#ElJbAo~JyA{QAAR2t;vmLaY3S&CDNy%#!3DDgK7W=#ze$EsuBHWh=F+ z3@3|RwX%BZ+a(UWVlLkMExg|#oRY>lG)sq&U%d{iMB!*9#p!Kz-2#3(Cs*GW>GQm0 zitAA6npd+&W7F2*ZkE!w*e=ox%y5~*h={bCZbvg_3F6XC{|{kr6%|L*wSf*0BxrDl z5CQ~ucL?t8?(VJ$?gV#t_rVFlg1fsr1a~<7+V`KUbNXT~n#FX_>ZCt07)Q4-9a8Aj^}c~ ztS{h$Ht%{&c-(@x+$6K+v!idlh!et|nl#R~iU`{MK7{>nG3sTjtxXWn9Xok=c6LTW zLgL`y@a@|-8A^sPU*L}X*z9+Q63?kLbYB5+G|&;%)G#P1O`BFb)6vnvfWnBQaPaYq z3kzp?1z;@DvkiDHhj%*(jChgDc|7N56uLx`0<(~5aXBT2G+l#tn>#|xIN+|XaMC{aQ+tmNSN z2!(m!E-?VhxqpR?vvgWdaXS=S1qS0smv|-Rk4|$(!Jj&=N9aj^+FSh~cp#^1CXA7%p z57xhLaDM1{k%vy7uaBFyxt-)7{?jz&&MCSaZ(OiM5m|X!q9dMd=qF=Jp}f4ZclmH*Bqu7 zd=>0DE;>bcjbwkK+t~}2#RGlx%y2XJoAzakFj#b(Td|W~SDkfQ@*Hn{i#ilI(=Y$;^s6JCFY@^G@J(H8Q6%{m`oVcKqlar&S$&#w7=!gg? z5EK-Yj?OBUDh(M~(D;76dd0Zy)C3FCpG`Zxuq?X+e(UVRvOhL2tau)Lxwd;#s##D( z-Wo6UKiBzW_$@pk|GCKtrdMyvT>Jw54}4yLv0KzpB; zx3yi#OhrWn5|EIPaDA`H9n+^k)$HXyJ&v$f&6D;~mBb$ldk0H$_3)cKb3<*T?e^B` ze!*3hbZ{J3$ndTsr{zh^oan$RoCG#hDYO3OWj7(g$|IdNSH?Jqiv7ei@&k^lPMap9 zV8-nq16jeD4h~e%M5Ej%BDH=oUb;d!lku}m*23ooU0L66S+e=*?GqS%4UP`-`BD)P zrC6Emj<)OJg@mw3E*{L*DW<}r_qBCBLfc=U-aulZE%+*u0bkB+jSJUv_|h0R6?Q?t z4!BWwMRg!Q)7vGWLM9UWK9McloLLFa&j1J94Np7mL9^Mo6$A(_K(MRgK;de9N5O9o znnsUxA=cV+Ca@?o0M)q2WTsM$BI@Ln!`W7~s$fpRfrx)BIn{%kMt5eXRl=ep)THg&8F?EL34GCFN6EplSiiM+rz7| zfoDsOs&x5$j6vc^wR)e|TRHEg0y{(G6RcPLRsI=SXn}H@#HcJ@gPk2@wGwmhXsYC= z!wdK{Tq9C5kOvj%a$8xr$13bYd67=Xv(P(%WZp<`!3LC*f}k$c@V}*#^5v8$oZf87 z(>C&IbRY(6SVTvXxESIv_j-Q)xm9k7z~P+kTVkEqq2}_7m6bN?x(R$n=VgQZrCB{y z^+=rQpiGae#|RX9YH=euIc9M=32gSH;onFd7mgwww!o!bV_2lJa|FTDB>d5~gf6xVx)XdD)-n;^&7@+tCz&AGDp8IMN7DS|( zTpHfWgAQK&KogHeP7xL5V}k3A;ks4ELRvsTceDAr@I7!8<(3?y!?Mu%iXdh%;Pmmj z@o;I%l{_>rFAtqUmSu!{SW5`FUP#EmfLlvzE2IEW*4CGsSz5w_CMPF}S9DcWRKSIi zVG^YL9ba1B4yLrO+FXe)deUs3VzM|KcCG`w;qguqs^9>!7!_LJC>?L!C##2riPab+ zF9O|%0h+GW@@EXd;)y`)a=LS%E%#yfBm5H9>UE_%L(SOjcubT-;V=aM?$_3oB2;NM zYZ#MV1upS{p~cVXt;1y>r(8(Q$Mk4r1p`!(qod{whgF zo!z>w(Xx$Mbe3(R?B%K}G`8jNMg4llxYB5F&` zKrK|$;!vHZctUMY8v>05%*FX4@`Mh|iaR(9eSf6xTmE(kkpyfkLo`yfGC&%Ykz+)0yo&3GpXQ|NHmv05Jd@!MOXtCpi|` zcF0Drp#y-4C@Lz}J4IK12Q&-9D(-K;X06LCkPrnXY57lT122>8icrnhen>Du8*c8w zdcApHt63GKvnI?y0L@vtwuBi`jGDH#wvNurZBljDYpYJ1D*&mkwtF}!C@26pw1Wd< z%s@Q-rhM0w#%6m*2gFAjniBx@w6m)yD|_6}_5;99L|`G0j5JRE_|S%AGVl6=sfR>$ zb|w49JFmmM39f>$GSt*ue3WzPO~3)vXhboA1XL8Kji8Ja(67h_8b(lQWxfa`s9dM| z42IivH_&cTJb~fF>zRR^?r82|_!zkZ3smkiex}$t`pU&UAg&kF9qUb>*yMJU!upAXyOHfKqcZr8KZfKk~$6w8@zqtJJa_GGcp$%FJa)(!n9gi<56)*Sa zIdc05ABrgCM60TGIkq<3aU|`NLK-)?f+DPB{@Pu6BbUk@K8bbI5UVdNCrZ*ARGmQ zkD%{m_tm$dfpNoiPGxYkgf_97HWH(2B1`XcJd`9X+oe4W6V@(?J4J_7QbqCp=*)je za2#177gP5Dip z=9(yKY0n~FR#2iY)CWgK#wMtipfZs&rEZ}-6kRK<+Uqf@;ZhQkiClluf4SMU!`{vj z=f1m$huTO*mpr_GwP@YcF6Zp*OhZpECnpCh(PoK*hi7SJ#a{cx$LFP02W|}w?j2IO zvaT)>Hny3*eok?*m^X5z=)r9|m-TGpOV=3pPMcR`qa!J4+%Q>bMg;*p%^D(P>nC2W zAcJG;l?=q{kWNRbTU*yjx0Dk$%l&v^$XW)jbFv26))v`V8E{duk(pU?X6DOAcR(vG zJ;;C56a|l^L?)f>?tFdUv1M~})39~{Fp!^}#j%(I5H>u>o3Cq;)&f8_PfrO^Q5~)i z6aWf>FI!JnH!CAUTSMbL2!LU6adGR0liR^QHYSBBFdt`K5F3=$t;w2%k~S$zzo3J* z8fqd?aa9U*^!+KJN`5hyCuO?hJnj4{%Z@P(|2A`Vyai+t$X^~T3uj}P@p~mfd-K1! zX0~~jhHhnlRhyJ9?(CrboOx+wX*oH}O-0|hrLb~$e=Yv%L?cvGk?sDI%a;b+? z>PLz1IYB}YNCn^Yu)VhW4!0-m_)0!>@p)WVP2?zo9o=IYZb=Q(&pLvw#M_)2;n_6i=IZoqr+V0=3Zi+gdn!K%E z3|B{F7$KoWU+5Mo$9TLx3FRE`8Vy@7VALIvlV!BdaX0a|N7@>i{pcrL!dGD2AE%Ib znbZm{5`GKvN`6=8NW3D4IlTDG?tRcFA67PyG!u`x!c~aGC|i7*4Wb|oNM1Vs%`(QD z=d<1k{|7pjqnQ>Y7DXG4(X|3MS*GH+2KU9w-Y<>2`0b)>9ZX11xu0G%q)lB&b)Z0H znf*xM5uVzoc~Uj_rE4V1fcO6{^_mDe&JPQXOryHxbh>`~OPlA~xE|Qis}-P+NNvt8 zpA3;tTXiJ;oDXSls_x>UJOeuMO*DOs9^9U`tk$?WWV$FnKQ3%i+$3XKygd_=GhCuL zTi=fL7lakVW_x-*I)9h{#hRE{n0BzZ`unnf7`4Oux}x9spe|AG4&~rrxwO344J{Q&xNb7Lc+ z;PZHWIGGQdJ}N6Eg@A$*6CGVuQL%INkd~I#*x2}Vcb?o~71HYj*?om%ukt>s4DqEl z^MZ*LQ%Ig9?v*q^rmw;Jyhyn7cWI9M$0EaaSEk=N___gK==R(p*)s`>9Dnxv;4+)QMjF3aD)f0YWlo>YVy!Nz^vfCO#r z^hCT#gAep8hs!*CyvE-a+RmHF$M}&q*4s|rG5Ocr>3;wIkqSOEycuk6qi}3puAKfVQF6UGCopB9(rhH5LQIdo1O+Tv$MC%B^`y$+t7Byx zjo`zA37biAn;k^4k(oPQ4cRA7hD_%}4A%c>Nv!Gn;M=~LWpA1Up-|kPw{_iYu&l5iG5-pb(8b@YS|2~Dv4?~CN5}}*!RXvQ z2(`a6l{?*euv|ZuHjrTQCp*29%i&{%_0Nwo0~W*NX9U*+AI?VxyI<$hhE{5xBtJlc z81Q@#TCB(322P!9k#sp1dv<@881T6&s+!n7kJhVQCu^_Ij5{1ny(bA!Lqz=j=KYo1 z^>Q!@|5vjQ`}oB!WCFNGlcsvS>#TM=BIqRqLz;N&_h5U2msJMZ3JBzX&hA=pu~r%X zR9lQ>S3O)7**UAPJLYWfwE5iVwT74<#xUDG8_7n?MnNaq zn>ai|E@$(tNq~UkAdnRmg^`(l|NcEaIoeSn{}IgeIS$EL)1O`sFu$rD4?(NL;+W({ z=rv8oQ}lah?@dbw>07kwaKU0`}MK|o5&a=d3X`hMhRalS@*QJnx#!|!b{?};_i2u;`8s7 zjiDdap?vvWew$8Sc8z^CQ?9uo@QQq}>^_0mYs5vY9{vL%%FS+Kd3Ufq2q9GM#*Z=< z6tThRSmMIVDbv!PSJo=3^~v_6-H6S1buQzLb{*_Ehs-&OdUT-WdA|^ME!)Pk(LdY{l9_xvBw%TMv5Sy;LUA^L1X(cY zhYq03GsDFAIs`i@m4O6~P#%~Q?CQ;LI%%cul1kiLRc-UR1}*gY7Mk6wN8Dz4{7@l4 z-|6(miQVnD=&9vmG~YZeW$>18O^9Die9JPdw)hPDiu&SDHRxGHUQDcZV0i<3{dU7h zlcbXE*4t5OsP8reTEPc6_u9)w>; zru&_)rRtO!OGz)^X|!7%f>0+sW+OLUB`eJ>~epo!Fxcp!STEdMmGG!Q%v z+PTX9fjN;o`HM%%HoDu1xC99U6XefA_|i9#O;6l}&-@Dy>fiVF8V1+fPWK=907l{m zey=dzDewD*vvnWWqm7m+LE%76h^~yt?3GyA8E9>A`zI1PYRgBz7n~Pn4f*8E#I?Nvsirc?X=oO5!^GL zR@17YsHsf@go?a(%QsKH_j&#%N1L{v4GL;B-huoXHun9%45qaF&4y!a&{8EG`HZI8lF*GHA%MtLY629J{( z$yh%%2hW)r+I&kSs_$N|mWoK8bdOF=#RbV@09G-LyFAl(f;c!hHR=_6MDluidNf$& z$%pP|aaO=Q`G@>J6mxQN0-{Pj3h8s(uN8f}I`R^K+U|KEo?n|jVK6!NQ##@~b7OQ7 zbkaQ8PA7628tKny z5C9SvCRw;;&mkx%C@U*l{Yqucw!E;=Y&Bn1O~hXA(4uW-w%@81D70Ce?OTSNDJ&$^ z`TBV5DMqiLsR?f*B(0tI<6D)u5_s8VqihYU>3>{55aTCti=T6(HPRHnLP#fv5UbXZBaf1M%_0)>q( zt8MLE%+mO7cW0|=&%5_tRoB@um12Fq9#4;M(KgPK!+_FCM_XIhA)`SZ0AXBRT{Sdt zWu_s&{rvee$E03f&-v-r1}4D3*m!0oqOYSBAQ}-65O_}8(J?VG)v9%U?*HU^p0!6u zM|X7i?2lzWc)vceB1%e0f$i4U^2K6YPnTxfpz24jQ}Zh^RdW;b%7?>;t#mVRB^_DG zzMJ|n!ueN01s>2d5IOVA8TwqxSU5v@FW0)RxoxlNxGAD3 zN`GG~S*fjAv%gvEMW+Abt@q@8n=d+4b?9gWe@$%mo9uU|l_DjSsbhJ#%yN0cT=he$ zGiz8J^8rI|IJVe*$-s>0>+kQUO`(fgv-0L!|ND6JsScI@^1Ky6j}r+a$18neecjJb z0)OkX#ld?5MB7TNI`|DdujMcDyDAJi5^1kEQGafem%X~N=7Ft#-CuUV`pkCCef;BY z8<4U3A;gbYrGj}hb5C&?yynX&o7~St{sz+zSoLew$;ikIYwrU}Ys)x}*Mee?wd&x9 zD8F^;>~H#I=&}r!RlbU7D=JQ@GHYa*M6S6-^^Q&?sm=;iCV$$0jk@2$rn8i+LCWB* z?Bkfe3OGU~_z$PYZg6i51idC-XEqTH2_H2=uV8FfcIH9J2B{ z&1Ep22t6S5e^(Ho8Sapxl|{2i8yg*^rK3~*g0T>3gBZ~lC;l10Q5x6D^tU1>o*?}e z|4xDWOB;}ThbH*p@$X{+1Bf>8+}HyDt0MeII-rXp{ZGo#znwN*2T|`@hgbR&ftZ*W zIePN80B}aK5Doty<^EqZVu_9Z(a{mx@%y8jCBTeUd-_4BT0~JXE+PV<3Cc2lXlSSg z_3v-fg|CJSvCsH8leAHW_y2d*6b{lE)55|+1JeHscE|<4|9}1Vzph;kOa5<}^smHK z7sYhB0xPk}Z)t%aBVL#-rqNiRJ|A3GSH}b#jEuF-%{ZXXXkkFF(bmQtGhj8^mo#V$ z#8HCujf_aKB5H}(f2WpyqUVfZVVyT<`~zyp)9l`;(R(NtYHh7c8{FPnME(0xjZlXk;WJ zB4R^hqs=C7-m1)qpEzX+1K>qKLbgrqtN_ zkGGgH0slRD66I{wM0WYhUp$;`Jp%~X^coE35sa^EaB1zVO=T@&hU)J^v^$#2 z;%pF#{wM^oqnz{MB?@mI2#jK2W4oO!FqSHF`99zOAQdZPk0#)Be7?U>EWKW+H3GW6 zQQji)&+zc@Xz)QtZPo0yMHXEA{KOz&5sHtCo5BC<1H#SMmrrK8vMk65U5SXisIz)| zTSZGrLswhz3Xkh689lAq)VWkXrmpVWY{Gig%*F%szvJM8r(8p)3hVt`r*N=GsTsB$ zUZd021ZFx|G*Xi=XL4jaR+`+E-ZKJVWrT!<0S%T|G{K;^>$Sw^IN)Z9IP5>Nu*@th zq>S&gAOAj@@L2Ek0sZkbV;2(@4Gsyx$Hm><-KE5Uzv%}k4z3cTrO^AuF+x@R+Zh0(m<_kFK^lpkDFFR6IU3(r;Xen5>^>YJ=R;PTjIas!p{y(Fe@tATzx z-)~KP8@G07QMzUVa8#;9Dh7sFjrC+gX>w);2791y2F|Xnt&Q?Fb2+Q7yX!y$i_v_Q z4wV1Mi1ZB>P|SmVkB+K*!8nQn&N{icnAEDDw|g+!idF^|1#ficzoXEyayg)(ru6xc zykQ)1jQF>1z5`8H2(k(ggmRk#YlvlI)>otO@~+62;cIZzc7LlyjX!` zZ4JK7dY?WvAF2K|63ERVabW5&}x_lM~Z0398UD9_`pALR)qXUr?POBI<Ba z=8K0Gwor1e{2v;9Vw-53WxH^>&xI+>viRrcL-BVa$1$;eb?;pHA8O7IyQ<;AY z+b{8O^4uYQbeb>Kk|3ioicvsduX^vg0|EM7U6J)?wJIGYuFhBm@rT9JTK76#=JNRW z>E61I0=}{a(t6KVj|LVp zH=l?E%87h%3f-=2#=tGz-e9-As78QI>DsY^HtAoSebZ__{OX*fINa@b^5(0_FqQQ- z^~k0?FtnU*M!Voj!bDEi6%J(198VMy6O7Gj0V;$t;r#pFFChI!Jq!;kL2gmuAdich zh*q9THDQrT9@fFSw@o70DOVJgx0IsiKjzyx%SzQNdS@3~^J0d%6N6)#gk0$Fa--9a zWj+43f(X$DKztH{Ys8JJ;1T@Oi{@$|NCN$b4V?lIgfsBRR8S5}Me}u>#>Dp-u9Nsq z4KN~FoM7|VMt1zOgUb4P4bSYubkV`FpKMDVRFxr^f zV7KZlhWzs*z2I$*%9C6Ha+*1W#FHOytSO(t{Fnya%F+uug#V1w1q$2-o2Fu_bd%XQ zQH%ELit9H42GbFoTJ^}D;{Zi%?TqAPu;nb(&1$QI)2S%P{Ly=}b=>l|Q?6@$?B~To zsbm1@y1sS>I%$^`L=ZAEGVm(-Jv_|J#B?-OC^e08FN*NIQV0q0bFlx{%tB2=Fq#uS zSIp$Yo9~>_f^6dYb>Z>p(69N*l+`Zrt6TnUnTDsA2J(ZE61y(G56$%AA1-<#PiK2v zt8q$%L2wNf?V{rdO|cV)`*YR0ytkxbJ&AS%gF|AQ19Vt-X<*|j&hnkJSJvZVBzVS~ z7moY%A5-BaI#z6QM3v`g55lnI6NxC}#2xB8SG&)tNzX@-)K6^mc;1Yr`-!W9{W8>A z+w1lUxG0Jp{eB%&fwEkUF5CMZ+hx^R4^vqZGg{Umc`w;0PiJ}44I)~txbiKvrLC=U zY2o^ks{!PGWdpz4TAfbiNJAW|^|H^W{_I2OI9rOw2&ND7^pESF{gUOYy#Ojk^um=n0s13sfyMA#o?HA`a|3VxUb5Fr0mkK5?QObt-M!eFFz0Rufz zlPwt@jb$>3c$6^tm)$JljR}At3n4LVkMy>00FT;umn)gVp~1Z~xLs=={ikPK7qpIw3N(nT%9i|k zEJch}U)A$viFE0<9zu2{2}6Yj3k9Fe`h2aU5di?&BKq*yta18B;W`u)X(jR$YxW)c zUwAknKCI|=byeP8ahIOiso$#wPqWWQ)4Fbhytk@tvTsCH$%dSa z7x^lFbdF|_@;)B5$m@Yw63?9hnUCIsH3xUQ^UTr`{`8$Vy9e_oli>1Vq=(tIFcYha ztqQs;l=CbDQOKVSlgLoo#g)&T)>C^R-;Tbf!SHZ8R2c7$tzn4v;*cB&5Ka^69({^| zB&0O%r_q{pkBx~JJMAU*gqku#_L228L;kzVMPgoe5Q9sz%sR5(>z^w>K`S4;D!ch+ zm}7AONuvRB#gCY$mqI(%lGq_ZK=cL@5#O@|p)%tz9{q?@h>vQAh|cyjnEU(>u(3#C zH3hmdrCPPAW4kvO>ut9HP?rfzPb^hch5c7rbvQ}??yPZ`9sC3j`sh1QR({k~_&%;1 z^=SECI-6VZ@=()}WLDXaq)g$VHfm~TPonY=nG$z!;jF0tNtjsoV8+S=O2a?73FT?heF z)4fD$HNdwb9%qdNT}3uE!@xerpI*7aAOR6UqE?buH>@k2QCbmth4S+vz1uaW%Vz5M z#75)|%LNR7@=gFJ^4X)$7yLdOk&MiTJrmK>_l6)XFNKY8$zozsqz@y7=L(0!!I|xQ zXo53lGc$y^5rr744>@zlckFG$tm7RCJ|NGU7_0m`iLKT;%B$`VOpmd8>|L^1wYGN& zqP@|ansrg88}=$|$X>KlAap!rX2TIuf`l-N9P>hKM$z|RG&3sh=M~;1lT9((7-7Al zzJnN_eQ6HA+iz0?6}gXd>e`ky(NUG<*Ve=M9g+rkcMTj^Oe5kh?h%3iQ2DU z<$**54DlIC6deuqj}%|g999oPF~V8i7xhqdc@ux}Nh6L;>;Lq<0@VUd*+IeK6m=%% z2aqE$V_^kUl>l{4A)5j6|M^n@py&uxh{>s`%kkN5feL$`Znr|$B#$iw=>BBo(=25LLDa(Lwgnv{hs=E^M3Y?>I5^N$joDlDbAGXU z;5Whlv;bvM%WX$|rMXPoGje+L@SJ}1jXB|km3>rD^`2wUfdaw-^2(}IAdW^i%puQl zyugJC0fF^V>H>kKojTU{xg?2~hb6xZ%f1<+$9?n5Sjy!!n~Vuqre(WGZ{7hD^SjQK z6f;5QwYC;*V&}nMAy7_<>)jpBdqMK@$~M<-vjgXsVjp8PJgF>goavE+D=ErX+k&9-9?# z4$!Wqrltaak#jpyKP<_&XeI+Q>+g~QH zE%ZF2O7yY6{cfNC3YB7}Ca1%?)Aex5DL`h2NK|{DJAY!*26f_+Qn-%={gR@mM+xi( zmqk}B@Bwr~^8y6~=z0JiAdU*k$|O;0YuHJJG-Ntp(OEKUcL^iAG;rRL7WTCR_NPbk{^ z&RQ(p@dz$xX8K%_4?G*|E#>`wkM=d$`9v{v;}Qn0sV`x$*eSP0MX*Fqp2G>XxTKG+ z3_Gg3MHkPVc$3|uacec(A-m3UPQPV#Zyr4xu7;V|QBY6_q=`BBvWanU+)oy2p$BtNlENX-Immoa)_ffB4l73pA6E z>J<8%lz1f}R3^#aey$jG7HWGdofr5SuC=Hr*tXqTM%l&gYVHlAAN_^!v;Il$Y(e=F|^ZP}$5&+b2sz{!2GQArIjj6Z}b} zxr-os-9`D~c%jB}m@x%mNr%m5CxS$q@%j@TbRy7d`gkO|DKtd>`d=AzOea)5H2HF)fu zpm?Do=mUHZzyuxjE&@KiFBm}hcJ5qJk!Wspw*+-*nssvo#q!mNk@dEjoE>53y+Q97 zqtmnJ8ZP_9s(KEUM2y+ZtA(i>A1!x>{h7fWg;>|!+y?sFlh09Z7PXlTfy=EEEIam? zUfcdCL)w8+)nG*@5E7R+{EM>cnPV$HLqR7}C5!@W%^~BW!&(nZ4EL*7xrD36WHT6< zU-ooGei4tqj*8QH_1v1;R^@TF1K&+dA~@K4HL6FtvTLF2K9;kqNYkZm)8=K~rh6ogMhM>x7CRFv@PrFEPya>LF9Lpacd$kE{*XyG93h_svg&dU19cFjV;fP-(iYR&3E zP>9Le=(=VfT!+|RTzQ*y`)o?Z4~{j&frF0mGZ|&PeXwpEH@hKBw%K8}X&iSFvwt<8 z&5f~l&e^MPY-D9*42z7EmX=no7E{pB07@!A>OZ!9p=mGhL!1Rj=m*O?T3K1870t41 zIw9~&OJ+_@GVy5i=j|T3G1!mzZ$L`HYmR4fQ2`8=r%QqhwG2I%3s(So>cXZV%h|<3L1Q$+$wi_{UMgT#Rqv4y>|rr8>4&Cb zd~}hWtm~xX?U933mG=)W4|EP4-?@^V3~`@E)t7C&O%)}xGWM*1H8gbbe0_cOyDl}| zs>V0M2>J8@JJ_ayW%bO1=S)%=ip(lsU+z+NN){U>JJ)6JLpSL}XUo13kst7xu0CCy z?XIo7=<8blTUg6WbX4-O3QJT$>C+R1mzqxiQlEP#qog`}FaM?PT-QhWYlr=o34W_{ zQ{o4k>%wIgZI!>XoPlqc2-!O{rfa3SJ?M`Ld5st?Wg`X~yr?7O9h`+RLnHTis@K+J zeh{*YTc5&TfRp=ky!<8Lihb!?3al~E>HJglzo@ipL65o#Mjgis@HzV`=4}ScNKk&( zh^Re0>D>tjes!TN58Ap=39kfuvjVB;WL`Ag8{0}_%skc zQ3FsCaJl_t6ck4PV-tV}Syk3;$Ruzh-7Jk=T$K2pw(o{^{qW!*)F$MEWnMu+fhZ9z zJv~4Bcx#l(gMoHaMo6IxDpQ&2{tjLCJEHVj8J*|{s%q?5+C-zp(Io?S`roCQiYx@U9HjL@XDDQ7i?EV z01T!_U@(n^MMeGZ(f{=?v?5IR?*@H66*`)N^Gfj~RdPF(%pjQuD()&=c=<-9r3d4u zc?gvd(BFsr-27h}%K!bR&9cg8ak}?8$eG!COaYWEbxw?rH3R;@CQZ#5sx>L0On0xd zV(pC>-i?XB3x29d+lf$0R*@6Rk*G z`-+S?JM3iHY*L*&5W7v0tho$#wKJ&T&dX)CU?uBl%3MaZMd1$dw%uaja#&Tgf#hoc z8`~d*i1EHZ=adb$d+ykmlxn<0YnSiC>o$~?H8e+_@>tA<|Cp;_myaVg`($Z% z)k456#q4AhpvL(z(ZzNAA}|hbp^IW`esVAF@h&eSTJD_L)hjI=22>&|SH34@uAzAS zGGi#TLe(n%0}o%1#I=^)zrpGFPItIkaZ-S3fhRZmTaeKJU6$+8=s9v(QERZLmrBmm zn2om%ncB=33@VWbc)e8L<1NBOVSNg8(JRd?YYl_4ZCPRt!XB5Cr2#o1)k&^TOt5kl z|M#N~dVXfqy{yYgutfyol6aZUbjiu=O+sg%^ZD8xNQ2pV&cS(QkFL_%jY0-y21{E3 zv7#^JJ{WP1-+1i}=@o>zC!xjW&!yBLidSc1PwQ7XAD{`a*;edCc=N?0qN+b|bK}H0 z=e1X)SsY;=l!=v;4Mi|*pJlBRqwfR{3NU}{h`;yuXPr}$PiE-#s=yr3X(&=~&Rcm=`7xSZ!gb9?OjagEKbinpzl;2=x(;iKcbC_Rh{9<1AV&%jM31 zAg=D~GxK(*oQr2kU2dPd#bUpCOPxheOJfX(5TJt$Vi`oGc zU_XkUe=ZE2!k9_J-rnBkfF^ku5F+#%UZgM@>ADRjy1fx_-^2{ZC^|`DTGL9!yk@^N zJ9=WgFa?bx|{dnJ2c=f2|=uZcSYs~km$2sgZnjf0F zXMFgI+MoZn=i|v_;{+1(xWeD3qb8c{&(4%l-=D7&S5L*lr@-do#9;#Is_UdDsHa$= zTLc9ah6h@8orl%pFj|!ac)nbuE%bB|44vH{n(-n@e z0FV9X%P}3+P#Ik3OnZkOv7Q#=dysshy{X&5YI~d{W>hERc+^3JenK-%U{9DJ_fucg4gbBY>!E&tG=~^X9JysO8Iz9k9nmS7b*j zKe8a5w0pujGwtzu@bl>%lb8Y&2j}&6o-iPQdXi0D(TR24omou3;54C13hu4h_Getp zs%TaK?*f|~wFY_A`tVrbDVG-e>UhJ0{I{7NkF&-zF^{cGrTOeig&SXJ@lx=&9WYijhNAM0u`jo0#F{$ef4fB+!IM9$}Yo9Z*_YJgNmDrY+ z=X97thor_zW?WdX0a`}G(z7$z!Hw`g`33MwL?J}Fp40Lgr_hC$YQIjrOK8olFQeJr zTlACDm8HtR(b2Se$bJ{laPq4)dz|5s?!~G-nGY6Zd1-s1ez`qUD86X${ z69ll(fq^B$W-cogYp*cCGzxM68!}^nK;i9T*iEB&vGnav>PMgM>1>L&huMc^Qk6 zf<>txvmU4HNn5v{!dwoskeo(TbREQ_UC)Bf?EA^tF45^a0h>Zh+J=*!gIjPeuXKw3Oaw$tI^I!khFCg)ZtZ&Q;w zk{jjBS{m|$wvNJXfYH-<@saB_zIl;gGIj#oJL4DHb%AsqrF?8F1(1Lw_q#{UD+svY zh1kG2E;;j(=-@r_laiAWo}tN4oU5dYvN&9cxEK)dr*nA4BbY$%g=}ZYt2JMa}|GM zCK^`B`+(VOtL*duU8k^FedS{mYgaRO5p;E<>!~Ivh0n*cK1)SMXLQp(txn-oVynX2 zof@xwpBIo``d1!vo^9Kj`G$L!T;EwP1+Be<6tuUm;c_P8^S;j&24Zo~687iA4e;Ap z4QnR8%Jou3>9X4%Obz#$p=8k_6BV!ie4UHGC3;xW_>nwZQ?9cWaQHkLgm9ICBf#*G z_IZc@Dl)CSeE&I79SV=o%=*D7PUZGY16@^ji0^T=&Z&nT8t388(Ns5-rpt#^lvDm{ zYL^BRbaQYN^=awx_Qo7q=1&Jy>ji!}s8U%T5i7rwMK6^IpMJgp%&Ur|Q9?1iY`?jd(8qB>WKcZcGnCx1d!{ETZ75fM2RvQt4lb|I zP%cH$}mYL)gP5aZ+T@4pmfHlEet3G6V$>EzC{ z79r;m*NjR)Czh(GDFL-FQ6i$2m`Kg3nXwnB-I#j?SP}{q5Ah??AwcS%WM3y^U+xqE z6INlkm_RBOf3g2}7&J2>ePc7bLNtrx%<-lmYvkWOTJt6)M zY)r;-Fuy20UC`7?ls;Bkk+)fRZ0A(inil(sI=36h8qTdsfC?+zW#N&{z^8dYIH*e~ z;L#i}&g|ZP{YZAIJvvS#BRf<)BI4pEH3nM>?m(_orh*XY=~a&C%*rf(S6N0rgXcfP z$cm~R#rYJ2;k{B(*j`*Od2I2J)F9D=S4wJQYEcOLR#>Fwl4O6WIEAq{=Gx*=dN`Z; z^&}&q!7?`!bIh-j+5N%r&`aySY{5yZhbe(YW?H_PduM2yQ50Rw&7}F}rMVn~`~Px6 z1x`Ai#xqZFKZF^6N|0<*ni!K>bQwMAg^Tp4oz85m*g)OY--88xG7#LF4IR5 zg%rK75cb-TUg;H~5-mQQpE6wP({^zecpt8Vs8P*y?8r;tRQG@CdLluOIqMoPKsl__ z4```Ttif@6T~>!$rp+{*?iV9lBp%iRgP;l7#gqf8yZ(mwq;%Z4#v=`lkQPbxwDy5* zO!cJ8h-l~jKx#T}in3!YNeJ=)-UC|3+!7A&rPE)(elbU}42UC{3Y0SYlo#(LMV0V5 zd!WR^utu_G!8!*+5)-sNsDmjoNl*WPFsevd?OioO^UYtz>^JpMQMVqj@y0=m+=NoP7sfR{qk@d`QvvpxK zAUi2Z#m&edJwJ69`41ypPCbU_VB4xH1kYos? z?FmIB+AuRs%+zWkX+(0+cil5YxJa0c12peL^l>o-$6~tsi09KS-Bn?BP_YaRA0yL?W@`qfg{~HIVg)-X|;3@_Uxwp$f01audV%YLH3}f$tKp5 z4e`L(ZjBsYUUc5L0t#a1t`+bzR&~1DmfEK>YnnFV^*~S9-`?~?Uxm}FmXl(8+E-c& z`{7DyAN-tE<@x}=ANdkeLr14%9J-}F_%~E`>F1t3dnkoXOJP2;s8&r9ZeStah36|( zKIg7Xy^~hZNb7@yhs%}a*_8L?f|_%dV6*T!o+D)Ruco~V6(7eGV)}$wX{>e z;ugO)x_Y@NNc_YBZQV~6O|N9#&M)6{Ac5_kj>R&&Qm3XpXrT5T1->4L^5~NyRfDScuPKam+c&$*J~@YjL?B^X;9h{wEMSPWbbZE`382B^3G!(YxDAwxdx# zw`&zte!Yc8nh`{7j0j6S`J$o0Bq%GK^lZ9X>NV|#^R-U*JVt3+=cW`S z5Pq}tbc>!O2-iBUJ9?{L#x*4$m1u>-s=Z7Gfp+J@p6i$Y>J9B&DolihsJd7w7KO?3!oumB!{p`aR2Y}zloR4sd zMju|xL^j22NYD(U}=J zR(4FNyTSS3ar?USgZwzi4^R$!P^2h7U+@BvGv*DYV=y}a*faeb?h+`&=Ipv}#smS* zm+kI1*=?h`{QQ1ZRklJY@8zmFyx!Y8JD+l|nlX>HI4*l8mm{b;*FLOOg;rgHD?5oG zv(7u#o=mF}b27}yUUR^6e$Xl|giUFNn(r;kOKgnMcC{By8O`1%Ab09KU2PT!#uvO} zoZW>yJ7aphrFgBrT&-#7EarLNvDr@MV+4PCTU%DR@z-w7Go|vM_c_$dKok947P%y5 z{JIZ12~z0a7BX_DxOfpC{2r~~{6pf{^O<9l$H{!rC9SP4EVYQ7#$ff{!mc`A;S0}T zR@;Ff%lde~l|qtoVd17vl?2!QAwtK+%tn@jEE$ie{cEMBxKxI}_IR2cnLW6|=Ir9p z46CDY=p2Jc&qQQdbIQxxYqRTI%sMqUt-3G?$r>Il*ct(aAe(T5?>YUtS`~ zri7|}Wuf49P_nPLbuVfd`&{6UOHJ^1TxqPIL?GtBcYiqFZF{#wyVCI>LFQ$3^eoT9 zOE9l{Q7^f@d(*We(DKpkZ)P1RC;{iZ{+7EMkX)VKrB{1}pe_;r`+~Ojo zA2%7D&t0&6SudYJz)+xwL5x!NjxCRo_}jOyPNb7FP}$0*b9g%_+IYklfinS#s$$^P zJ+{+#+Z-SpgZ{(bM&LC^#0PX+ii(oTdmP zd^I6fA$T~gf3N^=uD$201Ddj;#UmWxXCB_yU3edC{k4rrkLzOa49ktQ+9j?ZlQ{Ud zUm!f}C$&MU7>KMjzp)P`U0LIdr?ccGA+));v?f0D56!E;h<`!Ag8MQAW*+i}j*IyY zB=`@9W6U()JUhDO zU|Oeha=WBRuF6x;j2#YZY6h;&?Lp-+i!z*Kn$?Lx9;6!U2aEY03w`e8W6`R933>GJ z@OrzTTD(2ivx_;)z-}0EIwOS{uWocMk3?9@SBh96rsPnJk?h%10#q4Xl!f*;_#z`+4z%O zj$>k9_62m|YRy%8zo&?vts_k0UM|Z)=ww)7aMBsf4oc<==z_J_X)2Eu6pKqp7}Olw zss{*$>C>aF$W$;Sh^!L(;9{{+BrRp?17`AZ9d8P5~gtUU*J4-d`=k@SpX5N-R5 z!%a34oobE*y`5!~@N(Q= zGUrUll1u&c^lV)OCF5y(`0=J3PUyxtl%~I2QzzceemOI51#^jOU9gm5v zI$Gyqb{*d_zYLFuHVdFWKla7+@IgLZ(K8cCDdzb;GSz{dHb#H0H)>8p6SKh}BJLw9 z4+}D_Y|O$xc2k>B4u#LYGRWEMTJzrCQYvFy#7IQ6GcXOz&|_nUKXJ2y@Qa8FvaaSA zBz^)JUF$~)(nlxy?;y(Th?xGd+Ooi+$$r>H;0mmT?V*zA%UgJc<-7?KPE#qitL%V| zPl}RyO^ihIL?}<`HejrTu$H_tJL+M$TZYs6D&v!giP2#VMB;S6;WUUV!!HSqZyknC z&OXm?OA5y$Z^yTQwj&_kP~>8rcoL+0hXEG09~1LvaF7(ZKe@4I;uo>7U>)jT_XBqD zwB9I4$CQcop8Q|ut)D*cT>A_jGNQw0HW1OYDJH8jZ}yCArwj{_W9n#8X3@w9mk?2L zso`|;0Yb9>i7iiK&r7Jz>xKJ2f}q9aQY0yW#~iRzInr7`xho#7pJKr>#pKPJ#@M4G zv-2Yg{%CpRPR{cuCL+=n6HCv@iTS_!3@~dA#&5ygH2FiPmB+cle~1N{^Bh!BNLtH54rPKA^X2l4)SHt>1CgDSL{&D%fc0ue_YV} zk2bsUPCAW0KN*f=@I$03gH}a0Yj2&9b@u>MA2bBN9W6foKDqPfW%89!FNtHVZ@$YJ z7a3bFEnzYM_cQ(l5k-_#L@~Yg|C22Fp`HrDGS5Q2l~C#i8Z#AgT!y8k@=u2AFZ|y0 zO?Rvs+&)5G2E&6jzbBho{vgBM1UQU-=>hQ*BT`KbWU1K{8_0@W>hF z1dK;<5G)`qZVDy}LQ#{31tR>LO#z<6aRDBL^Jt<%fhLU&hO;qI=YT&y;NGCDX3k~8 zTbokiOImNKlyNN{D|m?HmSeP;tdEN?7Mpb1pQlljDdm0gy8f=6|NFQxPi$oQh75MW zYYTET)mYM%#3iStC4^RV60$>B1hxE3IlO14H~Zg#+4#x$>iJ58pEf{T_f8*Dh$^M0 z2CG|Z^Z24t{=XRzlW#)mit<0LY_es_QCO6rje&qdKbzfUC298}By_UB%EA6e-Mslz zEAtQ-(@ue;cY)xrK;{b?HxCizyR+10i=f`g-rJ~@B<8dCWpi7-SWuAtk$<%9&)6!7 zm4DAvr%sk4{R%wW3kPSTeh34?a5Vz_(AnV4tRAqSr|A7=#Qg7TZu3>*vYrs<}>@|`&G1(&xyDr9vmcBp?AS410AW@tv!YDYWR1Hq_4Qr zr|DYkjJ7Fl*WBi8sL7-j`7T@a`Hxk1-W%V{_6JeMpq+n@X#hKTOW+>QK3mHbvheOh zuc3lnIP>TCPaY?+dzFP~H`P2Q8cE89)ALndQ94&-_1)trF1XzWda`1P{N##FPjboW zb=&USjdPQ|7WL+1@ION~4G~;?)btjxbokd>{ZIxzp*>)R&-?svqSX&AY|`tfGS`0T_Wqd zIx1rYvt~s-0tE)u80}Af-xDN6al{k+;5u)zo%(k=_+n-DBe9lvaBzxOrEOqylMhJO z6O}APn-T&5fOMB@abGHb^i3jKRv$%%Gh(q%L`58p&Pb#qm_ss{h#sC1!-U6=1Nv`Y zfXIwOn*oX}S`? z9f-?M9^?Q9OqMP; zdwMp_?l5=NKRwXtG(VA-H!FyL0hy(=8`Tp#U&W!fMN0BKs^VGvnv-)OVc{MPcU~;p z*urK@#Xcra0xyHZo*=B@guOgFAsqOt;=yF6pcSe^+-&(JrrN2~{`RuyYhDwKsOM4V zi|%qmb=zNuFXc3UQNE)uVQm8w7F1s)%Aq>>g%PMp1N|)ASzs%Q->w89`@I5asVMRV zBJ-F8ZPUw$85kLh0t$@HEV_e6dn+nJ)(~=fJ6KI*^gFIW_+>w zGLDt!tHKBnnB3SPyhEaFvU>aERNE9^Nn!Rij7eLASP{v9#{T9SrM05^M=*k=63VC&Kuz)$Er6HuVq=}UIQv@{zmASf->OvRrUVOBLf<`ZhbBRdi$fDSl5y$7 zg=o%OTL3C|cc(j(M0o-o4hOPOdW##o1w9mq-U*iD#<`z+LE?Z&QGNGX->gF2IRZY5 zbp1%<>$Qq8FX)sn*4(8ELKhYhrg=ZttX z7q-8gn*~!Uzv^zu9&7U@c4?j|rT1)*9IymNjoWmtfGw|0ae2h{SIpvI!t<~2^SfGn z$-+l2wG3oXNsX*2;cW>TJkw#T!@nRu`>9YWg8+&%DnHqC9-ApNr7dWUXCivk9YnS# z-Z~U_MugPSVuN%`<+z?`2xmm@85MW=W-6lmlYr1Q+Ki+oa#B6$u}S& zn2B$HopO&aAlR*s`>8q-E@74&q{397q--A-X_9Upwy}0;_DG^syN#57cYOh_GsH!7 zH~B^g`;x7)tlu=cH#cj^(w_9weP_P#Yd+mN6{0H+OK}^)SCHI5SYO-WqFn$?P=~eS zXIA5d0)RuZ@#Z_vDaVCHcl)mdN%1%-k>jQ2wiZajgHlnsjgqGMH5g*3JX!2+m9#BR9HK4hX8;ssP9PVGxTTEW4xf_2=qXW5{V=$Ta zY37oD!thn^zi^HFCv;|*4gn!T5kGZ0hiCK`h83o?qCEO7o4Oa{!GN{kAJEs|AAX~$ z6qiSVt7j$Y=*WISOh*dyi{^EC4!71zanXK;i(GRtds%s%{D$icEb#Cs3=PWOi{SC) z!U$E;$f&HcWWwiKXID~K>w8s&LPI>B4f7}E)I_Z^Yf}nBkX%9*h77~=yv@iFJxcYw zND~b+bJoj)c1jwnvAjbvQ(A@9lLFb+EkWN5uai<6R#frL^ zT4IXp;wfH|<4+h8c?b(s-5jdxwd2KFRvqy*qsx}`Q(#q=3%rH1@x2Na1clBS* z+y~F77SBocP{u6D!Bs14XMZ)dNv@M1V0l9U?Y1tTPP-L1N4rciBJ*+pr$!f zpRCZ2yLow0kWS``_xAEqFofs5;9Uae;HCwqY%Vc34?NH1XrzoHnQM5wHc*lfR2^nN zF{Uf{w(ch!0i-6}P2w7HchKnT&XbhQX@$L;aWTVk*K*P-k`Y6fgjG+Kpsgz0y1H~* zQQu?x$@iV=N3NSfh76Bq^KorX^lqGVno%X?fX&1~G= z9%Z3oWQ;b$O~%wgcEm{0c=0V)zPR+kiJN&LL&@3Qya>sUc4D}SlsO2eMM|agoF-c? zP^hzlZCQlv{?%EM(@nzGF_@CbYqf{#^TqF9-;f03>>O^wYGrmrTO&eKnvC{3`mp;F z*G1ibx&B<29;H#PB`!Ym_H<9p&ejzC80ph`l=gn)*JYCDL zCvH@{`Ihnv@lOvVB{|jcDr`!)c3CO0ZFyf^$9;^hzj??Ee3)KLd?{%DDSY8s?x8H% zq2y|rqVW<=%#U9KS*>nOe+Ayp8a85tM8~8)$dtC%F`k|>B6Y3P`Lvx9c{UYoz1@Em z@CE#W^#!h$pUP~Lq~1ySu5mL6{--8LlPqZB<*KXe%*As z-?6ZbRl&Z#W_^8qZ=mXvW>9+Kd%VHDeF=g4#%qT!Y-wIT4kQhdfX-oL)~LJ4A__bn2a71f2-;jx zQFXQGiduPcD+&WHe(;BuLt(Tz=i}v@Eyj=-SL{xQ2c~$#B;C9ykfS~QJ9jQRRnHiA zY1CTzsnE6hD(~)ePlZ-&ZHHqeTS1{=GQDi4ILbLIn;r7R&(AG6J*2isYmG{c9RpZB zm&@D&KrL_5NEg)VseP#rO1^)`lxzrITJg5UJM9q%8%QFd{IHW;FH(H5oa^{k(Q_?r9B_A?h723R8yV*7;H;%&I zs-c@QSV9N*nWc+DLJ8r%wa$SCFbI;fPcNUa(T>j*~>RMfglS}mO zqDDCw^r7Gd)w46ru>CGbm?7Iu?^#2+v#_@mMTP{0j2|;y(Arf=)toCAIS3-JV|%~} zGWd^zze=MMj8ngi;aPj%=kS`jz4A4tUYvrKwmaSE`nI~C!aMJG@V#twD4vzcXgn8~ zCchu1pFM|Pd@B!q709?)y$XDBiF^|=uftotIyL;Q3}P1nr4dpgfN3~45fLINvapiH z^)hw#l1bxXrsxljC^k#>k}-jer-f8Chf(+wd$^7}Z?@QHHj{OqmVpoux+A!FG-eOp zw_E%r&)+e`!g??qm#Zuu$g`lN^TYGtt^gMHZ7hMBS+M_?+pTbw$JIUAIp$bEB5~Z} z{haF^JC&41B!Ydl6$qmAuhtjIdJKX@D>U$xd+l@FOIqVONP^vqvP#3fC-_s0DfGl$ zHLRC7MnEi%cnhHN{V}C4eC-9#H}%SBkF$kzCGb0nX$H%aR(uOM}WN%3A1HC}!}V&{p*y(>ll!f1qZ z=c~ThhaW_mLE)jmG4)|pCANx@_2i<#=}bRN%cg@Ldim!)Zy_>wc zxW0-P49^$K?WMJO`2G+``}%KuD}tySb&OnEJYmMIC`F)_(834^(ux>RM7ZnW>;+@d z2!g$Ksxd=yv85@6;=2cGCH&)<<&8?|W&k;f_{sH`m0$0jhy-*|>0d>H*r{ZV&`9h# zadxm7isUR4>zSJ#>?*K_FfrYgy0-)qdE*mMvflF(RF&lOnIxGZ^P01wIhY5KhlR-@Zh zd^e9iUAp{`Dvux_dOnD|oGZZrSP>*TymY%o^PFeXEgOEudxR0G~57GD`XJ z5qO1*V-1NDMh3%s+!jQQjzrvd;I~mk3<1VY!i9dTxWa0X;Gl)jIZ8=tYT4|xfrlh( z>$s|d_9PM4gjxRf6*^=?LeFXM_4t?6baGJR4IwRDnlGPMYOi0fD91(}ER1A(diBer{Bl&ykp;Esww*Hg(61~Rk~#yOFj z{R_mVwS~m3VW}O#_B14BTYkQ|;TCm!wg>U(Le?l&vt zC6zyecC9Ao@c{vEcd%X#B3I)&E2wVC$cILvUgJ8^jzqAAsaie<*tj)B*3lHt#>S`# znWMs!wzxIHpfAg8zTT|B6Ba2pka&Wb)Og(<3HIFB{A=6?efA#S#5))X>X}7P^?N(2 ztG3f^SbCgI}&#>7bZSj};vWAocqk8HC8H##=&ra0>v8 zp@lPLO8gAS0ig*yuL+Ssf~=D=hfYFnsRAdEVoE{#VYiV}CXR+@!n0@$AuS?(!jKd@ z=xc42ZX}ztPVfCA&vg*Ia^5>9k{H}yERoR1o@!juC7viMDIh_AF-t#2(>j>tvp=?i0*VA@SQ*+2|{kXoD5zX(h)rVrs`@{yLx0RM|=$+%~gTnkF8N*o&xs<%iw$v_bX%ZrvMK8_T*#Y3u!GmbalO14uU^w`5N#Zh97 zWlCtL%e^9f9&-4cghNu-t1C`agzzO{6HQQYDt>b2NhsCJj!CkHB3oe{mG1>hP0`H;A8 zE--B+J;2W#)A-~NT5W;8ISfBCb3Ka)dhA+QBc3!)&#(o-m|ip|Q{^PmqEu0qo0DCA zBj&F337GdkdAA`f`(ZanFFJy;t&T0f$Hbuak2#ur{Y6g3D|LlWD!$3OL8@m$>6`BJ zA1A!k6wMjRIq@TH1m1*ga}B9o!%8QV(bI9tI`Z@1;);b)0s-mv0z8Up(6ZjPvLx3A zafl9hxgW8dY><63;oVrwwA&Plkj4OOi#g+nO6F@_$&Zyh-@+NW7?4zx9366`MM-d< zms}9Qz7S?mU-^;G&Bw=nnur-cj>V~kO{!;YAZjsshxZs3vFDZW6{$+qBviS$0!a}K zxYXqT__f)kI6H5xnP$|FL`!gqLnr4Uq=ebDYLTTCP6mrN)e=iOWG9`X+%vceTCpg0 z#|;||R~JCo$K6+WpO}Jg6MZM0ygXDxTR4Kds;UI9aOt|pTI(=2Iw3hUiVwB<_|$_| zG8+?%g_x4`u%o6u7sJ>Sryo7cB(%85p)uEN&w-%uP9IQzT z4;slKQo3VYBT>KrMkfO`Kh2nTVe<4#?ulhnlE;$j2uK;h{#=m8tL&HcX)bp)k zf`=>*<8R@Ge)egC0MUCVXnw|?T)caBYT1i$Jdkb0Bq&@cJMGIPEg@gDVoY=Tc0w;f zMJ~aZ7RlLg>vjs;Wo{!E$`>4i`XQ}?SZp5Y6-rcV7 zXD3}~0W-}$J)3W8|EBNOWlx|fSLO}`XfR!qhPakR*|(|w^sQdew>ez%;zm+L&`-bo z{lAH6sC*8BBw%Ns=J8XgQ$R&x5x1hoOFvY#<4MUZ;zK|mdaj;ipSVsp2uBhm2ng`! zH@XK8re%rIOCaGvlgxPE{Ex!76&8%ydH+eMWq^D@z{OFK$+qtlkGbZ+vEL zA|#?SG%;V*)VKL%M#oA=%>HgQ!2|KH=)f(2NZV9f1XtUgJt|(X1Ot98UpDk*+C>tQ z6V#@b%k$`kPtGdFNKpvT_f5{=3T2F@8uBD|)MWVMO*wpv=t)LBHRdED;*`S8_vtE@ zz_2*rmrRDAv1e1wDo3kHF^g-vp6(cW}6yx3QyB;Lk@>!NSAz1L#sF&$0P zX|=UZ^YiGuYh;Og^o&zxi$#O9wlnExUx#Q9w%DhXNh-Nd1k+Ey_en}|w9#gknV((I zjOxv?>*1N%m^ovjS&w33R~(X%D@xQJ3*MyU;Las$O}TVXZF0MF ztUxxeB@9QJ4}5ge!?tJ(3?3jkLSE>mY`{vOl1cH4Ow8D6UrOEOY`Jix<1-jjwVkIf0;|YWz5k);- z0!GGWNGutLYq;gnFbs+@9!}o`epj~hlS#kzMKA+Y0aOx``dyBff3N_5yf>%RFZu5PM$-PVp!0jUq5L3K;mf#=B1OjpcM~DyrO1HW?C^IR} zApX4C{(E_SnKgG*i!s~ozN8}6z8CX}eL>{=kGHk3`@@OrH5y|P@lbZoNj$y{y!1!C z=A(Xk!B81pUA1&z84q$&-##MVy&Aez+DtP6iAs(Rud(Z*CuB*=5d{_Ab}&31$I>k& z+fWtk_}XMXg%)xlHkUETl!&za8T^0=&eFmpM!NM@61Q8a*&;_CAAbdl)WUYpg`aAP zWwCBh0Hu}(tDm=crIl~n{EAv$q9e5=X=H6b`zM>B;jkqxDzRQqwZ!hi&;k&$+DV({ z7>jG_mEkvKad|e^oQm0mhv-LuH_)x2S7uVgOTR|ilGr@#td5gtyOIw*769n^TE8KL zS82pzXoFEi$dAQM9YGTkAN-M1@=vOEhr`1WQL*oNvJ~1tnp%r!9KRj3icO=pTQ$R* z7R3%x$&t~Qh0=65FT2L4esgIF`Bu0Hj&T|c(%#k?h{ml1GD{8YDJL+Bq=m#*y|-eZ zrd{DAFvY)6c|r`4;XT zQOBt1dZHC^SxFY1V{L|aJzzw=SzE*?0=DpX2l#)qwXGV|_PCJ{IZ55A9QYO-%{wjs|!? zwZj2B(9H74z7WP7yxQ6GBt5w5eLfwRfbJ>(93J9QF%|Y{2xK2k+2AO6sPpC45OrVk zc{Q5RcWidb%g|Dppb!X-{?q(cBTwqdfPDVI7QS))I}m=u#^K`a^ziY+P-*m)T2jb9RI8RAoKv$J{XtipYBF~#>0;rp20ax&eyqrGUE zlc)B+XxsU6-geRXvhcBClLt%^kB#reysb}toE%J57R7uFMT`0BpeNdi=T+5kZS~;2 zZKr&^PONpZy*um_0M#l0`9BNp|2*C<#16Y5dfJS6wb9yhD+rzU`LN(@wQ2MABq&|? z74Ku+>gl~l=%axXjsT?s^S$pxdT}#FqSVW5wXtSTeVAM@baM4Q=HoTc2|BL?kM&{i zW1hpjmYZ%Jb78ujCG1J6ef*v~0mo#uwqW!=~7->$kFC|{x>|NH$xx^Y3AUqZJX zTJBGm4OuhTY&X2V(uxJc`N~tF@V%Q-ZJU(#?iQh<&CMo!jq7$S9a7z~Szkkw>FxZ@L=n{nP6f zAHCpAI@69?9Y^q-8po$Kg#YQ4A6LL?{@~L>36KLG zxn7Zif7tAKVyqKsqrU*U$+#%=|H~~1S1}<$Z#~{<#MzL=7K2uJ*#tu+$}7I3(w?uc zBBkmGX>>#*fxmzkB@!xzEvu*NorN*M{nGBZqx-m5!*@Pw6~t^N_@9n}xPmr`G%8jt z@?G^KH~kg_tlnf@58ab@-PjLFVk?imUyE7ler$AcUtS(Q;L$4F=REe=Y6h5H%@66_ zgqMB3?(%t>bL=I0YVH!b8Kx^%pQqp7us!|$wqu*KlJDlS+UwSGV9-F*y2dg0bY+{Z zYE<6dQS$LPgJ%H~bNc>^lEm%**&*;BCvYC7u&FnxhT@Ue&bW}<$LCq&mj~>e|wbth#C5rOY}Nn(*^!5YH8M8 zU)ejZc8K* zv?J6*Bx^gKYdbT*b9d2ktB%f~!i(4f1{FA@<9m{Ce#4|YhL#5!RmYh zY2r{DQGp>!3f-BY#&wzFcmOh zejQcCG(PVv{)*1`mY~~k(s^n}%RXG=f@G+!kcOM3Gsv8P$yUd{eZpP)P)9nd&{|Kf z5{xJq{-{(0z+fpLhxmK6*XA)zZF1j-&+DpSB00gN4c@PzrK5@z`J<8&iHh?6_r*! zBUO!VR+ISd4j-WZ1dWPlR6#R8;*I`c0F6s&z0EJ;TgGi;DvNXq02k$u}zdOrF^}L+65}!hF;dRrFw8G%^2V zcpdKP)0oWir|z2xL$;~Ri79TiABoK4a#KC3_gYTT^Sg?J##}jCnDZY&OX{?tcaiNE zYhooQoMKu5qnVm5KGy19yVfoT1FpIpr#g)9yf1Q0{fOCOGgz z?sC)xU6XD5gdVX*6Xv+0{iD=1hJ?iP9ydCgy zDz9>!64runBTo z5{(LbkHZ#cfaT1dLlFv}dK2;Dlqk2lZ6Gf+XkeEY5)YEck+Y$*0Xdg|&5sMz1IW|N z#0e~qii`}bT@2Ft1%#gvXXw}M9UecJz~##uc;FVe%n6=NnT-)kON4}Nms8z@NkmLc z*SXlA-2Us=l<={Y)ScQORVps!rh)<(Kum|7ghhl`OA*jT{~$(b0ImE zfc}Mfr&FnHatFRMH5o~y7Vj**)Mq$NDV3*qksx!*pOtwv1Ll+?OcK+mHBG9Ex1f?nOl+%i>tBn?=FFyy}wy zN+TR#vWr_yn_Tl^+c%QvyTul7Zz(6oTw%rn&r4rAQ)7GHpA#lrMu$9814ty%_Q?Rx zSjSipDDGbs=79bf2C)0w?F;i}hfN~=-R+-*<;tT2C?S%K1lB&omx6>O*F)jAOW z?F1L&iZn?DHN7Ab0jq+`e)%VtqB@Y#E4)8k?eRb4~g z++M(?Rb2nVro!@`G|BkHoSBXH=#2CBzVn6%TGHHI-RARgWe4;2_PW&asF&d*(s`ro zrbm~FTDC&1o4!xk;DXZTQGFflO>dGr3=12w=`$`dP=m`f0%XHiw$b>T8^6-@={S4H z4=(Y5mOJgc)ybfaI84gSZ--J*;@}w|Y~s=Hq3#zAxIb%Xrb8HUkT{Zjxqi^$UKXt_ zhdxR!j%#k8nZq!!=FsH&L8Y#aLMIl=zB-w6k>^M?HcJ$=(7&MLou`_nsh(UG-TB=t z_gt7MM@4L(GZXv!0UxAZt509O zk6(?8D2}m5IKK9oZg}l{#%4CxaXhJT(pEYtyyNP+j-$MU66@y*RXxH;xsG z`1_H{nv~Mag5)IP2j|{rX1<%Du_>fJu>Pj*0%=(p)(eDU)HJH5yTr%jw$wpp8ln?X zF#nAYILT9w7uH!-+??O3pcmH!r)0J}m#dsmt1%Q)j{i&m82##e(e6{9m*?vqAC!zw z{Qrj^FgDnl5e9^I6yJ;+YwvFtp)NY{BC4`jk82wM!+%3TKW9^RV8EY@ENi{|AeS62 zqDS2@1tF7&V^tqm?{AmBBLyM~ev5YAVJ9MrVT4CdcPr6U8qT<&-*Mwu2x8_4KAA6- zse+(b$lTBut_*wCZg8&P{_w}uydI=)b9M6Gt`YMufb zfIu0!`f)AxJq3LvKNx%xuWfgl&g=H7VasB>cJ1p$rwb$|dgOt+DVKe}ODuDnX{p6k z7?DYYW&oq7xx2yQ5#7h_pxIS^ZG3OVBZkLye_B=<$#;BSclCdO3|o(}=qFxJebMiq z;28(T!)AlI%rGL0cQKf3F7WJ#5&}%mz8n4uktMIDCYz=h{IRMST6Zm+ZQg$Txqs5( zQ-xvi>n}fcj!EAv+|ad2?(uG7%k{!PafqRpd1ZA`)qNg6sxFbr1QCN4UT6 z1gdW`*ghxRTOO7#6a4c4Q+IzxZD%x~HWSL~s$t=^5RBEY9IBuJlbfKFyk(|lp{8fT zSr&4VDA&bYQgiMVz}YR;+?OLw5Z+(`#Tmoi5`66(zgLhRA7A;gu&lrLy5AntxV0QE}HaqM)&q)|jZ6IKo2ciru zi`}5HnPoUZ#mZC4^LS}Fc!g*kUDk|gbkSBL4pTCZL;g;RbW$lC$t{_zw!SuK=StwG z${0iP$PtHw2Fk8F*Nh1^N=>j)mfRv6hFUJX>ow$*;kJP~-U8#V33D7Px2(Q$q<(_z zJ*X^J=RiYX9@k%{#U%Ih)U|M&dFC}L-XFLDloQU{I^rD(f1BG-&SnPX(PS74)N%_H zWKe4>9HF|$eZZ?Zr?W1fXqadV$IahsXFsmbf9ZqmPD(I?C4%5p*cHRBlaTGhcTw|R zFG!2*##XfWIu6x9V8&#T?Q#TlfJyD1zCSuL_z>{;BPAY;%a*Lx$$QtHryB^d8lfMQ z!YLZod6*zZAu=y5xSaI`FMwJUQUDi?K&+{0$CrevvSstc1U(65Azqs5d^)L1nE7mf z1fC=;@+c~H`aQ!id_#ADxP(ywRpcMA!Oefb25&}u&`oE|T2})gi2~<0w z345<_**HrsgoCD!w7!wU-9aZ@J zKk|oKh#aAwobg(@zM|3|x&sgF!N`f%R*J{QGtZ;<%z@DmW6!5#mMy&TX3fC1#OzCj z^f~#FMgBGa7&mOyCPxXCb2W^pG4Sy;`>S$<3=X2^8fW6w*YkCi_V^2PPTy^#x8*N* zKRU1ONML~fG4(r}J}G~2Td3XLpwZ2ml`fp=aL~!BkQ;&)6y?ClJclw zMJiU>aLUg^7B^BUt0c6^H-|E-Tu^0lIFBn`dUT|_YD6#sJ)3g~Ru0?0>97c)84wx79*e`Q*+(KU< zYm6q*l|m%oNJb|&oJ0Mype*w;aWI@N4QhXaV^RnI!epK9LCF)U23*V#t07xPG@gfF zSe@Zu)YP7u*{;_VuN05sa+Sv$Ic_KP+w29PAQuhU`9D8agK`ahRWl8x%)*g8h>G}Q z&HqSK8izr1&7fM)R=t@(0NL$~|7K6+XM22)W9~7gOy&uN#BVj3-1q_%Y!?heqxW&0 zV1%g^xr%{Y{!j_2u#)tbm*+$)y>@GOdv$fX1<{5(f&P(NA1ivsTFb#6C?!Uwy)n_P z>)!y&;+F23O#CK#MVD-RndApUZx4N+Y;%=|N4sum#x?#(q$mf+{XY?UU~>g`b&O)V zif+`!l0!=@p4iPx2le`-QR+t*19CaP)#TzF`5S;UCQO1#07>BW($$>qe1Qzlgm+;n z(|nOPKpGjaK6IZcI)V9$JJ@4Jvz;t*Qecn{?eG91 zB#TifUR_8MA--8{5?$FhyqLOcucb5{d5V9YH`_5Ub$q0QJ~YG7n_Z~8_ztYu=M=2SMg9gs{ zo?l!oT3fBD5+dDby%egiTnMKYq$4-EIFI}7TXq(1UcP{iK&Hv5`&yPzwNg4YB8yo; zK-GChEAgE5Bi0P;o`7|1#A;P$u7ZJi_tB(4d>+iqpSucoD*A;Hf{4j9RiVuDMZH)- ziJe!tYs=3I2bi#3tMN9a`%S93A>vhL#KXu-#`?2CV1uILQa#WyNCMC0dk2-D3ra`86EwJ zEsR@O2Xi-+TE_FG3cTw76Q;21CIv|dm%+n)ZyH)zZvIfloHi(8VJdVb;qi^wo~h z+wnODY3MuzfgXN*hfOFLZHsR`q>>)g$>h7zuBlfi&8N`_Dnn9S@Kse1_Lkusk^hIU zcYuz&>)OVr4VyHL8{4+oG`4Ltw%ypaoiw&>+qP{d|7m;Q&-=XJ`hVYVWzEdW=$tuw z?}O{Q_Ws2LESS_Ek#6yW#A1VR0~UeZfzx7SjU)&+GIRJ4?U{j9R%soj2eCFfwzFrz zcQZt2-2U@5vqQ!1jiI3nNFaTvlHZueR^SP)j>EX>Y&JKUMTm9Fs*vaVmgoCDbTjNd z=-v%SX8U{BrVqrp18aXN25Q@i36XVSpn%J6wXYqM)^76g^ zSL~xoeCSMWis#~QsA@BX0N~auEiM>u{a`cvXd)EM7Jn6b2Wv%0J*=Ay0YADr$(}h@ zB%Te=*ZEdA#w)6@H*%Eqe+<b z9!5|z{8rmG_wbVrb1k%9JKQ^)(<}LT4qV&nt3etKze(dP-1^FAY!#2XXnSc^ETfK!@dfBqW5={0k zilZXvXO}uT<@7pOs3E%7KU{wPj;N4Toz9BSA)K^YtyudDxXIN`<#p6Jt%$(gsbTYh z9Necw*j^~AIS3wnD9-(E9TY`-WX;JTFNWjoMVsuq{l@&~a5T z6!#@5~K ziNXDYQhMFd=5t}reoD&NjsF2EG24CmDaBVwP6Wh$dzAQ?(2V_5(v_*%;p#Kxm~Y|I zk$PC9)GSxjS0@IMdBBuz(A+M2vfYJiaHY(I22`H8HZuMe;Es8TFE9}gvHfjk%~si9 zWVL%YS5(6Zdr~Doi1GCsWT%l7EJ+qhVum0H&VG=#)|*^Z+hWa{IL%jDi9YvCGy`2(#rS zCikqeXAuWsh)(p?oQ0Z9yp4UIz6qXXQ@42Lm|C??hwF=M_o9avKa89_EJ6v{7VXwt z?ub~u#V2g8X}d%92(8RmbYime^jeCqdE|Q|B-Sh5w<#qV*rsOOjz#lEWGc?<>_tfeAkqVqbwJD+wV-kjGi;Er>)LoY*3E-oy6YMfiz2QkG{4;>@vn+;1%WHONI z1wfZ#qe~7kV+*LX;u-2op(@I5yv!eV29xLd7&N|#<}u|_E4LSB6FPG)@`x#n^Trk; zAyXTLTSQlFni)V|QcBH$Z4Yl0BE*F8JtLEQbhTB*76sH#^^Y-XaJy1Er2*)% z&w#cgZlcFsx4LM(7vj^pJ>6cur;WqMT4$za?XM#a(su<>2Og1bV#MZ}`=J;{BuOEn zL|H#5Bz17u8XKQAB^=8nn4PXyo{&0GrjJeC&%O1Lh)7TZXLoPxUz>|<30t+uSp0zM zXlEB2AMa6XBQ(z{HQtReFCO&?)!km~sJcw62K((>Q&0a11}m^mkT0;l!Oqn&N&(o% z4+Yb}x)#m>FpUKT1;ApB;;?{@87*>kDxCLCALi{R9@kvPFGsi^Huex&Y+mj?OX5qN zYz*_@_b`52{K}%8qoW$-*NN`Yd|+kMWO>*fiEdw)w})FsfvLF!HIS@-@N|-?06MFk z=R>Gr*l_SZ$JOVRC+`Q;yuQ@-yCjrx+ieZz7Y&@3MBV(@TIEiGFqm~W5zn}8&?F!Y(DbrY2 zTnJNo<_|A&_SKghk~%va~WI}{!_o#KlEL+s~qbQ&O*KQcB2YATg#9hg3<_I_m< z0VX?NMj@YDQZn?-ytegHEM7ud295>-=O4YvKVQ0Gp_5#OuB^vgTy_KNf0nw2@xNjG zxb2kP#dVk6+3Vr@x0_`i71t}nVWxWzJ)t8U&*z!1z17QK`-Zttip?jvnEBFh-uIjZ z>V}qGpCV{pv-&Ou&TvX)@l7W0-|U|;q+U*2k4@WeYp>)b2IiK@*B@{^YtiF-{x=V0 z!{XcCIrrn{W2baQ6OBG%+d^}Ja|ty6z@qERZL8bdeU-(q-8C>YtaQjI$EE5`VlXX! zQi6eEz<9s*D`;Q1d4JiZsY`pS?@jS=-avk8e0RIw_k1?tEen&PQEtz}PvV0tzCL%> zb{=n3@^G8jOE{zoVSc+_%XoVQF-8LZl1pz#lZimu$w?T>+oPi!Eis`yjzG-V0rqCWAwVw3$oe>%jK>WZ#Ly%s?|<_OBzQG))sXQzRcrs_*xfQI&L=D zs&gwz;(rTd4xlp(%RLIAhvTrVb9Oj;gW?emx4j<*1k2YScL=mTfrAA47Tznch0;2m z+}4lTM#WX`HmapRs(Nh^ORRBp>saWS{=-?i{(`3XD^|%<1lA9fuj{=ZNkZ--4*|$y zi|Y_pTP9^j*-zZL7Pry))jAzhK`Q41j~_mE7qu|Lw1+{BM&J++tIL(s&aUxmp!Iw_ zqX>$8i;*Q37xxRralcEB4)fqVnLeDEBDe2ts70NhjV*reFJi;-o4Q07BzS>C%k=CY zRzWH*GfE*5h8m>I{jA6Q8RgdlYvakJVnAuXGMXSH*_K>f{dvRRPR0K2)Mv%K*iv~v zIQ;arB^uWq@Ya+3zkM?NPoKhrEaTelT6rFy-(L$DN`1a$@Zzf}z`mC`-hUUB2C7-= z3BJ{}-iij24T#B))fZ^#G_+0|Qx67rL*gkZJx8+J?ozJFe7OWFxERn*Yh5y9E0-|B zxA4MugGRazeiW=955aA*VMG}+SXbAsVtyhk8eC*Zqsqq?P?llyUq~tVg{Gmah3%vj8(6>5N|c15qB! z*cD2ON|>%D>&U5@#9tfNUV|TGg-&aKU1Mm+YCR8lKeiN&sTF0vGLH$TiSvmE&aG7N zW*Vcf+~y7{!7ghGYM?&d%<>8T3u{pvli7jCb@Rl2F%tSptFjF8H`4 zI`p-}B?r$Y_+P~N)lC>J;ndB>;3d}=+WJ<5)jq#%8*2Xwx<)i3v3VzbHt*A>JYVLn z4+<>fGr==6j#HcSNq$uIV-f^(98jNG=y{&!)2X1PC1^4}WB+;4$L9M}mn$=6wNV8j zT436)xAOSF^O(C`Q<-fK(XWxlxa0Zw-iX#m$yAhe_uUHDLB#D=LEkLI=Jk@fGU$8$ zNPI%RZ)*Js1;r^%{$`rr%IH+SmMF_5p$Uo1?bIl0kGDQHn_PeYP0IQ19-!r3uP9gh z8;r&@dOp~N#fn>Bk}2!PcY(tJ4h*>9q~}|XTy0u6HA0GfgYIiGpP=sOUaGooJT?Ir0v=+>-Q{ zqU&x92Su22w$eEDPvIn&tuG9%*#ifM>+ROm4y$@Z;njx8E+rO7De^VCuSZ7pD6`*{ zH!=6WRffI^vYP~g(ZSw#jL*N8B3>7f=sP>>IzWk2K8e>V4Qrg4pOoh9B79adhG zub&vNJYN~6edB}tmQ(&dKX{=H*?IojWH~tRSSlG!>jwftvc7CPRwPMLrfwA7{G1$) zE!Pl4$yUTC6Qik#OoGc_GHRHt+&x=L@f)ahFJCekP^+->QlxJ1S35p#UkIpKUyH;O zg8Z~}Vo<#G3E|Jn^(L!cDv`qR@eC*}39KlkNS&9HmSS6mB=TZgw@)T*3wStnU+&^L zGjT8ETrK|UMo!F`J(&{SNSFrUn8tk>}f?SvUyUHBs zbQwryKx8d+ke0Zq{(NT*q&B^#mV3P&x$H6d9>)q}`Lg8h8OZ8V-Ow0LA z$NSkPn8rMc>8$~yCyF+;~!ObEnu>qs* zN&*y~XF}@D%e>F=`2ln~&cGvUf4Y^4gq%YmJp3IhE7UFhM*tM49d!F0>Gk4r|M*=?B&0ohV@#!QeN;9WA|HPgRNrGtWeDO=c&0r}&*vRru%J$ZIef#Od-8xru zH(-6utGRQR>y;|a!C+|XGnQXq0!}3~y8q;ej|OSBi_|fzHb*zk2ASKxl~;EE;k`YesCY$zsix*C5wE)%^JX(yCKR#7wJt zLw?LQh9c0(%~69lv|EJd6GlBdxaT3N#*OR*Br-zIdSv|5 z^rF$-D_`b-JK5jY87KZC+G94>atzK>Z}qslZ-5}yns#nXweIfv+nW%DMSCB*D-Q?z z+dFC;JdMjPhFkugYh1NTToPfqZ%>-6x;RvUL zhYA--{t`r9S;2XEl+!)y^=V#hS!zQ?GczV6L?Y_tiy~{|Ju)0C1&Q>f4AB@63`Vps`bH%qZ0V?`6-YMDBIm`6+fptL6e|$NY@(T7HSv?% z{{P6c%b5?5>fkW^I;uIN^5Up4m16%&$j7bByBiyOPB=}z0)K_NJFpsqUBuY|sMnpg zX3Z}n4oZz54aBCG6H`8>f<@2k;35PCu{h*LZOX! z!G^4Sh{mQP_{4t`8hUzdxP$5Fk?uqMz*kBX~_{|>Vi1cGw| zA@|sxwI;U$R(la6*LSY9ewFyl#>YpD^nD&6?HvNgO*&_CrQ@#};lEDcc(rR{EwMrs4qVx9A=9pR_Ls;pGMSeluP<%61oHx;hPaKD~6%wgGXu=6q5OKzd| z=7p1(_B}t~Lp!!xBld->bacF=LsR(_%cvF@sw*yYxfj&k!kmKAi+00s^Y&*{kNTL< zbcF=J+ws*@g8l`m^$$hpGCo_ENI-^RyKcwnLE@DREnQ5Y55piS|IvZpvHM-v;^w0M zD^v5*eWS-0e1zwnv7IWM);UsUxFY|} z%mvS~t=kuf^#^|z^oasmo`H4^DiDu`u8O@#E{R6IRZs<1qyBw0fS{dkyWO?k`_I2r zJ*|?&Sh%BTH>!_aznlTd{-;a!dLo9t(*LCd0U!AcaQxb1VBlV}mW$Ye4a*1V_mbs| z@AAJz;6{+05L%D3y}|wMDsi|(09TbUA6+zJa)n{F$HCc-?To(|>ycDNozZdG!`R%I z7d*vq_}v_qX_0x+N@X=U8T$uX2@v_tJ{!Ct?a;U181MYxx`q}w5DWB@42ot;y`M?x z`$_|i^CPs0FVN7OKW@B7d`MT4D))aN-@Y>CEFcTVggCi{+Z$I~RdA8L>0><8XA68t z0z=3Bz^UANw-Z6S;p;XW}azy`Zbh z*|Q!u*$owHev)|0BJsG!B`HjknlIPia?>8m6?N7n=+%qUfbIy?xhYZ$)5-J)tb@^p zt4Yli@8kar?1iXg2qY(0Ura$z5aUqP;2_Rzz$X4=oOoy!^|goJw*~CW_9iRr6b6j^ z1F!kg`}mqWgrKzsGzbsFa_yAE?OJ4HB&aYhT()BcOdT2BLL_Uii00PLuI*CDHskZ| zE)h-mv{}D{+2VqAC*49msyM{tH;rG8pW@PzG|Xvi+j$fXz@mTn0esv&JiTf-T+bUF z&(5&Z8gt-3u%gZh8oQD0egZ$LMw9rHvG&H9QGWZRqIpI{#D9DzCquSyrOn+{MrM*Q zW(-l^lKtwYZJ3FxPctP%m?>#G!AznOle*0=f7A%RkRiYP;j&332*FCz`My(W$fo}F zMMC}){!E+Ef*y~-f1=&qc`v!XnN$+c&xVn=~u&Yi$DGxxd)f~N)&@Njmo^}L>~ znV%s`oICQ=*M#inlOorYRFoY7>!v>4D_c}!42|7hZlwxmaUXhSlAc~9$MuF*9@(5V zD-u$_QodYYOzbxMlmrgNN!9O|?%xPY9he{*`$6+|)RwY;8ohMgV3aw>TA%3xj-+1| zk5q<3lshCEsFVCG(uSP9y#UN$%~YSbCva3~n$GnE{!25S4pE?l_-9wA%rdVT<#w(& z)6u3|Zb5m#CebH~kV@F^8=Dco7>!&|nfvCa zu0-G>hfe17l)}moMOYlL-ihVz{*|22{(>Jqb(Y2S1Dc~d9ILl>W?-C78o{xZkm5X4 zpH5_o!BY^WAV*^aXAX@nUs%g6qW?Z>xUgPLClXxvCaFYtGI+XyGCtINabXbXQ8ng& zEX!`cqXH5{t>rq;Q$Fm4mr_*qZPy^E-Xo=U9QRN>we^-H*{m9++(jA4?}=;CczKH$ zGySnQx)h&dXQ%G=JmMV`bqrq`#gT=bCKW&8O7;?osK@JWQR=adr<5D4`A14vq#F=pS_= zltSTaxb78Ncex&oyg0XjE)a4DW||f-)ypr>z~WO&zR%#ism8ygSXWEu1F4jMdwa}& zB{bMRg4j;IJ;!;w_>@v|09=;Q?T4O8#?rf(gwI*7+oYS988uU!xfMwdj&haI8(5qC z`UZNIFUAGej7u;@8zuJq36B@^3(1G|GNHA$p&>~cEEd&oMaG%<<~@UFr)8vVQ#kQ) z8O(_-L>37Ffi*>@MFFF!K@;S)M^Aa)iKt^33v0h}>b(rn!bvnkdSP&Z*d4hTI%ZWv zPh#gt(!5y5w!<}m5%9CQxhX$^9;pWmVnvwGjl=q9*ZtVuQE91B<)N%C!psaP0b8H+ z1t%;%kZ}Bk;2O8jy*y;hLQk9Q+K^`@!@f{d8f0N#r%;t5kxW9u=?6d zeGD||+`xX5N)S*GfRoGc&@z|&$wex1>THZAudfu>CIKYqkdhS!osDD_XDQZGkTxbb z_7A3eS2BT#9fUDGN?q@;mR-cYL3%~$;~IPb$1pn|KBF!GrC;?MmtDp{$H&{naN<8X zET1sZn{?!sPMl?$Vc$+0eF!{h)Z#8KPJ#ThpTNjgq0@am&eQIAz7~eTvXOHMBNvXwEG#VBj$ts`Slok(aZ;5{ zMe6thh%M88%a%}u%Czdfxp;pV_Cg6qqEuzslk0mzT9<8ctf#x#Ja>CmBfv(!?`ZOB zEmR_r8TI6JYMQ&TYG|^Lg+TlQ_%%T^tY|hR3$R52OJP1G=bEZzBtl1*@~j$1syw|E zYR4u39~h3YQR88E&ty4|qJ1feIFpRlb~mb6f9G10gu-+bm^KfyPYcd{5rzjiu^*+Z zO!@^OE{nu7^9v2q%&T)+i6>>0Y%T5u`%AK9`PeUc-TthANjM7C1OVE~2YckUiWT<9 zWq1d3=&N*Nk)&(#9&Il3n!@WsLDRf#QaDV>k4l!%DC7j|e4~f-W9yq=7Ct#Pq<$3r znx2kzzML@KO^+Fs7ooyRx&RI_yihn{G+d|wkwm7W`#if;RYV=AsZ4870dsZm0BLQz zTYwW>jWAnNdn~$$tjM7EdSfd3qu+wiTv$|AtHH{7oSK_NfTZr8NlA&B10TgART2z! z5I`@(DJA2sVP@x$i~19;>%OkmHG8jz>|>~pHVr1FUIE`r%8o7?G-*r?Vo7vbkJw5L z8dH$)RMwzKkyrwa$Bh>Sh0J%ACKC$NXMN*hXgTodUOwBYVRSb9v~0A8R#M%*liaD- z*lHG)h<`iIOJTt|8fBWH$o9Ofg$vS(zHGQBwep0Qye=r#@iazESC=%e-C5m3ywE` zt*`g73mtQ(CM5G!ycBA>0V{s^gr9Ur^=F7BaTu-&zut{9v=0#f9ZM6f^Jk9V9IRIj zmyn8Qn{4;TfF;+NOy^eKlnT5N2xcZa?k*xFL8Oy&t-uPI&_4A{4mC~}kgk<+FV+$h zBv;iivPf61OQP*gw^N#&HSI`wDrwD&BY?OKZQLinh1jPN+Uyo3PF`dB!bRb+t3cT^ zbKu=Pz~0^V!(pI&=ml=NWa5YImGImSy=kqSIRTks!lC|%Od8mbFdj0XJ_9DQs?Riz*&OnVp45v&3JgRwc47s@pnj+~=Swf0^!jN)j4^4ls)zyj zWuWD%w;?q3W?Ua0ON~F8Z>abaR4t$lN2|hO^C&Ei4>L|K4nsp%f6m@i%BcKN`YLQ~ z;A|bDgqa(XTBZN}o8(QcN=hZhr>z6$`|`Z-w8An{i|b&drGeZ=*qw1<)U|={xX~Xr zoVN~*PJql0!H*k6J+8HFLHr|9kWjS01Ed1Rv!x?@1EFDx;rq_%BN`P9$pH67!8>&y)~?GwNV z*+J_su2whoFnBqXY};%8^ibkl96wad{q$fqIygOa5X#C##92(z`V(^W7a$xLj?^QQ ze8EyOIg}v3)d~}wG@nv^?T}SQ!@ACgBc=KaIZYZVaB8Ijqo_b^>tUi`6Em}sv9Y$s z(+bs60RaI3+(tY)4kqi@ji~+>TingDR=0J@c}I_@Q(zqYY58>1Xbw>;_4+hCYyjAx zT7BhNClm3!8-%##=RF~k;=WewUO8`TWNvxTl6o(3oy3Ts2`VUO`v`E7YQHWfDcj+^ z-|=ZIlgXf;Ls=M0cz-xKJ_^I}Ft2;B4c45F%Z7zLDLrp8xefI3?s0JKsX9lY3Q!Vq zzNih`EXQ`ciO3M-c^P4tcF}y>t}eIJsNVj`>bV^=KK7~8ooV89Vqt+aDzuHEZ27cK znKCFSD5ILXzOD|4Z*5Hw+^}soy6E<|QwKuVnUBtIbC7m@`cvxE4#LFPI4B~bpLN9q z1_ow3*Tvt_+=1Ec;uz=sZQbyw=_&1H;_lbgvz8@=s%Oag3EiaKb?G-?Z9;#SvuDkX zXPUf$1{#uwpeeCkAP;Yb4RQZRPk1s1vEkok#>{7;~GZSbyB3a4&AVPkB$k%8L#H=RpnWsv@dyfw!nv z%b6@ut3Lxuwu52{8G^U6XAoC7B$GKvSoU5XJw0_Ps{Pj!n4aGHIh7nL28%~Uf(t<0 zhXA5!G5WYdqIZr+++R7Ikm=VorVqc2Yg{UOWTOIF#vth`^thZ>d<;p@H}sVp2U?+T zoWKZxt!0!6kGBnos1uXl`~-FkeU0Eliw{Zi`OoCdr-u&A`cOK(05GpTFT%lSSPtfdrOx8HCG_`Ui0Z8;^+m9 zy>VcA=?v|;6U{Us)cu0Yk0L>RcaiAjY10S3}rBfoW52{ds&@2qjNOGMj>K8sO=Orwj1aXb0=Fp@|zAp7el!E z5xRimGeu}5E(Gg2a~R9tGy3orUyIQt@KFj41}pkiYu$s<;k>ie@D9g8y0huP715`? z&DnJQ%;y=lt$kTQ|4O@+o0~>nGVDzo1#N$$D3#>>J9_*5w0rOK{i3sTARZS>=QrN) zc_;ioE&F?8k)6c>(IlF#S@RY%^IRaa>i+s=d_!_<%9&^7lHY5+554|i%**k^l>1$r z`7;vF>twtD2nMc}=RR*|0q%{Tuj3bfV8?%#G4P!*|LORph3jLpEv|JHar3U|rC-J4 zqV=cXEcevj%^@16qFCi+iUht#X>Wg>kmU=3Lb1IgPhb3$;CnTrX2Kt>-H1F+Qw39GAe#-Z)Zz=0@ zPG#<_$W>>@Jntgt`Kf=hzW-?)9G_!j7P?`;YM!Jhy>`>1s@>64KP)BIty1idyZMOm zFArp~*&A#9Z*@0Mukix@q@f&tjqgwI4$fHqd(RRi^r9JC^D;b)Q2BC~4~K>y^BEQV zzni^3tL>qZ%IvB+bLQjedbTmJKCDQ@8H#53UoQeC@dSEdtjp>;by>I5Uvv5A=_VP1 z`w6T2cutxSn7Qs`<5(!UC85=qaXcQc zZ^~OP&T!7cI=Pts>HOa@`h+pU8q)^rag|ozv%E(A_*ldW24G!+eY?LCYJZOLX0tT= zoJT7?zh%xP!DXlJ{mr|Q7cy%QPkb@p9LL<|=!(T62}4qkmsXnfE2I!2K~PT6@6Wfp z|LvgE*z8~K+h~6if1F3I-k%1W>f1aXqD{9&7x%e7yLE@$BXRFO9ac(VOX1Y!rc@Zd zUA?~F7evt1-iN^0dfZKZxs8y0mwQQwe_e;{x8vliPrElcoV>#J=ovql7y+BkIm>VV z;Jp0h{TgKZM2KYvRL-~g!a+vhB%u!3qpTgf`Iy^dGbeuLHlkD%y|N%?d0}nxe9wTs z5h-ED8pJ8b%VyT?p)39F1_E0xiuTN2EZyIo=B!Mr;XcJwu07FH3WibI@gkQfKjgf{ zrKX!>>74%)Mcbve;sj`uGvbn7+sm4105Ad1fk1@Cn8G7O$O60Qcq%Z)GODh|sw1W) zt0^ZUb3Y`-0I45OYKCDiO#M0b)i_JsJbYGJJI&$kcX!SX7l+X^wd*Z38ZPx=d*)1c zhn_-dC}nHGh(hD@c2y3r9{|=@NBwh#1Aa@iS&x5#4WTB@P&6y8w&;0NO_ygYL z#_>yPRhf~00O*$S*tRJ|t45-bKmam(nw)xjn%EvAfr zZXPyUiXB<8L!Moil~J_0SZ5x*bDj19rZViN|B)@8$izX`!_#FaT*4jcAa`|;>TE*; zsH~6EG>bE0+G@6DwA}y5wRh-Al7=#tpM!(>D|@@z&;T@q3=rwxJMht{+jkG-kLx72 zbVjI6En;8?D&NoC75sX4X`oerHgmrO*8RBRJ@D74K7aTKNa=QNESfuXC?M)v-N55F zes`XE(0~gOdZ#>Pzdes|-2x8fvDE0jd(kcj?RMyd;bU(iT;@(wqOVi0Mhe!3D|#|z zmP(s?aBX9jS`_Q~wZ~(5f}S{k^e;IL9+MhMWuopRs*}EtySvJWQMpDwU}GpW!&dy5 z<#L`f)fQghN0R9^^3w7y4nWpI!|{^TPeN)g77_)w?$uOGM=@hC#MCxQU@{x9kT?ib z^&MfY!9klCV$17zur^bX1g%G-%?UJIM4ts zrF+pKHG(`IO-Kou){^fW2Q3$Nb2Dm$BBp05(bOr@7{shNe?z)Hj60aji2G4mJ4gHD z^m&={mz;d!WbcD?jOZk{i+ zG(NrrZB^xKQqZvE?$FR6Ww_jC=4=eZj!r#=ciz{h@^}=L6!we8MyZI!2PmB_SdOjN zF=D%K05#V2Z1Xe)Ya zWpiK(h zO>DErxtQzIb$bh$63mCs^%Esuv?W!SX;3oAEL(2NHL*^PfF!{*Hy3*@}Ityncmzy1*&>{%@aC_Jg zqEL-#(>Ad36n@|}ZN=2X4<-$BtJ0~cF2VA)PnHqneHcN|#OZ^P#|>F{u%(ZzgyZgV z@|AiY%s`2;AUv;0JjDj+TAbU<;*lrCZJ)F$lFN6}vv*7)CBm|pF8wWudNIrjG7!!3 zvboWe!K9#ApUjKJ!9*b9%F%{%x>y~s-2lwU1WS}ws%R=B6D63lSYQ(?dd`#E=^Qu#ICxu?v@JFxhb^GRPZMgJ5Y3ezLt$gEWGRVTxquc3I zm|4=f9L{dibDo6eC8K}`Y|qrEF84+W_|{9Id}%WVB{UQiZxj5M?xgD*94)199nClj zi;O{ww%3gKrmnq{Ih|iM5ehLzAlQ;4TI!2u@+E*-b20dyjwEr;Zk>AlH=p!7nHxBy zTHCKvW#Chicvf*MYn8A#sB7V^W3_^+>8a~!eIaoBk{J|26$`K@9bt!)15t}YOcC?P zv{Hng$is1{*eZAkU$Ru#$tyHLX)#BI))qCU%0)*;B$T`w^Ri!d&baUMOZ}qS?uBUN znvT=Hm&)tA{P2ihnUzyh)Og7XTB{;juLlbb03MTg9mOc=PVbkEY=$X~rIxU^kdT@! zdjrqD0OOU!_#$`3Uv6wdg9Zu?yng`*$dE`oS)JGjgb17F7z*N|kU6L*v3~(i%u0PPLgG*misrbVuzs`m1%i|wIw+v&9_5pX|5*DZwSJfACqc6 zUT->FRSAX%mX)g@pXF=+0#Pqa8jAkP?PBe(xmExdQ~Y(=s!xS7E;z8YwQv}Xr-JNb zY4|8AH6!j1|LX}BoqgDJJ=XNO!$U1NxDuCGgXz$Yv!Rr?>P^(7(=OPPp#+ z)>=PC|J~I^>*c-}1!6C1oqCKfhl~-&VyZ^y&(g3>$^>|H%kF|C69okYAYAG>1Pf|g zZ+t47;n-r(wJ?vnZn8fP{&TsB7-}^w-e2>kQ)eG83|Ik-j*srHS|+;5@~EB1zVoc5 z{~|Ka;C>I_;cQXhsC^(od&?jUy1jxZKQ-05c!-IRQg96$s{w^q{{Uym`IMkl#w^V% zvVBcBvG*Qk^1dxL*E+*0!`>AnZB&25ZWvR5A=5LsskMUT$9*>PM3G>3V(-gC78gRajTs>Qa)^ znvj7Y269-==};v>XS)sD05ztT2wxqIUF|6joB0HPyl_b8(OfN0dfWIe=JwpYm#gV< zKHHs{!CcWxf7tAny^=e+d^7P;YfQ&vAb{o+>^63jD$#FAo1DkGQYg9UDa$nKhpx?u>#2&C6LJfcJf1IT2#8NQZLn5TuS zu!Dh4gaBl1()Pb3B;QoCn3IRbegy1DDzrZ@RR#x#H%49J1E#c`cbuT(s!WDk(c1W( z9SCoa&W@glEgXJ@f5d(LeEcDMM7eElexM-{*Y~s>v zQBP9!4=Fnz6fa)*u4`O`Nu-O`FsdoI*4fX1j(f)XrP{+Fbn>5@UsGqhqU@Lw;_0Gm zT$y3=uBv6XC^&fcz#>5re(@5YK3s3smBswXmTYM<*tj2Oly@%R5`>7g7$h-8l`z zxq_Qx;$aEsyj6>J7#ZhEc|I>?%6tIe zp{-94(HD6v5UWKyO}0ouw@3|`GfjK}n4~69ITDdcd6OJQafmePM}v|0FPOUczT&mW z5;Qq*Q`bW6FjYQ#)0_GEH2CmMrzI;1f#l!IY0LyVgM(&ni4fAv9{hC*s5|_hym!ou z$p;I&38j7TefN`-hXxhAs@x4AT?i2MuF-0Eps5ewp@( zVje5mkEWo8ClDV)>=#KB`h6tAgaC$D{P58vY-T5o82+qC=~!j_CJ)WY%^W50a4Ax0 zAEvRE$}AO377ZB<5t$B*l$7W12naBCfNmUC<=_Un5W!;$OI zuN<-lZb)!ZD@EwB-xC6Rm+zXQAlEC2r>1yQR4hFw8xv;7@`@W|kM-d6RK74VYqIzC zCb5L;pCbrH-p^-Ohy|t2JZU&Kr{ze%G88W9>`I4=%1$BoG}xl-Jnt{pSy@?L1t}kJ z#mliC0=}_ig(8oK$QkN`#US|a;@$OOj5}gI)}-pc>mHAIsajcYeUG4g+RO=nRlK@R zC>`)Gh(NY|Pbw+mbQo%Kx^AlC>JJ%>=%)}1PiO33smDh=j}4pXk(sKvS%K61S|S_x z0npX0Fmni9s^n^B9pw0J2<_z7pq9`;q6h)NiM5NYK65hiNu%cYN0LXj_yh#ZyKr*L zO_VU?)C8ad!!=BE&F!wVJsk_aE|RsB23UscE-8p|wcIRZVdO`7sjpLHO)m>Aj!IQ>`!H{^1HH^kboRabweNy-$ zS(yR8*BAm@+fW4}Z|)0A(1P-uh-m-|G+bb=b|&aHFXiXs>=~V`ZYkSD@h^~*CW39v z&vz`dCd6$y@!FYA=|6=D@zwPh2q_gfiU2}&)SOyZKAJ%5Za5w;$&lC~rT>XO44gevKzneeh728~D`a-Tks(vlGe5V&Fp21^GOr-|wrmbEL zX?Su^8~{%_k9^t{pW417@?p6=HbyVPa>=4DXo9x{>9p1;o@?b2$^N?aQNv|pn-e^& zQlc40l{pp@QE2lEW~s)MDy;QD*5)+Tcb3`Vl|iM$P=PJIcKwjf1cbJ6%Hokse0+3H z;Yy}qU}MT_D8)Tr>q>y@`4@YPx+f^boY$inaPoQZaNLCuz{f#!m2%JWvSD@Fk~FBv z!`yjdE%0-X0bAfwa{HzmWB1D*CImJw>s-&YjwerE=X)jLtGtk8RTJE zq_@{x`-y!Ql2$gG_uR|QO@25W?v6WC;wfsB?cdvbcF56TMac^i5fh=8jtTwP|NQP} zcoU}$G#%CdZ8||*6T>OWc}y>E<@~JPFIKX;`WnkEFXh8}+*T{n?rG|2ZM#o`Y)qjwrd~2s z|MufOsmT{)HaV_6DZl^`aupL4@V%Y^iJ$j9RI7=_3e3n85De|Vms%rCN^h)7ke_>6k>KZ8T2NoTfQq23h z;Em{QQNqRMUQry`T5E89tz@Q`C;PY|ks4IMn7nC6=k(-)s+kOTG^_+y9AzuhX)ff)_!RcGTTFa!jMmZ3DsAWpC41X zYL(bmpJGE3vxZ6aUECqT^sTU;Wkcv(ZuOPyS%z;ARYI8qlP{8pfD<8@s1rP5a?FSD zUREKju@_czR738SSmTI4XI?K8xPvOS^fz7Wp3V0~jy?TNZ3OT4 zU5Am6AqDG;1`V9!^>*s!hYdr}>&$SRvnXO%l_?&VH}i8;WU2MqDjwb^eQm?ij1n?E zyEgEd1SnJ~f^n_Sa~jA^zk; z1HK=uR`>!+^J{!W(rZ0m(reYi+r{aJa<&x1aMwTuf;+NR63c z2yKl@zme>@iiW3|o@|PXE1e#K7%r{wng)noLXIOw%1u`391_5XO5O9=moCa~b-L%^h+tj{ zcoO+}lQQCC8KIefBJ1-#DiNIm%>tnk_%FFhQ_AL*jo3co-p?)eQzoU!cQu-v3e(G$ zO!qSxTvkPcpJY!b+Qz(p$1QD4H)PLw{^;#;GH~#XKisEVm{oi+Txs8ZXSm;WyYynZ zjDgkE|FD%rf}_BhSG=oLX`|Q9f)V>A=V`k$kOP-@@v^mNd9TPOc*7Ws!S=i1goZ1z zXkK2=$5F71?bovnqM5346kMPjKtso+bQDTRNZf}aqicXhk~e2coDxo=XxpA33mTSUh3(GhHxE27`!r_K$wu~o z&?N+E*A(jfB=y$PU>tU%5NO(`wbhR*(fMeFn9ud+boz<}TpEe{k|*Vh+v6}aIP57{~p%c2n4q%>_~%LP zef|5ps41$Tr+ZGHv(KKr)?RBk679pUf}HjteC;ImE}^D>sxgv;H71d)WM7Ktv>IED zF*9ScQCD74N~lxuqhZ-(2t9lm+iY|!FRt=q37OMB%G+41XOl?_209nw#2|zQ$EY-< zE!It>AX8O2vN|jYD}*Fz3{xwaCfk~Q{q_IgFZt1ZUks3R6rlqH+gSLyO?5R_d~-ng zdsiM?ok0TT59y{n;t%O&Sy8bNvzv^eNIYbDxrgiFHcxagP3k26SVi=4^|LX8)_`mv zIYrSo?e@p1&*h~;@PkQ7~d zLmR20$s<^MT;686>#LJ5L@*J_9DLukH>hS1Evt#p#Jl*R`@6aroxTbkU}9j7sPQK| zjGgDkDUs*2uI&xTH1_PF$yHz8o<4Ru5n{l^S*yYqSA>R-N1#C<2|TXXJIT(bHTv*F zBQ#u=VuX+hYo^d)>K31vjEGHAQ8qNy zQ-H;dMl}Ijggacf+sUTVm(E{QX!{`<=a-4{4K<+Pi@`qhp1 z(b_j3Y<;lbI~N=52L*kWxDB-ZFvb8Yv@)3#4nc*D#q}A53vzuBZf@!(;k)&15o-;k z8%iZRiYvk1k_}6qVnYt9oM1Ols^IZlyw%NCG0ma(S?WX!;3hLa_yDn?kGH6!zic%j zdO1qSvhASxANDw0my=ew2xa+k5rLsIS}sZKPz+F;kE1sn?bPg-X?(4CKHRBcp^&A{ zVcMRJv%%dh3Y`bxh9EJCXL)5YHG%oG)?URiRlm=f-=A?OGTEoLoK2-==Y|Ntkoss- zQ}CJ8@J&D^@GJ5i7Dq?$HYk3%#or-k%5WyzeEOS-aq1dy4lWW znO3dmTjn4bG>PCNB2~3 z5`=;*tg4bH!REm~b!wr-kkr@TJ}4b2D=P!3e>jV32G==lq!KQiBzo@ooJO_Or=8Rs z`qx<;NCrOJ|AQlwbkZ~woIK)M>msOMANp}L6h>!8PPKALd+-tg!1w+eQ)xN?l+9#a z)z#Ma4h$IJS=9vu1lTI@kiI#{4%IRYX~2O*PCeU^%ZqZ?4A7Wf>xHbRWN>lUgv$GT|eI|U!`WN0h% zzws>u3_!#P_hRSQv%SRk=8|rBQm^Xn#{uIbYOm1lM z-sO|on=eK$@n8Q30D#}%ki$s8T3e9UG_>=T!pFe1lpE|;Pyhqt&jYQdmjBKH(Pd@^ z1GDCSh`d;1;^F0V)mTq+z1iQDmW+tV^tyP^e$RN#rJ*PwR0Z!N%J^K$D?{K@pq@4=k)&1MIcC(@c` zG-wJ^vdd6!xkRS@;oyBO$7{3A;|PD*A!6qq)PH2P#+j$!3L)FTk(O9umTu*&p}#t2 zokn@a7m3pWtIrT_|M%@7L1wt}X#F4lTx+eY7pyBZ<84!UFA|iJ*C$F7FkaeAl**_- z*Qx3Phq5=h>W)x?0XvP^+1dZNv#3pDgB2gAfPTu~pI$)f(B)#6%(}}(x;$Igr0RvOt=IQXb&=)j#D)J*db9;Z)CM4umUSG7iYSUefrEIZJ zE&o&TMF0cQBUMW7RNB$>9(d%VaLQxEe-g_tJ-j`>x>10EsfE5*x?bJp(#LCkRSw7> zU3pIBk#66JYWFUW9>I8PiXY(kXC1GpLn+q3X@_9iKqAh=mQ6w=!@S?MO`=+MT9b$E|PK z?>XSJlGc7Nn;YsO{VtWb?s-;7wc}GkBiqU0ojjtt+$`;+gMyd9?D^Kc zn)ExE8L-GXER#uXe)_Z2N+T;&dP0|;PMU3N&Yw;B-1UK29>(!E^a4Crk|tOn!U$g6GNw5B@U!!eU%a>$2qixu@a$ea0hA18VJ)PJUI^06tnG zYDM$Qz0^SqZp!Hv6G}W9*pao_A-s$h7H5@Jlo#nvHnSV;`4+0n=qTJl7BlDAnao%i zk6P={pet3lj6E(n!-k^HJQV^W;&F`z%||Qa{-=9~nGp~mDn0cH_EzdYSv|U)VKrjd z*}u72Q(^fTn7P}~_tIOvoMcul-*@JW*)hiNW4ZX)7bqgdCq^d{n%vAriX&fFJ|bF= z&pH}N!uBnE1@T)QV-LezBY)qTbcp^R(=ADF|UCq;?p704>F1~&ZpqM zf5L*?@AK!#0+kX+5F7$3C7aKeLIc(H+|D}r;F_U5NjF_>kHboTbTgdZ#ZQH#S$lY9 zHJ)8&nIVqT0IYcQO^Z${M{;Ijn4R_XFgIoXmounI3T0$G^)C8;J9wcO6?u3 zB#C^XAA97F7erml_?{wmDpz>=RbsJJX=(7v`N@DQ3}B zXO`^omAYHMB%3`W=f_S-b9~(=Q$*izw9_DZf zoh$3&V+?`fPTve>aY~Ccf=)JbuPn7%QYIE78m0Tw3mw7r=mvLS zTn3f-DXTl#a^q2G!bGsD0-JHAhqywS%(AVQwXLaQWc)`_8Ctda$qr0`aJXCPn)=*% ztHBW&n0*A-G8HZ#pDetH+`ZMM;dx%%3MlC@-Fh+#i$w3US{Xe+ z*nsI94rlUz5^A`#oN5ByrQ#UX#A69k!#nw@Dsrky^K~<-DLp~;g;-uXKr?oyT3AQ- zN2KE`j`R=!Hl%ISm0Rhu?^IEK7dSJ-9OunnhB35ri}=3=IO+L$st ztE-*UvLot4gvP(;)~a@zZTl9ehMq>(fwD3H#;>UwX`^3w)OUvK?P)lU`1UdufAp#9 zX-3!cD0!V>nzXHr6~u(v4s-!nu|!r?O^F2>0v@c}v z>}m2qWv#!PyRYw`!ID5?BI(3)8+&1WSXZ;p51BF^6~`TX67R|Ri~`WfcUa&G)F>Lf z9R<=fl@!#MHa`lYwi!x#{rVEdJpe)!mO%4tgn^tSGaGkxOX=_n)CdS?)Z+6fY6t|i z95U)qAkr)Nie6Th1_xy|H?ry-1M6IKE4Ck}Vjx}wy9rZRqQvZ-Gu*rnIC4l>nY^37 z>zh|_I%%1aZWm^u7+Y`TX_ zNxThPEU&uUFE-7%M;sGsbA5@5fI>`2WU>>=Hs`U|z~(T#2hO%rV@gQ!E|tKoaSrf9 z&JTj^3@IhlUQUHF-CpzhCu!9!1~85jBy0tJESA*?md^}%=let2}|PH zMs#`8QYJYR z?G0@+R3TxZ3F||!P=E7hR9j5rv^A@SKX#C7LNjAvoi`qm19SYSnt$r2Aj;KI1-b-S z=UgUHj>yln&nM|^mKPnK)9CfKUd>n*>QvJ{IM-oJ@$}fDS(Arhs=%n)D4lk7d7b!uv2LCgN&tcc)DuZ8$}nE{$~aIl~vU~eCR>G5&kc`}aF z2Ywle$(9O&4{Mv>#&bKa7l&B;M#D8R`W_|dYDk7kxJFVnX=~uyt(w&OyJIH-Pv$r9 zASy+K(#o<9&-~Q-h-`44_3#gdv{V=I(QbJAo?b98MPFLFGt<+1kAD*LEa2pzL4pI(Q$@7=4T@3@ZLl^WoE?m5r{s#eJ8d_njm-T6v zBVebm$}1K15`=N)4L$pq??dYP8i&)!20ra2g7}f-Ay}5bhlsjJT{!xMK=u%d0n7X| zBrud!CPcbF%|Rv|@IxsKhLw6sl_IoR%`DG8NgNu-D{alIwQ|d6PA@u2H1Tpo7@<@h z{6X1_Oz?gjloDTKKkP+`Sxx`A!s|h{wHUvx>|*;FEI^z^S@m-h-5(blNz_2x(fcxP zPuvbH8%jaWs-zDZMC#cz#72$oHAILc2eBmrB4z%Qvf)Sdm$FgF=V2|F6C9z<%Vc+n z9LH(@)Nhk2CA-mXg#z2t48Gi#Yy&qi6JXviSP9w<(b8f!>zBCsuL#%>?KZyeAuHOG z1`HhS!#z9}{{Hbrn1+?>!ja8r@-k72$b8^8DGcFs z&nhb$A&5qD!1M^LUr3ybiDM~}0K0UEQ?-$=VY1#lHsTstCRQtHIv22@n^`$P_YXV@ zWTIGZ)#YN-aR0%f2%1<%5Akx8JI6~~u2V#wIvPqFojM@9gF(L(=)N`9v zD?I$cdHggOu!!H8!Kx}t&4tLi%=O2F&?5qe0bv_CO`07euXJNZUV8+FN{q7wl+oPy^(Xryq znlh_2X0=wBsGrxG2Emg>iocpcaW{`_o^0|GYw&_uJS zo5xg39WN%a>;`ur+WLoghF}{J&;oS8*(_GiP%v|}>;8udmO8nn6`ZbjsuUzc-DZ!8 z^tVbuZnkOfYHe+8YYQ^BT5q&90Gat}Yp=Oqw5e)or6nd};@~{{N5+uZt}zjoy6-nn zJb{5_H-3I|IJ{~!Z(mmk5Pjez=zH>?EaOj0@WMoo`GNJ-1f!_u_IiHY3)Y8=dq=D3 zWa2Xp59o59mkScQ&QMthZ9uJ|(?9|$xJ-VG8P(f-+%JA7Kh(I;vuDIB#v~5wKB&AS zN3dKuBssazDP%BJ3*l4{4pP9jf1F^%lRbH@uX{LyNY96L0DO$xq^=3Om!v~LlSD{> z;l!Fp=;BNtr85#4udrAis$!K$Y;$w%4>BL2Dl)4ajAmmfO`MNdOxMkoJE3r!dRz4C ztAQJtk@wmQ|A(Uxt%GSp3gP;iQ9X^oHle%x)SZ__N=d`Yh=xHDF@tD4^vQm*eTvM8 z^d!sCOa5{$^CVDXtVU%1hK|}mUtYi-ElFmQv%nCL#;Bv)b?hSz~ zb1aVcIN13WWv4F6rpj-VB2#RJjs8WjwhdgOCHh;RlTLaZqR7O1mX(o_F)|_t`)On}KR>V2>T=a|ka6>Rd{mIx|AQEC z4u>WO26$*m_;m>Y6W(~b8~gv$2SIH;C)%s{4K!AYfEgh1m;zVPpRT#e;^JO(Cfy zuQn#ScS4iNe3SPVpoH6{a9-c`1LZHZ@U&1Em6CvWf~})7y6M(Qe1&eZ)~LO4y~jE-ho@O7V3uNqsZGb4s_G@le~pFud zWZR+fSIjWkf7B2rd-Z%E1P-kPhi1QWFgWYhT|@BM;*avQ|^9iw3)4qiy_>>@uMX2F~W z%b97LOL2a~hS_-$pZlg#ENbrZmS?)Zj_-pTw9D#w7V&X;`B5I+mF)*>3R;Jcx^tOM zLY9O*&+h;v@ZK#OgtfK0it6f_6IE$(s4L4k2A7Yj)@OOJU&fZJ9#sF$1<23hI!*?G z^wH`3TAL7K6xLa1QcR^@SN)IC}^3YwhW!(z{ORa z+W9Hsq@pp=rvMIuK{^_U!1WnTmG$#1;IIN!bP#w!(+0<5%=rEG+QljNvPprpE?d{u zitCToG3ZY^tITpLTz1_5dYDIN7jQA@1Edil>yJp}nEt3Hu?#1nR@Lwa85N(+B=G|Ab!?e`MZQNRAfCht{VGKltn-^Lh&%MxExR zDLab+Snv0j8(~q6HPU(Ul>9e7cV_}DVMjhN;Mi%p^!<2M(*=NE035&%^%0pn2yHJW z$G5IbwQQk6qhwyCq<+2xWalC}M|szq83Ybv2jFiM5Cdr=|DGd1;n zP4r1@6Pw-Gwy?Tdow4%k=Z9~(ABerWm$M0-VRw4sa+~sME8|qYtfdw1c?LDs|Hv{O z5w?H98V=h`$UW$^E6V-Gt(BbWZZ+}E$thRF zO`4Q?N-M6BV^|}eX z0?a#R)jGA0`rx=HB*MYVBo>OEY^#<`NTTH9axQ({oNV{r9d_ zYv}fZpz@DLk36;8 z*wu^sGS6j<43(Ie1;Ne@Lbvm%!9|O#%HSROZ$9YsDkWn5BM85zUy}rO` zc#MSqoKLRaj)j-;3GC-HJ9?h&?X04a3;9VbS-+aRXi9I8EK!2OhVL;!1v3qoE6DEr zy)ahANDbitEq$WM9)^ec`Oxr0!T7USze1OM!)D4(p}ppJ9~PW|(KEs6!aQn4w@5TJ z;G<<#qO1wf*Hv%8T$AUsu@MXe##+qxpXoGBQ8SxpgR{?<{^klX0`aeG$2;c?yr(v! zpGQ~-r#z3A-kMTdVTbyrlx*7JpK`~p?Jo!g5R;*nq>Mfbko0#}f_DJSjcW;pIdx{w z&`Q8j5(@IZxIB3Ox=g79y7eH6h7QXdFq;T&W3Jhh)jqCV$i(Aw5hDh{s-Qr*_Liy{ z#NzR2x}K|X#89^S9T?F_MI3n-ub>x_+m(_3s;W5#x!yk{3d>{H?A$^*t7Z<3TiGGJ zWw$-5aS4bawR(sM5D5g8V5Ir)TE*r%Ul+rP&JS(J%-prnP(_>HPDv)_kwb<|VeR}% z-uA_zE|QSQJYdT3iHvg}l6Tflc*`7z1|p+A#ktMLPrs?y=Bj|MrI^2@IPD@9=L}1ysvmB&Ks7Be@jb$sd_xl3#U1 z8vLjY4~o1m{mN-uNo1m&t-@aS3F+^2>UYIiotv9;2LEMiWp!SpSOE9QPo+d!Ts)c0 zn&7(@uq~M===k^;0QeY$5B96z_}DDkyNCmqwb}Y*Qo+y3kEhzO>Fp)8y!eDGX$?H zu1u1;Xz|b)OkqmBv_C5CNkJg9h^Q!}pq@PyWn~tLVq=>Ki0?>4e-0!@P(whIy!p`6 zGTN`VTe+oN(GJTV}O;~-D7LLBZvUaU{Q$$2-M)_X;9=NbOo!#htY z=1qf#>p3BvHc+?K(>6}#TfDj8ft6c9Z!l=9BzPRKEZV|BEFz&lN*HLTJhN?E@&NY1 zFN_Aymi*L@R>DqlxQX4EU*_=kM>;NhJ+Hux@rzeT=Jc34t4PnK7l6WTe{l>Ne8D7S zt^{TI$%N+!i61xKq$%1PFLx;aWncGaEH@8K+$vJ(&vR>jkqJWFW_@t`Fhl_k&_b>H zF+-@k`ZnG{*c346c9f{8L+eBWpv?oWYG&I_OwrJizcPBFN5?RYZZ zb~mr-{&23{;l(o{J0_X5b&(?o^>jR2029KoK_}C65RbMH*^r0)#Xe*0!C)rG76fdP zL5;XyD zR=&~^vW$D2?w%X(iy{ftF5g~wOniHU6EJAaRZTaOzl2ET?H#1 z?IIXEq}_qM#^OG`LmHD6k5jGNID>A*?QsA+t@5hgmB5Ow9`EgnTc*)*1TXmnQg1PT zB_Sjg0iUwjCG{u2galM^MUk_Po6=|RK2~G=uL8CmjU1me%T!9{&Rghd!7&feHIM~@ zCaqnb{NYX_%FN&9oYwesmJYpb-ILO;K{5uiPzIn5mVX^KeNq=;HA@!bqoYDl;dfg; zXX&%Td||^=kFOi7-@EKJxg)m8n@EG~YE6Y>#07a_H6EA*e?A=vFCF^ew|?B%e^(U!}dTLbjyqNtNS7y^UC> z9Bes&0HPYtZc*4noxm$|yHJR3`3asz?h6_In$EnXgOJNEcJ?Wn1ooNiF@Dq|GUvc=3_WB$3- zXc_v;?0HSW>a#L@2s(XxO-;6}yLp{vX)c!<(#h>` zOg1T(Ic`E(i)lTE%IgPEYplnw9(dcoA0=^)nfI=KG+fixQ_KrNxn+-&d3 zNOWik+Db4Sbl*^X@pwQ(lrC#c1*Ns2+jn%)Ncfw-*iNqRXg~5CcER z9h;QOyiqjn$$IEu9qlk!vycU6YRWX!fm&{434efQU2&qhiJckG)x1zEww@RVGGmUd z)LK?;bPg;J5O^ygb~QAdmQ&0H5FtKvaPpVpo97^lDOml%lN z?;W$zULx~jrcC20zpTIIGdj$zVasQiRAICkwlrLrhH2X`IA-BVU%}jttw+EviH^sy z%wT*Lfr2KrYCYL!r&7gUOvoYhax(hi$RMxOzt$0iE|`<9mXm%icx94vtA;bGjQ7V& z@KnZPd7w=B%X~8x?JyX0&0}IP3$3)r2vYWZt?;5iU z_Z9CSm~D@njEKFYJ7Iv}2cDX%o$&WYTk$b#wr?eMSEtzekH~W}gww?nQ z^8=yg_w8-3za92kB5C5YuMA!M#UEZBUk6eNn~Yq$&N`BCDV=d}zX~n9zIJ^fNP60$ zi}d=k{5+O9!_5=K*XHyvzW45Nc&3R6eg6P2W86GD&*QpC`OE3B2}qj4xal0C^JDp1r4o zv#q-MDGVs--aA8)hkF%#j%^|QP_Ey+fY#=`OV^Rc55VhJrH2{7&_4q&REhuvAKxM) zrrumyB~hs0E}e6uSl?FMzl%?J4*EJhTBmGTT`6qPSmDcpo{@(R4x|B)fU%z05Dl8tBmBPAE8MwFDue zO#Pqi=lcJI_f2C+jM`m6wEE|!3ROPms}U?>z{f80H^6oYIf$G ziOiqCr|dOI?f2b+1wPjy%=^%Ksv!4miwo59OwlKF0Wp+>t62(cS?$_30EIoR6&G6Ru{HdB@-EgwB?pe8tTznn2gjcJYP!R;Tvyy zBO+zpby^tOUuK84-2@S@X9x1W=`sp3JqCVG#hutoEucfgsEDinC9u^=Zxp^khma*8FNNOmxFeYYAnE0Jvf6GDdxAbS zt!piIjM!dN!g|r0oSmZ5KOC~#ys2JvM5Pz<#-Dr*BIemlw2nbY7FSvhF!Bv4Yq=jR ze?%k@wo$rpcV}a=k1l6u07Unf4h9{u+ihx<2&TqW8QI!wz(89#uO;UUz?v_Qsg+0S z)^|6U3_NI4X*T4VK)2Un@gQ&`JQUhiCVDr#u2lONPb7H0XsB;I#0|zfS7dZiy!TSX z#7^#t^k~B_pv|gcCYv z4wF?ILvzKP%si11r)P0tdre@xt7smLF6)XYw|aO2(`>67hVuq!wwrjyx&yXg&@w;u ztnIRYV&L^jQFH&hHyV7vJ{W;pX)Zipmqfw)NTqDTy)|wH*6$zZ{<(SYqdQj$=jk+< z_uZ{p=7PfQUCWC(9+p8EC!fb%qj_Ox(&KS>ziWe)X7+)z-AS$5X|D4xov^4eecLGB z4_@bjFz5MLTv^j$9*GUyE>@ZxAw70j%%^A@nm zn)3{|KKDpF)xl~1?0H)(l*U3uAw^`>ypV=wbjgye!9v2;&;y^@vR6PUMv;Kiz_y=z z)phGI8u+^5{$wTL#M6Yfgj>@A{JKyYkrB@=NvuD1hJG;d-Uv%|u7><|yAGDX*2bV= zg#jfvphJE|!Smk2paCM9){>l6Vw|t5ld%>#lw{SYloNY6C~pcU^38!sx$@-3?+(E0 zWF*mA+tAipNoL#-z1qsrR~H16PZfav*gW=ZKdHo)5NMJMd>c)o?2kBgGTf${H@o-F z@GK^JY(80V!GAv8Z0df7?3yA0MAhA2Cbq!1&B{ZE`nK8__XP{$BNWIL$Rl}}clp2A z*!TJ<3(=^Z`XQTBa;Zz1?qKwqXlJ6ne7e*kPU4D*o!%9(L`bW%Vf^$dl=+TAcJOZM z22<2AVIO1ny;AkO&Cn-ebAgS+yi2}6f=SoQE(Avn>izWH)29A{I&0Vii@2ABwESC>o8LUc(WQ*Ugu^NhZoNCZ_pk+W285v+3lD--%O=?HVA9pu8` z`oA{9^cYdL?X5Ng1Gv84FMek+faDA7hq(^>bS#pOodEabyt<*yxWME1ncX-!ympI` zg#4Kn1UXwaT|#Qn#+JvIHN*9VvWx@L=n_{|-`R;a20^qm`-BnqZ+ur zAKxbjon2vT zCZi}A3J<8kXiU{VT?~epS)jc|24%yw1xIT0SqX|?reZB9jt7+EJ6eu;rqE* zy32fVYK*6FEPK!t0$|UUno+BalYKI$ybEgSQ?~-+t@2d)Z&u<6u(Y7G_rZQ6vf%9NIEakc@fR zXAXWmKjjsIWFi(LLO3OOP6DtptH!cXihZ+{%L#>#H%;#P^s)|hbV}F_0`%1soToe( z>vM{i#XglK^oi%=Sut6i*+PBpI^mz7vJnCosL=t zqS|tzyvWg`@R^kk)#dhds{dHpq!_|1hMmxNi6hDaI+kE(h+8qmD2~Uhb-RT{HX5ZB%yrF zHht7 zwww38X$f6rBN@gdHnu$bP6ii|gpNv!v-3zVhYit#;I6H|9lcuzDgWcTJ81jjraF?j z4$a{x(LxP_VNst#4=IO=6a#Vs&!QHfMv^l7CrJ3j)iBug`Q%vJMbaBU&5FSXP$R7R zVHy7>U;kU}>P+JAe_}*ELQOa%h6B_3Ml-XjPd-=WRc-1ZVNygxGZ0OkvkOHmI`ae@ z_6|&r^_ckvYO|%jJctIbnsz{pG5?n0V03bNC=eo+{?d$~5bpgu-}@y1!7mmh+Bbw- zu*xUm0$HL>*hL;vh59w!i73(E9cx*{rAX=3+nKxDC}6~!a^peAvR(qD59SF@vb~+- zk^6B%JB)?oQ%L%)Xqq0Zv?g?(#uuzoa_JQ@M_uwy97Pc_1)U3`TuB7_ElJ;K;8gDN zoGqCR$zl73rQBxkY;wOHIL49`SgkJisGsdFZ^By@EW zy`aZ5=j*(}qUb=EHCg!4fROO{cDfdNNPtfJ%@Fi>S|D9VyL*|qWYU<5P_5cc%NuO$ zJ~O18<4+ey&xkqeeXn$z#-UX$bcM@Jd-Wwn(E5`n;1?p}!WrW=er6cwmu}?wlPCPD z@&T1vf8f|-rD&FcTLLN^QpnOmo!aU$)saLK>AjT*8g(a6jNbFkiMeh?c>QgC#wa@F(wuc8d#Ryh^dTVi;1FuJkgNITqei2Xc#s;Is_eRH z8Fj^8k3mRrc-Gh;>1j4P9-c@)fV&&r%KqLhIl(DbuIwQJR!ebHO<)TX2a6s!jrdP6 zX@E8{c}zA3C}9#*z=9OPep(MNUkU42rrSpANM0`XEiP)}XyOZa{+ih3)j`puRS8qW zKs+y~o)7z!uTUe>KZx=*er09xKBF|B`sP3{wKtxR04E zxL$xh)SShaV6~L&s?LS@5BlL!m>9y8pP3ft$7N8r&{-;F%Qb8QeNJrb2fEsLA}YtJ z7l`-Khp0hmPbuVi7S&3W>*An@q7hjnPJVySgLJvZrByRVNYfcfgoVe1hkwS5;eSkC zHf86Za=_1tGtO>vD}9a};QU*}fIUvim!^CCxo`wl*O&ks{8$>lvWIRD$7VSBP25an~ z6hvI~b!o>Ce{H|~epvA(a$zw_ji-@>u^-AAwQmVeALoeF9Q7juHg?5iI7|!HNz-)sI8PoL`Pk5eN#VXC@s~OV*C@H<1_d6z{{b#O zV1-QJT*HQiw7h2wKd?R0kwHZ2Yut{j6s9U!g;p$X6Gr({Dkfxzo_L*8($Qqkz4W=p zB|b0q!oH}si!tCT%NlmS+=_zu2Fqvww16UM)o>%ud6lSBTziI_80-~Egvy#Y_IP4B zf~L^M^h2vk8u;!5y6u%&ZRK1xK})J!?&eTki+!haimgc$tKA1Dugiv&qx^dqmmI3q zUnJm1*pCQM982lmPayx~4mg~fIht0Nbri0l;TERjhvG>2YJU2`R4;qNgt>y7MjsO1*W!qau%=!mX z+uKi^E%l8@%tFzo*L@MLQ}KL$?rmRFH(9ouQj!T^By|;f4@vrHTFjFet2#A{gjpPo zd|$p_4U2B%IDHrGIkmakr=}s%H)WvQw!Hf^)f0e&3+kSrF+%}-lf^Zk4!V?)%GA8G zw0d(U+#~fd-c0Mu8Xh(p>Qq3^brxzQxmaDY?l=!eR7B2CxX!+o5g!dVjecc9%lp4M z1%vj@UnLdcFrH0Yp?as${AQVye{R6a#tyEa1Dj$Akh7Y8L*#0cvay%2@gzQ?%GZ^FuMP93bdksti%Lk^Bpl($WK4 zAIp~1L4EOOb7r{YVR1K#ieVcN%>Io-dt)R9rAdUa%|YEv;dj!(x5r}^G{UFz0sRu| z<6^W9J*GknQF*-@!Nb_*>Q*_`h7^k|OJj_WflT63&d`NiG31B5&uq##x zIKG-K5{BY1DaLgbzFYet<1(h<)wUIwBWze;k5fy06>8O}q5(W}E*OqTkNVsTNevAO z6?ekp|AdM+6EdyX`RaEE$MY(?u&7Qxp0C6JW6DMn*S?h1!8#jh6DM8Y25&PX-QSHb zuQs?3@-lHbzI^sl1Wq_|^pth*Ev>t1ZyT=t8@Ts;wC z$vk5eNNyg`@c8)LJ!8zhPCtHT7{$U;*-W%K3!4{I9|&zu*H`B6zOJqPXx;Ub1vEcE zp=7mjo8E0>mq5?uvsv#yrIW@_45OrSs22Zvw|UFS`y^|beXkcXhAe9)0(1R@9s2nY}%Bx))v&rud-yh%Ppbe}bp z6}gtu+oz)}c)C0!6pZL624r|XCt9vIC7(E`jvNgxywrU1x{s33?-mq*xZ7F9==Y;; z(G!gH@xVr^+cLsOo zz9D;`{lDj&eQ&S%Fl*MDce++jS9jO1>gmrur-!Kz2z*|kR$YnXJ!o7=j?pk~;B=k) z!OnOAK(#%b(9~vYR-BUNi7>GEhtuJWhl2(3dmSy5EB`dLs| zGJ{q^%^)t=m{Q_mIpL3yLN5YV$Uj8T{pscClxtG^X=f)FcF7FF=es4DA-Rm7=;b_9 z&y4c9ThfNzRODDP7DwdCfAp1-gV)r$r3S0~cq!z;RITwCs)6Q29pj{3?wT){sY#B` zf)xGjTaE2()e%Jsnn(vG2k?q-^y_ooHl9VL0mm-Dz!=cp`9?bxN61o}6*qy-pSy{8 z<=A;syVeFh_JuM;|3;V}>@S~b2Jv*j0zY~Bn2`a~>6xX2PQ;A3G?42T z{aUDk29Mt-GkZqjuk31QRHM^qJIe`HFf+@ZwrTA5cCI4uJ38-B7x zN`5nUbuXPqepl&*r3HwT|5sbuv^Fp*@|d_=OqLqz;mXBO!nhLyz%o$dJ)lmyl5;qx zt3JZR#^zkemI6zTn|UGj8!Yl@4)U}7hUzxs+9^j#`}eZ}em^sMiFThM0=3rvv}T28 z+a^h+ZMz&NiTE#0`oP^X8LWue9TPRMyDRnbvgQEZNVweQ%R9 z->fYxj7(%>ZH_TC{14ds@h_OGA^D#KA4Z&P7sA)L`95Sca(0WoOvtaHJk;T?)zYf_ z{?MB8w(&5pHvHu0=m7i=woRD;UK+51s0`m`l2oj?SXs(m3jGMe_n!akNHaV*_Z&(E zik}_?f;!757k+@s)#o>o{_ioOer|M_@29Y+Mi;u(>1HWG^Z>QnZ8I=xUzw{cgn#k< zi%f>_b|$|@vV(i`XgI8iHPv>-5entIfXAKN`ki2E2 z9POL?Q9q%QR55tpgb*ixugG)fcT%5Gt?)Sl5>~!Koii6_9!&PW_}xyUEPC$rF^XXF zZicg$yL1>A-{!9!6;~b9T~t5j@%;1jcP_53K(5N_u0B@b-!848$?Yhu7jS7+rkF#1 zWL(ZEgSO-FAK&B`6kwpEzj1?sNd?}Dkq(zMmCf!^M$^#K@6r}H`TE}Ve`8zz^E)%RW>xT_otj$3-U zYcv)&5_vLQQ0$l3w}b$@vx$r^x94#-PMCuv`th@D$720eFTQ?i#&Sq&2jm)VP))$8 zDWPHknewQ_O^5q(9nSv#CN^H*4=88SzrQFlqC*N$FX-7ydNk}iD7^g94{-^y)4^cq zg{00lDT%$Yp=Yw4?1FpwMjGZro|}jTHDA)=nhI=S7wvOSQBmFLX}itaHH9Hdm#F?K z)o~4<=+?F8Km<2Czm1K4-urknCOE_H@+`&jRcXA>Po$^&oC*zK=>jZ0CH@S5M|9dS zt8}s2r*$~7zabw%-i8%^2HcK$_%E=8g|nRMB+jO_>fLgw3L&4``XO+IMHL>Yn^b{X z)ao8~^h8}tX6gR|U;(4@q_+PUWw10G^RKRN84(-CAV1~4dV)in{9Y=!REtDJvKZrH zBW8l@R|^~a=mw|3|HPW{o0MisD|t9HJS%?~Wv@&b;%}Uc<{tQQ6mzO0w1D8m3F?nt*psr+4t2 zM}S-n9xZudn%$AXS?(e8l5yTTs}s3ontV&s!a~U+#MIctRP^kvuK_d4j;XT&VgMsDX(uu7~Y-3 zZ3>?0Q+q!%*5q&$4L=1rQoyJ!4nd{r$U6Oa%WBaKD}{!>Qeq^Kg@mX%Wxk`yUe)g~HW8vWcsVb6?16*q&5 ziQL1QaWM27hPe+WY-LV$Hizi@Nd1LBpc>8QC&l8KSxGIF{6H$cpXi(cD;2KlEEwE zjCc&1qPtiq6YipmXTz;?_*WV^d?d@dWcNFoJkWh1@fD969FcW?xVgCIPSpOuCLm+g&6g}egE5(PiQGOeg}HluYasCBYsxotrafCGCi)%#mvY(5 zIYKZ4EWa4SHqNwkJR@Kq4_ha}qiG_uOvm9Xn~e=_(kPgI@I24^xCAZIUv=Fo57>MO z#eOakf+{jG)26>qyQL^WLI3i&H3gIEl`D6$NJ@6G{F@1mhbdd$yl4P?;6aX-QogU{ zW*4XyK;_pww#YW=k=Rt$^r?9(y4uX4=nWs#o0bY;%`?{RoRU3a{GGqA?r;-Cy8L<_ zAeOtPVX3g&3N{moc;o9Qs@RmoV<6lPvE$!%he@ih?A_> z_C}&$W0>nP&!QRU6m-=dXn$|MnsyRJ>~X!Y3gTJet&6E7@VlDM7b{rZ3;~wX4|xF* z*MTp_Ho0OZbIk*D!4v1l`J9Dckd)*@>TWug`>Jd#c^jZGnC>TMrB2|wkfuf54d3}4 zPeF8^ue1jJWs+~+Ahf%wOa?bpYyihSqgiVx!_HJmqpBP1Os=OE7V9M=_ft1BLz&(tzJv_)ILFbm7&=Z%V4^<>}}Q;b>5MzTH{>E_(>L>a@6Ai z-j5{I3VHEIKzU-3+5kGqq>ZFw<&KB`?fs0cPpUsCTrL&`&(p{A!R*wJ0gO^t`gcsm`Q#~&3qvf(z{Grm9y zP&UU|vl&l@Lwwwbgi3)rr~TX>&{_iB7eGEok$h6xf01f_(St}$K{$Si_BR90t0j%LWJ zm90zfG^NnqzL`+ap54R5b2}1e7GME@y70lMn4R(i({=umrLPNLe+6`R*Mb--v2Oh`NNSYz85E;gx$tRTaHy%ay|A? zZXmZiOGmBF)RvbO%g|-=*TCb8!fBwHj6(}K z*a3ThJH`-?q?IzT3Qc-abdp(Dbv*bsuK)Z%NXbmtrIWiKov@7o>AVoRz3QCH>*!dsP+tV-(QgecgdCo02N~h?4FJZ1W$e6{WRBm>sKT3zH`OlqJi(f zIOWLSoRZ@CU!1Zg`jJy&DE-ALsZjeAkn$#g%_BghUp~{3k_h?{&^dy;5I`fEG(06r z{UQ#%`8RQqK3jUHJ108IsRUW(8Az;4^o7i5Q#pDElp-R291~UlMOgaRZoRJX{Zol@P@UPAeTvt3u?DxIf=B^EfN*J@>Q|4FKL?S;{WU8~rl6$8(tfp_{9&7)*U-U|~tT0_#1JZM<|G zvZvp6ta#f=5ScvB-vCi_nfyTc)}I?rmVrH2rH{~X7OWD-`KwU4k1vVsNOuaSHs}}m z+)q-wZiZ2?#&TO7($AoDTX4Miz*Ue9FtaQC-J)h9vHh0l$$`P3gLmjVI~^b;n4|ph zS-Wx|Cp$g6rcGEe1^JUi2;vvz7iy9zq1^AW`~|!9U5ZASOdZ#LWvLdvwk4@&Pi!HV z)u)x6<${c0>O<7_1UsfbXR-K9&Q<=ZfX%D08mTlDi;(^h5u5IPt)A1N`(oy;)!g|~ zqMCrmDYGcP=76uh_Tq>1clCnK`9%Yq+wGvY3lOh0K>qc2 z>oi;HZR}df5%S$G^}hFkZW6kM+DUoK<|bC&$9_$djMNjWyz>H0G!GFeNxk=wwD^AC zi}eKmg1`QJxmd-T_-=A^nUoA4zcrQh?DX_}s;fqz?PzZZG*)+h2B%c~%LOU)=ruj& zrDh9fd5@f=fB@XcQ}DZens*EWSfSY-ojkXR1JxGid$cDF%NsnrPkDC2_xb&?igeLQ zv3fj<1$)aRh0n*vPQjVi5jJR)5tTo(0S<)o29FZcK%B_1tyssv*aAp?(c^usjQT1XhjbA-T#1DU>#4Q znc;N7n|COON62I3*TV+%4Z9a&Dpy15XLwgh8{%-R2};_mT)&t^^|{=kYxkQ`{#~0Ct8xeah;3shNoRU^-zVBv==iv6e-W0B+@&{HSuvJym zJ9^n+jU{NWbeVWIVhCwW0IrR-c~0=*7rdz)s9^>@-yv83f|91Sc)fbmMiH>d^UjZH z{b7&#X(w>s_;KKi#QVy7!t`YCO>v0KyOu*Ur<+gFQ>8l$WZYaTMm?CL;vXztv3vdI zv&IQa$bggPN0Q6^XLjS%_};~I*xg?&M}4U;^j+Lv@V<1hoGz`d?(E67146b}YtiyR zX~(7Df^lUE9V1ggUQ=1tzQ8?bu`=k^>_$R<-gTDg;HSNHzWc7I1d_|3@zJUbYRR7F zgXdj_ao8Y@t4+39UbAWds!Zlnn`~-rP?7ffSQ?K!Dn03Z84^D`DvTmBj1b{bnW37l@%~qNrA?)4BjMmGctD;#bUn* z911_KpS#r}+h_t6K<_O$HnWP+o{GUge_rP5I{TswEsST;r}0Fx@{>7wGE}R zw4``ZvoJ`p&?1E98eh6@qaS;2=>tZnE=LsPFZiMlwc1ucoI5UO?aj-?ZEW-?DvvHf)lX;R3(T?*hEGB0|M3N*nE6SWECzG zaj(a<&bFZ#SHvc9*%nx6@5jH5hS{Z&@&~M}o+=Ufqr-}S>Z>y@1Y9DT((gM0dI5Qa zxlvpwRjJ}x7R8y`;wNh$-^1@y#jlF>bed2kQ}N@^_g3hcdFMKEdSc3=I=SNFQ+0`b zj4BD^Qj-$Bop^BGx`uD-kjYF~6A9pT!!b+g+2;fJIpQ;m{=EeKhU%sCGm%?xrI^q@ zR%Z~6l(USbAq_;jmPb+MP$25R0b$7XoqNas2MGJaAGDv=s4i}7fG!jT(LG)aY7*oZ z+QeCm`!HQPsf|i2W3}=%yZB5#p`DSiuw-DOw|o0!J8zgZ_3k5(iu3!=<)|Lb7B#2; z>nHCk335jiAPWn!-~)Vtw^PExj*)@hlI#{6Mf~^iaSWtj`P=b!3TV zEL<@bw~(HNH0&P{^6e*+IIt6WY|Dc@&E@^dMcGzo^zXM1 zv13gr?RW3ws1`aYZYq3Dh)H}0EH4wWUy+N zE7fNQVlSu8R`?zRC$?m(D1PzA_BNNt5;~|Y@L3y8B)vUnXhRT_X$vTTypl-sdk8uq z%zQ{5M`^t6J;1j3eE-qcsVcMf+fK{z_p9prqexRizQwsCU%N`h*9ksQ{RKbQBx}7o zph}Z3b;>b2xAt4<%b1qnNn7vRX$iJgM#0tA(?$u_{fA2t0aFa`F8YQ&reeWzx23Tp z`@E_1!gwN|Q_uxg#gs`S+R47(O&t|h%U-B~%2U^|Ii8W;o$Tx{D|%{gj#PsBzaU*i z%Lag*WlhW3Z5D)EYq;LH79yCpmrV_;%%j&hWx^qp$INND&`85{EJU z>0d}WPavHOXCkZdZXvI4p_L>rJ*}}@a;o}wej~NQQmOD%2)}zFfz<1T7H2jr$}S|q zhc6isMbg`I*~6vcZ4p*c3q1v_1e#PKGIEm_Wts1ah9#JE;}Y4q^;6slo&%x^p1^fu z7Rw*I`@TqONXALkuGblHrB&*~>2&$EZrE}SC)&T);7w#zBzP#?4 zn)z*Z9Vf$K8=>ksto;3v*GkYLgT|em`*E>fHOX6@^3AW;`XateDD?jliT~}?r2WS}YQ?!+cvCtCVf)}iV2|T9` zTAffxIp<;>#kjukrtkIfO$S|*-HpY#WX}3v)TXkT5X1dl5DL?4l6U!Lj7^*q=kxZ; z6Mnn(8yBD{oKfFPkp5*R%iX+loge1AmB9eX?SLD<`?wdp2Ggh*%#_6YG$CbGG*2D-6xUm z{d_Hj7$W3xK{lw(Ehs=l@P~HB>;9DsF#Y6X#3{opqn2d)g=9-eTBU25a;+DzKF`;> z+5@G=fd27hvHsAB@(H@x<;@nVNF(oA-bbTwcRrbFp(8ofQ?hAIko*yjN9pAi)q^tr z&DPPgA&Nd!5%=N{t#7Bz`uc*kM`tczRCMsdVI3R@`#q`45bt8#R8Z5D=W;3d-4hY` z$l7VT$s)vk?@QU7+3?<~hwXu~h(%)l(;ajsa0FTLT@x{uvw9GcWR2&1Z6lBE;0fra zPa=$sn6KvM8xK0TN)M_j1dZHFQ?a=-M%v~bquZ}NC^A4_c5SR&ZpxcEUP+Q4!X#}k ziiz#_uvxs%7ozI;dED@^Boyx60R+@B`(DiBxy$a1er<9YVtA0oFSB;+6)4X9=~vGX zM*Zz%w~;uBC=3xtG_SSb_&Dqf^x?4eS%JwfT_9ko!}uz{sv-}c47|5d_2na?xM?d# zX@p&>lTNfzB>|#@{zOJ_9UhS0;p(n}g*B|_Sg=9@Xo380%~KH>zVmi=yK(Q&s2sly zY^3nH1JWVIIyF%bIX)FTjl?c^`^sq(Z9-T};IQg}69@0o^~0~5j;#>wjngctPlic$ z+O&4OF5kNy9YdcfDTyff$_h+6pE}Gg6C+N2OY)1|;qb?(+$aM|%8x2l}+ z#m*L6TB&ahiGTkV%xk9xV^wJD5F3J&jGNOAk%hGB8bu5=o>XtQ28zPT5Rk>Bn0``} zI!aCWVd3^Z3%xTl>l(*h(CY?q5liva)^Ezoh#1E>z*_>S@7>*i`USPSUNGu3*jZ8+ zF0-FaS@eR}tYR;7_!1{-M}xu%hU|q~^0@3Zh_;`={W&FmDBVt5?dMFr-Y{tsmL9X* zF6_VGZ6Iz?^I=zKca;&!rr?;XbXU4>Tqq*`Ii$!p%csfOJk%v1F!WEU^rcCVzZkac zJQX1O(rN}>pNTBbfV9yZqMW96N1@Y?OO>9tns0asuAb+&r?D8WIZD;=Bh+;mNl3CD z>em@h!GSPnK@9}hRrnffd*@{tM>kCm%iH~;n^0Z|DMWlMOym$pM(px_CG0?GOz`;B znX5|(!)3t!{`_FmE@>S>7x19XnWPQM)dDdu9TSdC$|v}_p*aGHWtWvG?QM~d;|*gl zcg;31oDsPgTvqP}yn3IGiVm=uE!c__WM_`=WzO%~T z_#e;b8YS;+BErVox~n;4YHnge)}o;@eh{DPvLoVD$s>|pba=QE84K>db;EUViBUh z?nQE=LJ3PSnF+bg=8b3N%`Oky(3Oz$!tEev$k{wmC;Hls9?zQOV zo46FR;x~!4<5p+K$B8`7X7a0MBw+f!v-l$ZwU+$vmsF%Nx5v5Tv7<2ty9mdNJSZyYP@#5u~BuE zIP9Zy)(=rfB%!YsNueP*^oN<)3Ul9JMOressUPD(wS=N%gcvSkm3$oxza<4>Wq_|(*-T3@w= z(09h-Ba&2YKL*4w8Hen2S$o7zIUTWf9C1Ua?E7@C=5`A}350t-+dzzZ&lc_CyE6IW z(UD?r;>XGIb!^fsnaX{pwFg0G)sev%3VB(`{a$ES<$hIC*ycFGftl>?ku^Ld4I=Bc z-aB`sm2|J*IBe)C;x}Q!MIzNA%5UQxqK?&3Bo!LsxS)7G@$vg)D$+Q&+dWhyXk=A$ zdeC0KtW=NdoJAc!`a{*S5+b=w)~>-{Lh}qR9X@h6d}5-`8kJt6YcG+ zr0SFg(bd$x*)*Zq5ZS+FV!X+}y{9Arm#8p?N?CSackKkvYX`WiVP@WLi+3)}_27|! zd&#(!3L?SeQm(G9EG#Tz`@?L6K0HSI(?Y2{mHf3|7g>JjFr*auX4tk}86yV%vJXuw zEGm+N+%I!=Xh77;rwOpI&OvTH$jHcTK`7?ik4SAx=Irc@goI>lS#cD4hVt2~);n1i zlKrGbad>!mKDhkXKvGn+-ylIG4-lg`M^0+wH$U+_IDA7K)@#CDa}(eMZ4c{F%$FOj z)Q&r-l#`*wMmcz5)eETU{BGYtThhs@jZXw24|z4D@;i2xsxw9U2MYC=U&h>RxEMj6 zJ~w$Qed+^c-ok6ees_~?JeIsY8n-JE6>TXO#E@h8n@mHQr$c`y-JGK62AA`4TH3uk zs#V7VF5Q+5#v}j6m^xGKw~jV2YHGaoV~<%eD~+977IwO^Bf*OqWOtZk?35QuD6&dZhII^QE@ z{;iO4&C}&goYJ=WT7|z!=fPEWR8|aw+hs~YNPWcYkG(L>qE7V(bvHn{4X~-j_v_KG_1rgj)v|H*!1Hr>|J7 zo2#1RYcsf259C3PwkBk6@&~@4idYKV4d2YlRdg5r;T*9B#i%QB@Yb*zmbF2;33cRP zr@rr8uiz0c))y%r&1i0P&BC9^C+i~AyLH|!yIKtxbMMl*y=Ct-)7tm5`%*dDN(%Gf ztvqK{?@%);K<;+RMCxj|tXP^M@jQ1H4_f3hrI=QJ8@6C@KK{v?oaEiCm~V)6!eCl6 z?&I^PYW6)u_cGPi4!|78T$@(*NT(+xPqvNA{7%aK@0Gd6r@#tY7mcr%QZ&Zits%K6 z(?cM8A{aPp%MRwkPRF>rQ1T=#`)Tc&Wk$-OHQG5Gj@0P-67p8x;= diff --git a/contributing-docs/quick-start-ide/images/pycharm_add_env_variable.png b/contributing-docs/quick-start-ide/images/pycharm_add_env_variable.png deleted file mode 100644 index f408372211400e908c414c3446e796abb6faabc8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 86158 zcmaI6b8u%()HV7GC$=%MZQHhOn-kl1Cbn&7V%xTD=jM6e``y33x?Sf~o$B7F4|@0R zz1He*d08=7C@d%d001i?F02RufN}x=z`GE@KQ#=dO6&jt+P#Oey0fBz8@|1xovDSj z3BL0mdlP&UcMDShzFg7%9+ ze6e5v{psk6m9y#rKcN}H>q*Zk*nT#P+6uUSKM%W6qQ0jR{kXe?OZ8d%&0BN5J+|v4 z`sF=5Mf>f@Rdem6%45dowroV2|Ei+EwsK{J0f!UBy)nm{?CNk%o?dXjQ})#}Yxk|E9Tb_t!cC5Ly_j1^($1Aa(xHjb+&2u-Ez$ok28Kn?!Q zf$mnbe)U*fq5X#g?lI<{_=}!z`u62VBAwHea#&`7#Mx$|e$AL6)>x*GBMPL$adL|@ z$j9xYWY4FfdJ-k3G}}Zn_Ju3AP}G6Xf@-dgzR4Djr#^kc_` z^=&u17!DQW*~CWZ$uCTMiq}D6Zx{kkC9lVJ)?*a6*vC<`nn``+aB+fAL0ioeVyYqG z&m{G-yllY;ilR)(a?+*+rxAvdOs7%cHeJDRl4NbkvZ7>Np=g#xZK8)lV;SEa z6PQI;J@FhuXFikbY#Lg7ZGV`aobwDVK{nmyIEm6EbXVw)S1D2!UFWuKjvc-9HXNiH zS~y##paz>XxH=aFEp{hr+oWh`Vlj7HgrbwKJ2!${P4AmHe2#V?Th>E^g8znlG14}1 zit-puDRVmiy?E=i-3`A%8mD==(=B!|n3KNO&_FJ~N0sNV5O)%x^V!N4 z`nCkMm`4@`m^P7WcEkLFa;an|V6cQay(kk5>z2;9+DTN>GbIS8j#di8;DslKEo9y? zHzQ1$sGAZikJ(int0DllTor-r0_-CeN zk;a*J*F1;ofysU2PLPF<5-i*4mv@{RW>@u$kq%MO;&QRN%@*qdZtQecLH}P<<+FMJ zFiJ(a=o}4M;trm!%^j_x?LE!{N};+%2&{~Nq&y8J(~snU7~-zf>c05c%#)n(vdOQH#8TAo${zyV;o%RT0-JH zncBtdp=*qiaJDt!(OH9*5EXkbgoBx(6;Po5+HcI~rArEn(lbT`N(g7_- z)K3W&3?|e+OJE6^KZzHlSYTbTI)QKSxR`j*^~us?!l}JNixO)V0gD4?5^jwk$R(`bKX~{lQVWMm$YJ{FF{<@P; zE0`1R#?jM0`eIhvduS8iUvza%GVcjUeE{fiwywaKsDUFzf7y-GK#}aJu^Yt^p1$0h zx)$0_6Y$+h6j3zI2FPM|{*6fMfMA}0WyiyeRUEaf1>eUY=HYduyE^P{J0pfa zJPiUhcCT$U0jo7}XLc!^GorcFgg=7W2-9$Rm6?_8Zev)zR)r(T3s=+FzdjODkf9;M zO4eGgiYurT3{Dg*I`?7m_mW;loK09*`_xQMGf)%dqt5cMNt&RxozWIz&bd$9#rcrn zwIbp_6uBpq9*|xclRBx+9-ZNIWje>c5+gx$80;IIo&9;sb zbfH|Ad#%CI%{G%ABNquvd_HdjWo2{2=~|L$5;zU`_kzz3zXdH;AlTuO0FUa(luR!b zxBlS5eZrOSIM_pN>Jq#05n|Pb>;`KnMYxjpjxZ-uwXq%pxl^#hxe{~QZku7{@XC;) zgn#4#+kfY!C8CuQfS3;z;@rAnMcYEiA;drBqiei+eI&Ico`_wtj6PfMf%7aOv__5B6fhXaAE|jVj{vPuUXmIP={roqTNn|2 z1CR2)^jOamp)B8VfxO>jp;l%C-vh0;gUH^VWa+$`^J<$c$l5UR96IbZGjFVHf4}6Oa%*{T z5Ic&sg>;AgV{~ls5O$j^6}BtePY=)iyxjTWya}VGV4w3y3L|FwMEfeMvFHt3z=4gS zX~T=<;2K-eUm=yNhyfV$s1}qpTLt4MUqR0!veTD1 z0vvvaOXu{0YPeDHpi#|nKGCpk=36raIO8f~!hi}9fmmvu*qAE7`onh(3=ZxdqL&M_ zX+jTT;`1Dw&~u-7_eB{x1F4MELM(u1$f95j7@5wMW()iAHytQk~l>$d{d}R+zQ8_`k7IOgQiW^amyGEvhrwE9mw+d_Npon6%`R$sEXa7w?G@}2@ znDF3NJ2N>j;Xrl&XemTooua4=6b53_P^}YTgk>c$7G82g1u>x+p|D5Tuf{emdbSGH zySU+uS40S5j#PZ*4Q%zWDvV#4J62ji>4jdo4oqOdTZyPd2=1VUTlCx@W8gtFp+KiA zVB{dI#60FO<{ecwN+Qs~jWf?lEDYk>+z1{IL5T>WKKP9xpzIBO9KnqyUjBz+uo#+( z2%rVE{r>cRCmfI!+jF!Z2h^>Q$$bo99kOpblC3NbF78PS#GW%s!iPI1eU94}^S{B+Bt~ZVz23U~38Aq4qK4CxLAYg@4=DpAl zr3V>>F|fPh@sm4#tu|H&>yA`oj^-cYG{l7yO9Yno!Ql8?DcG5X%J83nahF!@lHSqU zQ6vNDMW;mc+mACN=|tWMH!TZY{8FALKBeDR+3hO;0VmqO*G>)qe!FQurd(TaaY&>% zThvjYnHX30L!~VQD;6=FDTbZYxf||Yp73kw0E>6U@34Ifxmj17pR=f}5LL z;}_2z=}O)epzJRQYK3p2gBwxc_&r2!l9BSl(dPCH;I^Ta3E!vy)t^?+N_pY8ID_f| zF#DpU7Cj%%#{Tn_u5HTz;K>QHf;M|Hs0j6-bS4Y#M|3E=P& zRN^KCU+`BWng0+ThwCt%F0tDb;4($vKtwKXC$%NV4fowp!bahvmv8LO>skY4{->8s2dptJ#+wlgWK4*uH?BKAP+aiuSl%v2{ zi5td~6rlWoCF|AxjHQ`K%LC?T!{3H5KICK$S`}?gLy)J3Y<}!&$e9d3WkMEpHdRPz zgruFdibU9(?l0baY8-yYyrT_IrMgCcd-h&9P)gy{cBZBClZLs6BF-($1|B>sm@_oqO(@rN^urCcmeP{zRxqqR#hID*tAA zz2M#LW#xwR`+QYzDmfP_had-w!Yl2>IVzPdK)y(@=fgIS5hk|?E7S%0R`@JOsvZwR zQXM6eB7 z`JAbMl$8lzX)f%)=m+EY;3y0s|EG)Ks{D7?2!gmlsb;TYRZ6NR;|&E39VRb7ZYB9( zkI_>XDzW6i*=ZG|yM0wbld+*sJilE2%z#AfAdmVV%OJ{x(Cc`i3Ok{30&ryYT2s)I z67LHfM=-xKDPS!X87--LXA1=jG$=htk^)p#3Cnm}%+(ml@gpy_k^toiz{JT_Q3*4l z@?ZGG_+smBv@#lIMS~cq@vM_*EM`Y1eKiMo3VVIMR!$^)#a? zGbJOI<9@EntQT;T@Nf{S$#{S=Xj;upEb}U(EWkg(15&Gn2m(q#Z0bAHGh{4}2zplI(%pBXdLYeB90?&o?v&iA zrjaH4)yA0#=Fw`SzNw>@QUw-<-w9zccq(*H{t~Q%2+355kKA=ci2&r+R#tcO9j7y+ z6Q|}GEtFva<`2&OmbVV##%w_}p7KDyc|@_wjoRgX)Fgf-BO|<$Z-iu&Jc!-D#2B$6 z6rkI-zKfpv577fL&(wO@@<)K36q%H&!F=FIIw6Px^K*ethlZ5X4l56t~ok&Frg6i*Kse zR%*SYplhnUmm?E5!BKPzJ!JRXgXDAr{z&VF0fto){CiXx=AJb369xZDL8oWKQ4IF6 zF}cPXX?*UWHOH3g?@62N>*ee6j-rJ&a%fl66L`{f6I|;VahBJq@dynu$p1=X#^KNV z8{Z7O-;!Kf#p3QBP6ajw$>4$2t$2PrflmCMmX?xpWp5LG-dqR3ct3y7Nyt1~d`A=Q zQfQ-7-r>Yo#WLLdy`h;8J$Pmjs%)Vu)?%jIUyB#$!?u@tTirMlwjE=(#y0=K*1)_i zbH6!?N?m0#&h_ph}#LfFi$^+ zhZD=r#x4k`*L3m>s24`SJSqVPVuIWPMQQ6E;spkfHOdS04)yk1oz-cb1^Cm$jvUg> zM#|>?z43I{>(oVr6}|Q@-y9uCHCSBJb23noSd{>is=WlG;gBRiOnqd{jk%T|Aj}Ji(r>sxw0g_xX20vbLfr!&$-JtujG1)5;22SHG*>?vE zvV)1qq!)`FXCym*^_=*Aw{6Gzbq#N)Lf50*mF!k5Hcftc&uc6sD|`^)on1Zux_c!o zp=Pi#JjJ(a_H`M~cg^&@BmBWmnxduMWiRCevwHixs0U#uEX)G2fFGBT{;8Tb&((Mq zn;T+Nl)HJUw*Lhs)@)3;CUE0#M8&hh;HS3u7&yMGW>e ziuB7(74}5S;%{5*uH=ZgR}c=~{hdtBP!zcDH0#cVg8rk5#p=$(+u!qNf{%nEC)|!z z`7}>VXXzzuP&BTEk`yZ_P8%eR!n-BUq!V;1qS)F-g&Ob0pwoU8&MTm4(DNky$s~D5 z3&J|eE`cEM=UvTIUl;>dACU`~H$YCLtE-G%Jt)4xTjB7m`VebzTd>yyYvS*Hu{bHa zeQL*nh0diShDHxeh<*_>1=3Qx2cqG;4Tsg&0aG*P1BBrypN@(Tvhe{ps4!Em7|=LR zWo|N-Yph;|e*0$n&Y+%Fk&iOY1^ymFA*TjH@?NP5xnQt7RpXhy z<0(3`g}9C5Qa!hj~oipeNJAR{lm*0J|I&m-Y=k?=unb2_We@s`zOOC?(2L$t?4_Qq`gh;YU zW5~-zU%#Tpp^j;$EZfHZJ|ED*95yDKRL|px0!OL`Kmak8ljp%eD`>JPN*9{*UG3sI zeh{`9J0|yz)cHXvi^t%7!27sa9hH!77RuwCc-39TD&V(5Gg~%)dRwnYBjew|Xj{G|n8-PD&axyP|d+@Oy5N0orI7uRqC0HBfoJAnXyGckTD ze>qFYi2OQ$hJj=u1PvYJ2LSK^62bz??rRsDZtBPq*nk^1wrEfY1dnSS{yKm1h(m5F z5A|uKi_a-Xx2G*0o^`Y9!wXo?YeH8rRDpQiq=cD-xaRjyL}8RLztYm{>)Tt%(UjHI zQPZvTRgJx^E>FoK#WLk`Wy;6z1ubTod1lxg4O7$Nsw$cVU}piZf#}j@D4mwxl;7Y3 z@EPFT=t}+gMccd{Hy43w>0pi#$9x_RZlWvoGKe$A zCpiSR8#SmNvm7mKbnED5Slp)0abR!~{1ADRp4s?Zu1);aFEq-~0QQ9yu1;bCxreMqQZ=Zsp+36T9YrD49i{PW+-u_|!d)AX(qj}M{VW(GTLs~(xfs4@QgiXy6pKZM}{ zN0=HJw-SGRR7ItQB`5mDMhm{L&hj6_lPy%{0*5u2=Zmsqd4VMB#K~1>xQVQgaRqhB z$0A}U+)*)os%&H91A%EYng|>{y`ApMyJDS^_~|*;)^J1whQ{ADc2+jFRXf_(lZLmt z=?Yh`x=KEOlgmFF&d|KUE_pyseHH5!s|8MZ*njkXIM1ZumJ^dweig+4Nk)0e8D z8=?L<6;@pm=g&ysR@^jVBoL~MifPk?G>VJN+ka?wSGD<-h#3CoO)LSYOOMwEuqyyb zg@f)xY};ZTUc^~1uee>ZL8|gXu|aRDH{mi|8H^=ASpM;boUCuV;j*;hcQ)LXOr*V$ zueE=YcdIn-zo99*JxhFgnxB|w3C%@(EFZ=RJp<2ueaNHy4@_0NDiaMAJdkQLkKkZ2yrba#HJIf zds*eYk^D@&J$wy5)QHYP&-u# zgXEhi4ji#|3Xttli+DY85~-C>(^gm3G5qLhy}r+M-0mg8?+foWp7r0TU%dyke>MjQ&vmLpDnJICT^v#U>s6{*+ zJr;-EWQK1>p+d*JD|OB;D_Ls6+*WFAWJ;z!mKXbE6nK@T5x!_tc@jc;zHYi4qeak! zZgih|D_KFnF=a&r@wiUj+jpqLx}%h-s6i>=PPlY*bTypSXn0#-N^J+;X*jnP>ukS= zx+vxt>*^Zjh#KA7L8Vesiw?Bd+gBh@E?UEc&Rw_1>6#D)hvzoOe@aD67f@hnd_ln= z{BtFjJC|;y(eduCaWP5!ymn?2EWyJk@?j75r^|)nTbUImPrC5pK=h-? zp#IzXZ@zMBY9u{#9Tnl=dYI6%P>`O9{6zVx>9X9Tv1SSQPXkCi*@ugam|Y9ouxG`*73;IvG96 zpchJH+%gH`q~%U@HoMMlop2r7wp7fIi7Vl1B-@{^bm3_B-0nzxo}VW!WH^0K@Fr*< z&5#i0C4tWkOcZv4(^0<8&4yWP_Uz_{gJ5ICMMOzieQ{toow^G*)X{9grBSg_IWOR0 zvYxS&8w)a-xF#p62^?iK;cz@|a$5iqp|cT50CMYVs{eM-k#e3T=PPuz7+qkf(!9k& zQHSH4VzG1fR1O$3Oxj<*9jEd0d|(j-`H&M!VBAF3Gy?Om1`;bP-=eDCqau2$ zAqbXAqQJ9#@>;)?G9VvUbA1XB75dwvmS+cLXLOJRsva6cE~Z+T^z{!5{yo&z zGip>+v>}B6_8W@`A1xFIQF9Y=P&GA8S(um&fxSB@bUwS>lV;5T1?c~MVY#0HHgVJr zkT&Woomg41j$pQ3sL$>$-UioHvI9>>z2{COjK*$yV!;RUGudJuiToAua>9N(+>lS2 zpw_l6jYJ;Iu9h@i=f+5CQPf%+Rz2Xc$jdo5GN&CW-^JZ^ITM{$N$5ogCwTIUV5i4w z>Kxm8zA$^YPU>`+wutQk>EYZ^>&no?;QD9%SE(u=Z*go|N;Df9G#hQp?)t_#c;5Ov zOfG$NY-9)IDpOjxG~S)m#hJFfdRWA$D%;I5QQ$kn!mNPK>;W^tDo;(qGTEh+<1-D+ zIX+DO1wt2$juqgBx^RjiHY4{e{!-bwH*sS41OEZ z;x$>LY}B~rR-p}1L*!j&M$#O+#3+8YXLtOIzqcr4lwcryIR^M8UY@BZcb>!Fbcsjs-qsB=5=%}OSev&Fek?*Q|c-&~S)NYeR|NLm( z9q;4g2;FI~Sfv3G6_X@>%+?5ZQr`bMy1_&8a0@ce+UPstNYgmmwSoPr>9c~Gt670K z+Bn)!AebAS7;LgGSKV?mG;C{diY+eE+BTyGNz%L(KqI{tUNAA2Y1Du@8cRV-&N%F`4RJM(n?GP12` zEX^>B&FOaY+&1lgtVH+)?vsNcxZMPm3r^o8nc|Y1Q zcW38*UpFUz6V!#Fj9-Aly2u8`SGP`XPXbLi95T0Gtmcwr z78gghf{Gd~v00IbXroZNWF{y{YCvQUXDy{QVu~8e&30WcjB0sX(S&RaXCi1KSREZ1 zamq`ZQt%=u0CGlKT^~ZAl&%isXJjXBp}X>g*Yb(wRTO)kwV*~w^M%^@8t<~-go0XX z8K5erl7C?UKM3H&rR-}~!Lbd=Z=tzzowZTSh$jH4xQ90X*7GMRow9Hm-9ht$-PeNz z=44r3yqQ7m{h!!q`?y+aF%9Q-j2F@2edCOrYAdS^-{avCYuCfg?CTK}{9ECLv;nC} zL1ZIP{O7McKT={>ksHN)HSFNfphGtVA(6<~Pkg??*qBvxTQlKm{(9$KdD|Ikw}_3V ziE`*vDOG>@n+v2on|yn zhUb|P;kfn2Oef9466;Id#7uY24E{w)g|BdP z4NA(96ncxvTqwV_(G|Kp#>ghS8-AVr!m*M9p;57&8|T(fTUX!P(1*yw^Y7s4_}Ol= zzkFwM2VXOxX`Qd+q~qopb3~*6U#7QM#h1Yf4+*8`NW1@5cZ0Sc80F;=do|AMe5K&! zVMseVi@4Id>Y-`b2|=aH6ERV=`ulPFT8qFrY67%#IX^S$0EaZA#A@&0NVmo!vN)SG zQ<3F71mtT^^EaO-7bgdg!pH(*8b_La5BOfZ;^^+hm|2N66O4p+CH1ar%l+)qH|_#+9RUohfauf!@#8spWoLKgCDsQ5r!{fW{WSY2*{&a7=81YelI4T4|@W7$} zUp)7&GAFq*P^O)HdU%|x!tD=8qJx`SK^SFVm^+l;n^Oc4?_!FA}@i{rMZGQh-T^?NP_J8o@{~z|G|4GXK z=l=f>m;PV+CV=@_r~lWi|0m7)?{eljDqO=4;muV1|jQDN?Sy~P7CU#ZP|lZ(jDFDNKdq{x_TuCHfgW*(px)C*H2 zC1S?`HwFE9uLq$Jsx!D~tkaHCiW~|Cv#CsqaFw-2ivk}d!uT8aPP;Dhq^o2k2vDP; zCx z>z=Qe`EQ!vq;E%(`);V!3tw+;GGI*8**qWsC8c@QD$TO(@W{vx``x}UM56AJN@S?u z-Q8V(e}9$^jd02Ql$<>hu!4QM~p672l zU)}dT?|2raMq|cwabjQq2nYyVXyEQxYJoCkvLp$h^Jf5VcKZnk2ySn0JKV0TU6&;_ zHL(EdnwqMfj}H&du+Zx2>dKz`$J04{zF%H4!GA1QS65L)USD2}jg1S7iw(nnXo}O} zVOaGeAIAWAbS4mj2h72N0cP_VetZOTby0GZtsdX)?d_gDk%42e_otOf+&@QCz&(nn z_=I*7!D4lo7qgS)+SKQtrJhuTM5x%<*homg z01yH6EG&ceheW)k239AIWfOz`!~5ZbhJdiBD3r)S2?+_P;J&f3GvvVNGceS%CdvqP zUJ}(;X;5H)apH)mDC4@?>vmQA#>Vgw5}~6PJE)ZPz%2yfaSw#lhUb!v+tDCFPmI^}+|6rFCDOs=(Pf7@t;7+y#)>E>oEFf|wl7*JZ7!Nqpb zWL0ZE;FjeLbF5I37y^{25iLgSLmYX~5}Pww54YKDpqSXA;JcsTt&rm}*QQ;qO*=g~ zNfWLG0|T>S-TYQShg+Rg^_=QcQa*X}w;)W;V&OaKrI0yTu+H<@)}=BZ4ITX#K$^5c zBoZwLsiIVG!FoUu+6WaeGdue;;{T}JF9rTSW%H4{9&IQn1y)yoNXV1w=Q+_ypFcf! zpD4#?njCCYE*FX@1>&*;64t52CU{=RS$CVnCGV-vL!+>f({P*rd=G|(zfwbQ$Bj~< zf1N!&{Uw&;eZPJP?w}PlOEl%h6{S&Uugn7i!tY|Lv6;46+mFq9DUh9H%LprYo_O)x z%|54d(m?C3$`p~DQNW8onb^ktT6tvFAUDG9Xg=X`dd);2dtB6fJQ_N)U|PcMz|N6! z08^dX0_y+THiE&oV5g-)0G>4{#MC(?v@KzS-6$35ET{8%gZ}D zI$DfTkt9TW?5#Z$p+-YP%VAM{9oP_R!dUlyX$Px<{>Emy6*gVH(hD%t->!Xc{X?Zn z^Lsw3was!QmxX~6p!s(9cB)0#;xmu_`98g%yHw*Eb4ekhmCxvdgpAY;ubV3wIM<|%4q1f5!ND$x#YbNG>}p2!Q&kMp zCB4=7^kI90F^8;l>vw!}?`+NU0LeL9^#@m2Yx6@3?|6X&=a^*Vj$fP^I|!OV-B5?~-kx|3e?zBmxBDer zjh=T0#+rbs=Buf(=hnuD7FPPIPHUKAJL7=;F3#B1K*oU#f(KppQ^y0NDt~1qb#Bi0 z0fZvA;SQ6xVh*Otzi@#&aj2tUVP!78*I+cgXeCtV>Y18cbi|;E{NvjqePc0)_ZIQQ ziCSVWXUn@Ii|Tqu`bfns_cF;Grg%;I?H>qXpo?nOaGAfo#$VM8bf@A%C z;JRyu7IN6PgBObJv7-d=J9?U!hnn(Esgi~O8%ebkcO7R-RO7Ox%>7_8ncB9Eh^sYn zsB^oqRh>z!tGl{SrsE474t{@=X+lyatz?;n3O zI*K=ay+sOt6p=Ejmf+yvGxgx%{W81MT=MeTT6vvK8c3t^!Z*IoBC47kUZ$1f3MgNX z>#<*T5!KaKxdMUY5qtLR9v&Vlo0=kQnG4Ez<7Q`SW$~liclY;3Mn+O!)ag4I-ZC)f zdZ~20+g>X{6u6(PVCTLFlgt>Mwf=7=biR8c4J7E*rco{pKlpm35r^G57ZuVOJ(0L( z7Lw#&mowP?}$!(}%^f=xv6Mo!%Kcy|5 zv19OR_<7J&*W9Dm-pMVG4oHG$a158-Ek6lvZ;MV>_sep3xNPP=(G{k;J>S}_rM2wO zewvGFUWq!K0$XOI$=T2;baJ}3Q$=NEoi}kSC-5w!tI6q3Mt*rORA9B% z7`pt#{)WBmZ}Tz{%&OUa>B#I zD^tcoM>jJx4Cv`@0<}P_H`i5fKuk3InsW}8eMP5^Fywr5H=maiCWP9eGYsm^2nE}a zmQ)eYAq80MUJaGd74_uxrjJ1MeBo>ixj7d{MnnL53*&oux>)bZU^4#%#Z<*Uo3Z0i zejY`$_nFCty?0>4ggeHLv}LEleX5h^n;#`d#wQ&g0LeIyYZK~Zurg@)m<#&Cr!f>} zl>jv=!tBPWOzJiJl!$RYCTI18Pgjabl$z0(Lx|@lQwHkFKKlk>nrz? z-;OLNGeuGkObL{x7GPu^67%(#ch(LfZDTL)fmudF`=wr+s=gDLvoRG6YU5OtGGnAi zL=@ld5`O4frgl>HSWZ_LmEaq0B;Wnil<)I)CyT7DQVxtKnUtygq8z&s!NDRvK0X2h zK&j(2f7FBX@(e^pLzF2OXJ=pandMO_DYs3$?gXagEc6&u6^4;uS>BTL`{(BFm>w&V z4zx(_Yx8d{h$Vpe4Gr`8@oSr#hjO+xQEHv_xI3aBdJkTg?q7`Asgfa&j7Tc_{^8O9l1W7QC~{~lOiT()R&~Uhc@vXVA5ltc@`MJo zB3M}1DQ+%G#o1Zw#qV3iG7<`xNIB1+Qz5?ZKc~h;epP<@!?R{;`koAd;eHx2+~ivCgU z1*zZw#clpr<3F)$PPd8_X5Xj!H<%>^M6l6d#E+bpE~}J)hHUMP8^!iAl@h( z|CKiIF9BNNvsE!gwfBen$J=+jx)l9M4< z0{NcL_m(JMUaHJ=h&3mDXnfq+6O3i^RUrsjXuwk_fX)`5|5(!iEbL3y)GhXuv@*D-K~tgp7;N7hLU?#oybFm?u14>yy5N zQqYjdR1?@1i1;HK&1C}9%Pr^6SX2Rz>p_vRV|c>I%J->EwpNOtF2G!I2%!Pu=R?tz zr7mKz-56Z|x;`kO`g3JRE_f2ejsk7BW#ya7`g@TAXV8Trr|{bG^9PQHmRvBj-qwBY zW};sO_*7m_m=yKR`4u(n>M7`zea$Q+Oih6Z=#k=jIf~(x|Ic242lMSE+rxj6ZV61Z zB3vz+m%mHb`8!|wHr1L_jUF{+?%!8zTpHQAv?Na;Vel=^Z@MG(n6Y@Aq^Qt^O#dZ| ze5R)OmK9QV89Tg1YI(U*zYjiaHbpHoR47Km|J$vFY_H{)26;9*M{IHy^^8$@Pxo!! zNNF?!{vLX$`Upy;$A>r84_y%-~MX7ix`zGP*1J6JnaN{K!xrtj+E4pU3d7gVhZ zQ>C0NoZycwbgXxjY|j0<2&67Qy0%b}#Fv@3+v=nU7v0<6*X!|t1t5hNJ^-7Vnl83_ zAA0@{`#cr+ftYWXE!%y4eE|UhwR%NZycn6J2oSuSU|eq=X}0}nFx&6<5iR?f>D4ah zwa>rP82%A5_lky6_+scOFG}Ifd!8K7zJtr0&%8?HJh{-gEBk2TR2*K!18H-zSvxlX z8D!27DNt!GU2HzV#3Dk2Uqk6`Hd(CA%&5w?kFlj@Q~JXLq}}xFrlvNLziV+eHe^#N zI?&a;gsFQJe493SJxqtIS9{zZcM3N*x(>m{Ubu-d4#NNBD_ySL+|l!|5i4_exzo%htNacWK+E z+6@uqId(Mx<&8$6ga<=98Lj*mA(G7($}8Z?3U4fpFEd4Iv)YHuO)dKrHKH3-Fg7Lc zuvso`+JmO4I4sE-Sp~%IYurEd_2N8H+45-(MJ*W+|9v=`S`ls)6507BU39nDO;XjI z__4zC6=`_@H%2l!rF5KETH08Vw$k6p&kXh%s@RKI9EQ!7!p9nEG!>HWDs4z=tr_vT zzP|hi;UN-#D2ar#@F*H;>V;=8{Z_?(b!ye;t4#(xd$fH#;Ffj947UGAOzM`)TTIOa=ILjE0I?nL#pGO?ABSgz=mgbmP z-5#t?%f*1>V)~zRipZojTI`litW#wA-%`M%gjpj2nu=*&X5&v)7JnpTdZ$|U_xh#U z&-Rg75Z@9Wmoo3gs>P!?;eS6V$(N1tlfUb++2lB0jV$)L;yOQ6RywPQxl*s}@fgL) zl`TjAnmp+!dA`fmX$Wt=6$sL~%=bz!85-^y3S_LXo^~-z3r&= z&+gDvLbbY-KiwV-!!}v6Ergk=D?}D{l}mq>&IbA2?Y?1Ga(@5M<AG&QtJcy%(*E3nMK$ z7|b2DHWod( zPOs(Afr(hE+$S_&Fa}Qs{|RXEmuw^kT_@L?`AT0`y9_*4fCsC~`P)0c1=rEibEuQoX&sVPazbFoY!ID{KE1)hbNIE}Ebj_~7YdU*~kYy*&g} zR8$0no6!t~t~EZ-ekuRJqIlZ$9E5JO$tN=x)eSzjImBV_ylWMA_tut{XY}=~A5t+q zTw%MrG|%^E(>@uJjtq%4h1mxi%H!FGFG~i46I=T#JO`fg>CpD*6__iz_ zo5KH72g}1U(%0X)zK(spIkYg(eB0my&G~tPVSNuT<>OL(27}zQjpvZ%%h$YQ4*?F` z@We#r!E^#b5mwy~V-7(83b+8@kJ-Wvef?1cw!8|G-|^aIWbW+OgJumR$3+8|E6?oS zH<>e4=;RgNU-WB(fp5eCgUS7CwRS%ZT6TAD7-T%YbTZYB8H0{De5&|R;9D)aT z*Wm8%PH=a3cXxMpcMI~=dKIdH5d4GKMXV#jvx~H}7>Z-eYq`oG6`Zo+= zl3zLBYC}z_RgvT2K!-G<08Xc%>TL#`8QoWj`GzvV#x&X|oZ9ybDyD1eN&@S+@s!K- zTo?W}dTr8~y2AhT0k~xj?`$`eTC(66>95i9PA8X>Qw{l_n9b%eL`d9?Z&-IMwT)X{|Xvo(TSrlZwelGS-T?d?ZqTC5R(=ZmK%}`$6r*~T2g`Q7g;{S$X zo@Bb?+uGQxm0!A2b*fpYsim$%D*dKQgPU!T=jNxf{L?iZrD9_umE+)SesFh%hs^8G zrCXgH)(!^A9S6?s$J)u9>=hqp6kjml{afixQR70%^yO+@p~E1MU-EphdV9854t>5Q zC5i1)=jEu?I2FeL(<#AzLuJ0pFqtsZMt&%Y9c9afcYe=CiMUQ8eU+o_bs1Le*0Kos zvr*r*D|3V0X6%I3TALK8H7XW9Oih9UPGi{0^yIMF?fSz zj)2$I;ch7!5@7ILt6f3(^GvC`np$|EkXX$$UEeon9+#bJe~y;Xc4`Q~ll|i4-^}-Q zms5cRej?<4;5u8W*vvW`f&u_ga&y5aRLSm^OcgVzAgW?9-Tt|@#X;pl*&7_{-@x25 zQiYMJX<$G=34X%psJML@0y=t0d3m{Zd7YcekSTVk9rK=Zic#tNZDmhgbUF1~8A-(! z2_NB@hqd;jPkS&KAfP@l^>rfsh2;xnr|q8X518S?#L5muu$g)dw!|2b&c(3=cEk{V zyv@?N@+C{49?Q-9x%O!6HT~zuFO;qghUf0!gZK$=?bq>`cIYqF=XbHLLd1-WzvR&d z3aFZtO;#~x^n0TXsWZ~j=j9W#Lh&3=J-jae!&qfJk0qj06rx~rI8ko2NUyHG6aobZ zeg$u#+KW5Qq`m2)2%x-6i0~l5+^qZwAmNIL34A{se7Rnm|3WzcW8Pa`b2eqyJr^2- zK}@}SSL}H>Ek;Pg;nV6u3$`9{p=0yt^7Pb)LHUCUQiF|}dVw`;vy(~F?$w9lRiRCO zh|QHs20TxsQRTZU(;nDphNr&Ph(d5BwN{XPo%xkRr@EXF>OSjPu7_#c&YpnE(TI}l zg|4>tztPHr&mhf=&RJhy|0?`&5!|XIM1hTs9UrU!_^aO7WQ#w0hDL9Sg|tfcM(lAz zr3eO~L>*hfRsM9JIbhN5X5~l!S$Kr1UCjv5?k&ISns$GJQ}NSG{xS7>JaiwVyEir% zpiZ1fN#Af!8jzPn_h%oa$Z9V4xKz!Pp0NpNALf!jrwznE<8VQFC!!{Q8oiRn;rImbqIuQ-L(CU5 z&QWVJ8UcwmRx(pt7|E)tO@{^m2nf`KY$I=(C&EggC!XY}RB;e%%eBeC+x+w_@3xl<5wc;R z&`RdX9#`-PCxhwq1W`{K-CJq{xd3>cK&aw+S&WR{zY02=dXt=;Xu2g z#V76cSFq2^$X_~$9&F@3GIZ@O|zyP zd(w3I+Vp1coo|Px6Q3@7=GtG|&>b`|GsNh{*+$c^Vt=5Z{HxD8o~`ne61H7Ezqxo@ z{IWVJZRZFuz0bH+0p!W-YjaSDMTD+dJ!PMEHaakF(V>}j#23apME6Il`Kc^~FGx`y ztjrEYjm%FsnlHL9cVRq3u+mhO{j0)n{NW!TA5#*Ag@rHf?%pWaCf32aeWj5gxV;h( zf8KrS;Zy*AOGkZY@XaP~uit0{OU*1@C0cc>E24#vqI{wS1cr?1u6FW*S+3sKdKqEKr0JD|n8*(ovCK%>CzR)m~S9s>grqq|JjX_8FXLF_)0^ zenQIy&!v&P4|SF8AAIe#^50~?5l*WTCY=y#skb-MXv4?OMpue+((XI&l2|R5fQA{v zbn|g}tADlDW<0(hE<)z(PuIIucDOistdf}VXwW`P`rXQ+hEdhY&F4#Ja_YrNFIiB$ zuJNgJW1XDP(ozaBS2A^)l)~X-B>Yvvh7N!$dF_RLKb^N<$A3-8#z&R1n=KRwj^2d5 zMw%-F%2`?f-HNZTPdYuI^g7)w|IS~3?wFsVRqxnahX)l}9)DBR&W>RZiI5J}+) z_|iqeac}nm>g4J50B}H^Jf4F7`AeS>F(AsoH-b~cq`e{hT{ZKwR3bz*m>0?#kj{1d z3DFV3hLrO2cZkYyx%mlWLs1e7sk>9V*9Gzgq6KYG@T!-))%*8#W&jKW;XxXC=l}(a z-MzC-LmWfd`$G1Nr*r-}+Oq9gvyvr&Ux&taRIu3Ijn!@mV!2yQRq!8NKo!PVr=v%n ziNayK(socsQpj`x?hH42yWx=^q0ku-Djj$OjF1H2zb2Gi-R@3IX?~ewiyMJn8FgDy;t+Ug~ ztb0}LP>@o9P{B`c=g!*2%kRidNh5ruBR<U2j5kyY48oc~c^g#|I<+HS;F# zeTQA(Q~6pR=I1EcZb~m+O6H=O@Ttt+W)eRG#xO3X(ZkdmUY#!pqyPY)yY697kN|<{Xs;uuXNU)&`$lAxgm%iN(nq3T~%e^)OekrAIP`b(&$04xq z7o7Tj68)NV$*$^n3jlb1#qRzd_||l-{7n*cw3c5&v~O!5L~UBT(t8jm8;o(RVv@1^ z-7$seM5Brr7Y-*FCrlMLh{E#pH983#GW**U0}MAD|B`6))voU#Zx7MfF_ILf#5d(cxza^YnYUV;)&N;yOG!lO7sU?U04Wma-ny$O`8&7 zvggHBK<0LkNk~AQ@gRd@kN373xx=l=s>6W~2fVR>at3HdXgA507Vp@knL%WeY5WSq zI(d{=-=Vi|u7(D{?Y1d>jU57r3b?yb(Z=P3#uq?jPspD5 z2~m-rOZmB){g7s<`0EU^1TG~_4~^oZ(39n`J&#uhBJDOGX$2#2JJ^Onu6ms@-Jw8a zhk=&$<=^lNxx0OghMVQbG8@W`5e~QapyM2D4Gh-8or)>;>2>AsSt;-I^@*etgt%Nv zu1?u->lECj94o(K8(0dpBdB91j*pB{bA5S)7Lu1^$diyO00VT@!a%3jKR?B*tXDdi zv!sWAm@(Owqs$KH!`JMhJX0Qm(6_#b1fow{rpr8mbu*8H02|d9tzQE`W4{v7*1jFj z=QT);*<$;XDjqD1MYO1`1?n?kLcRLK{)DceOT7k@QPNx3TSr@5?TBIe{83eQC-LIvUw(_zJ3air3Bg%YVu>H0HMlZUU4k(EstJ;rckZYOgR z;TI|(ML+$(018GR7fA3G6vvF4x0cwI>gJ2-uLY0hIp2b-KK#@G!3v0TC2129MCtX= zea|XlgM1?-==964B$VdSdCA{4T#*9k;M8@1+>IwYFuTr{r6!Hrclr3dp?zvx%k;9& z!8h(`vaH*4dALTwSv-)i(Wq`^eHEbhMJk66= zG*QJEK039WEG9Fv(m5EkoFuwF7Gg9;c#?ppEaWcB1M~WtqxVx;YsIIZ)iFM=+E1o4 zXD)O+%$$x?S8s8>>yD)r@RUIsFY4<0e7^JN3ijR>hg1!{wz3h7N%&8}hhh~ZBqXd> z-sLJ4&z1%)?M*aaR}M6sN%DRWDzTvRr3y30Qh)LizdKM@Y;3d?(@K8=PD>P?5hK$` ztiGCE#sB>b0Ie=gQY3x?R3IXTTzAbFmep89v zk}@06pyL9SCYD=YH~x@wzOw278_W@jFR<>Q9S9D6*u=g@ET*5%W-lal5QV)Q%&bIe zdPH%SrrNGAHR~#IS{*T(NV@Bi5p(kW`a3Z9>*n|5*6j1Z!bzm$_$Vmy@O$2p>wPjT z!(b6VBst(8{jf|~da$N(r3y{6-QoJp50z^oz4RHX!3 zf5janeM&F*+<$ML?N9hMdm?B9VvOD1kjUW6N6X;eNcfZQtQ_?6K)f}q=SU17b;8m` z93k_xr>|U}!raU6^*{{y9iN+Hs-QQB$|3d4ulWmcO^g&8_ooLb4;u?Fxabp6>MuUe z)2}*P9Lfv*oGMR_#r_-$^IFy}bfL)1I?7Vbmd&Cl_&Aiz)yGF3W}1-t#_;1yT|>2qPYvWr-V0K`Y>A+sk= zLBrbEj6cN|SM4jSIN&xn6^br%N%6HftV+h)+lL{R?g#}!yC3E?J(!UgcI}+LIQq<0 zgQDcYu$u0A9RK}bZ{!QWe=$ZP2;jwVeMgsnuho*YXc@)#*vxyCz7nR1=f3Xdq6Z59 z_M%zp3;eqh=mBjO!Z4o}#8?!~VNuT}?br$nv->lNN89QFR(ty#?XO!|y(Ss9ek}xK zUh{o5t5f*hxE%-Tb!Z4!)Vs^rcPg60`M# zZcbNLIp5epdlOh2^psk{6U)5_#X4)>qh*=@PC*kx{ZBFC?*~PWr#<)RS8R)v!HpFR zczCJ*3^#-`y{oxyET)t;fBQP|cN9M>_c&j_@Xa;9=O-j4Zi;M{;?mc6pzc%_HCyNi${kY~lg<;~ZlzD7Ry>F*io4{=Dt# z_ugEhom<#HjCrHr6O)Mj; z|Fe+Sg5u0*_aoyv-d)p`38WK1NHNt@x-T?N6?ClaHv-4KImBjn@=68~hz|y2Iz8N> zl}s>4YJnxMuQWK5u-XQ4A~2dlla7K%JANN86)or7pI3I3Ywv^hu6_)rqJ$Fp^N_$m z2#{3{^aI5r|Mk|rDu3{gvvE+EseaIdftbTW?giDoEI*LhD z(xQS!Cg|ytcQXE4edp2?Nb`r?5+t1~ZLV$O})k2Vdzx5O! zRHR(eb@BG*@&$z!#A^PupAK5Ee*u8#%s}y`nAB9)r<=o#jX&n*rzVyJ6cl9yiieBU zxla-x#~bO&OUgg4xrSXDU&VW*4lcI4G7HF5S68PpnUj%_kfaOq zB?^Lg7$6Bx1OEbmV>dnT|M#}N%n1O<5^k?zsY{NUky>hacsnzx zpD2rrOvr9xUP**KAFynOqBbx~X*5})_S^0d{Qvg%EyVExj8D9w(vMUQO|9#|6R2Q( z6W8OlyZDRxC+Uo%ztwIYlZ7m^d7jNf!iO8)VN5uS#ypbIs$DW7sL;30S5fDS2kb&+ zpHr(p-URs<9LDfy-Jb)2RoXou;yBVb|L|~$c&6*U#i~6lqVu*#QIdEuTg|m)`f}Uj z%fu@q>HIZzr-Sab{bJr~^?VX>PdlfxthX75^&mtP(h&YjW~NBUw`+AyfEO9Z%VhKj ztY#impN)Ww$J5OB)HjobJYa`>NgD@(-gxNqgq%Nuzbx)}SbK}TdLbCwbdgD3)xlm} zgnL4UTUr8ftQh|xH4ZU~%}-^TDdup|Mf9zcMd#Ph%C{A%eQfMMmWVYRzTLG~j3O0f z=v=Oj6nusK{R7iEukB&rr%VOzwuT8V_1W#s#>Wb@C8|-^I1v0xK<9_{Alvs&m#a0F zm6gpDN+}r&fLOpFem5xY9OI~?ql5GfJa))zrNJDTiJ-Op=3L`av|we!T}pF5Z(U_- zDlV#X_3nIgv@wyfU?eDaYNAKUfr$_dG6ER)U1p+!$^`!xPrSp4mEX%+LlPB!gkYzW zi@m#+(hm~ZACf9AwQU_g0>lmyOl7P|Eo(G8+h{nr6SMeHe`u1-srOhk>?BJQAV6uA zTa3*|@#=49I8XX2+Cf^?$@Myp_V{ejlwwVvxhQ5QRDmI5x)QTooy&%G zpu%>+#Y#6PkvmibfnaEQE0vYejLYuw5_Jw;ybNINxEP80k8_YC*>sw_}ig@;Kby3u^9cf${rBn>LyC>ns zu`nwID1GxYDMl~)8QQnS7x)7W10!+OP4-clkceo{_g+F=+~w}X*w|R*QGzrxoi#S= z%f?Wyqa=FWckXD>&yxaDI2=lMbF$PujP_Oz*9|b{zh8UT-BNV-jt;fWXDScNLI*;x zWaX}nXgpdOD~f$3-<2d$d32MVO&qqDbQv3NGmU4a#D&T)>&*=R~j4_o+yso3|u-C|1QR7 zYeK7>9V^{SzA(2RXUl!3&diinDpkb!9IquuYSA3wGIH~%oGt&oOl?3TQ_HooJ5!##qDvXH-J%y1u>%F` z08w89J|TE996C~DdI;Sb$f?~ygd!YO`F_m;^KyDNbao#dE@NIo0t_uI;EO4Ki5CS2 zI2SIMc~F5@0PBv8b((j&>@sB+IEiAx0MhaVrRv|39o80rk&&z&I(Fo05qPi5T%QXv zV>j+0-5KkexWy68s2Y7-i{vXSD>2s_Ku*c(;&Q`U?-d<|>A_Q7*SeB(9b*cb8Y?JO zE1`^9^4`UwwI}&+00KwF+jF>$-9Or9Mn)#*GX|4ewj%k0$|(M~%)20-b%*whz{ENR zbA`45>wWj^yge3W*cZV)sy7Y;%+N&R#Vk4lx$^+Yz5f0L3klZ=j@UAb%&7HbRHOYx zi#a*Bt7Ft6dNuo_9AUw=?{H^^OZxTTqyo1U&MSYsrU+S6#nex1OJ*^wtC%^^h3L6w@XzX8YM))V~snpdqU`KYQM z8Z!k32v1plP_>?~uCSf}#+~ecHx1>PjhjtuWMgs&>^I zTkFD5H#M_`&hq6(%f5BC9c>rKekwbI-RBpikA~$qB67U*X{Hv&%8_Yx^VfO%WmpKM=w;Jy?C&wB%Pb4NG?m-68g#M?CkLkve{`lu{MX%7$; z*=;1z7+a}QacY zPAF22zB^~c)LWWyG0*u)TCm0}TuN=%slu9=%I$_vbsq$|Rxh_ey^nE4@qii64o<{F zWzBAxzynHpN3@v(Sz1H-MA5ynbmt+_d1 zNafkt>}keoRND7?P+V=dIRY|KzHe{)BVAN^g1hI3`iRvU?2R;(VR8<+?8aHFe?1qV zv^$Fz!E!YlzdT?vdJ>jnCDzn77Z;XHUv$T9AG(0*(RhrLJ~Qj_LOq|4>S%H}N+9{u z>nG#DuJUxsw}N^b5wzx{ZOeQcB;}Ahbu7fJ^C`;Zj45BNJpftcKYKm2%x@28$<-9g z(dmSNw_8q)S4(`_)@l`5*=t#`)NH1H z&j-f%usq#n{4}xTE>u@8DYT{i~cZB$F+JZ|C>&U zP1faZ_!1+B32|StN_SV_RI9^+|Kdj~iaQ_iElmRZi{)cMcKcWx&S=TNY?2Ma0iBw` zWOuD+9rU{Vq}AUCf%OwVISj?OWjsNfXJOCQ`=_JBGDLi}fD>8pS5C`x)2?{N&;duJ zBhGIaeiq`mO0eVbU<2o$KU2Lo`?tJyheuNOkW!z`?LF?WRD?k>cMU( zKVr|l92~YI$g}6BE(f>f#(7oEz*)o%VWgfIIa*+1JJxz;hzXDKn+V=4UhA>?`0VO3 z%m^EOtpp6OUIr%6@>YFA6XnIN`j%aF%OWcB_%JY+UyMr`CTU$%ZRP!Cd^`4@(T$?pyf@UJyp|>KdEMxzkf!oOFHF}hQ#HcJJ6>J?;IzM zRH0uKA~Zh+(0d;&Uah>b;)VKusY;6uTw!eIj+$$%d=uxd4F2N!@La}%lEz_u>yf!Wu&@|$lfKPm?4`PfuW+(+p%5s? zxWC#~J)PF6y(8@fmt$v}_S>`s;cYv%pQOasaP&^08~*CjdchtWuEk_v2lto~t|SNIb+yzOw@|9@l=|T_7zgP!7;pH8 zih!+!_pUl}O{WP>F>O7kG8g#y^)^{=|G;KfR(syQG-FNY1T8})!xR?h;wf4HzBAUP ztasgWmftEYP{INtgND0TD}OuLThBi)UZ*Oi4JTsKX$N5vtqv43zjHNKbn4um3J9Fk z5lhwlOmL@ofO~ddoPN!$LuGfV8qe9B;-tgD<)6atpfGCq7BR`dd?=3fUb_$cf}Si5 zzozq)@I-&2tag%ad~v&?^8Gd*o;N7xY>UfM(EX2kV$hM7vB!+O6~D`HL9x{1PP_F4 z89f=VAL2V(``_b6PRjFazD<>@r;cS7eFxdWnhSKq)$E-dtClcfmJ zJl6{oXx$U7yUukYu^bfw&l|G+lGquynQ-A1oi#vz>*8&P)DD9vGKw9Y=%tc<=0v+f ze7{KsU!9@q+v=bD4YJ}t)Y|rakIL8_8}CpAA*(e#*qjMKj0@#edT@YU2#|vUo92&K zM1-W_#|ZZ;b;=BC@6fj4`V0>LHNKy^!t4**pmNNTcca^ejw;8^j^9r=owc%lvnB`Q zYLPV=`9b5JrEuNyIl>2I`+7qA{WR7_9Y6n;FE}a+isLzA_Vh=mbc@QEe2`PFR9v2V zik?`c9%TMhIeJ+H$`UrzxFRxi^^DR;`qEZpop~+(16y*aN;f{v^&3FH;G{A3TlUMh z&E1R=W6=6eRaGiPxL5!Zz8Uts6BmZP&Bg8|Z`6{-+6RX5c+(fQOj0?T0J0>Yy5pCM zto&?%khS%1dV2l1=!79bNNwJdenC;s_X}Pgp01UZRNi778#xjrOG`^|aQ|_U^cM$P z?|7fMyriO%+obq`^G*?G=RbCl-fFQ&M0W}UFSfsx#{1SvWa_SN53G=#!3i1NzZVv> z#l{VsbvP`l!3!nD%>=#q6f0488sPFL^bWN3yZoq2{2NqxZ$TU-sHr>V+!pXf>0n6= z*F!Hdi`=U#=*SA;JG4+dcmL)T~kUq z8;?!D+e9D=(^4)=og2im9|X2N!Ok-*11}V;PN+z)%k!^-ZjT+BOTWPCd`McYHBa@D zSgpK7qD{=hR!RG=2hT zXhfbpxtCKqy&U81zN|4*7&9z`7*SH0oE%1zSwDKoKH*O&h3`DTXf(gS_`dEpKCIgC z(V6n~Xx+8aFSTM+;Qhv-`t^fZy}X#~u-FC1xud&Czx9u71K5Dhr;hjHt-7tSnQv4* zWbIQcxk8qfYEa>B5$o69SYPda+bvCgKMfW*=%_`Uz@pVB@rk^X>o5{)l75Q zrhi6^Pel9$+(SkX?#Lytsj;xsY@pieS_$((Pw``^$1JWp-TFfGi^PkuW9kB!m7NXR zhi-kg`r9umFc8E=5;V*o^{BNxRH@g<#DSq|ZZD+R>?72e$G$x78i0$7T9}-QL3~+Hr4b>n^ybwa&wK@jU_V9X>Lv+ZKap<*4ZbHx zZG_m+vAbdskq`vU&CY_@W#rfWEsv!U>+b0pS`x8wS@F1YO!lzi(!!9siVC#wIt(vE zFP9B^6@-?*GpGj1jZ_W9r4<#E!bZq{MIA-zcy*y@G6 zJbs(1j_--L1P(==FgKm4I7+VNN~qXcGK)8WU=|4(1+gioj-YUFcUAGEZIzu#fJel8 z`mP+ny+A}o1UfzthuwMjnPXEY2ao8h%X_(1K9vp{n*+rS0^scnG+`m~_1E+?C$>F> zQvzaIYne2XK~NR+0zF6L$_6JYyZH%wt^KxHc)u*IBg;JD0;`W>T_4LK&kHF|_vlRRMUWk8PA zIriYTD~0~0v6dk}wg8(6Z(q!VwEp$+SPuPwv`{C61DkD&WbepO6mVQ#ab9TRtIl0S z;gIP7O0m2%3@*1@*AcI%SYNyRqWnfB=khUAUH&i-U7wuZ+G*G6Y zvQ9NCQW>8Ex(q1a5zVKoS!H1YSxYU$n&49t+^@sOvwXka;h`X#TJ*Y~I2@-hW-&6k zt+ooO)$FZXKxQ{E2y1M18n~M;6*tu*Q&HfBA@=zTq&?y8mjm`;C;N&?$@BH~8lF-G zO~^#wU1T=Vpa9&pRap%ULw@mFFP~68hCjag`mk!c8=^B&cFAk;{Ci@${pCxns^q703WAw%(~DU5aaqaxctAbK zyINg+c%ygt;s%r=ke6YUBfox9&bMGm!9U%xSlL5!^!Xt-U}BKATzj+_&(6Wj%mL~m zrMmhV0>tq&r=!{N!MS#(r*#J=FP(X>5|G`#gSU4`Bhp1QQA zCV7X@g!4M4E-TEQ!R2Munt0Y}mnPWbyVL3QUBk>0R@TGGyl1wQU5%yMqt)*+LiZg$ zSO?hQwV>8EH;ZcN?&Ngd(1u#-fAb|*yOG7y)M$zty@lXR)&ItwNJL5%p{?wp4S(K8 z(A&rF@Be-~-MvEIk;|CQ^!2Z&g2X7@iZS(V`iaDB-a|4t&RJ$~5N`cz8^s|cE?;Io zb|I04HBKB;*Nm)xC)snY*EU(B;*4IuCFXJLv3B-f$##2&mmk__?77Tb-9{)0%R2$- zwV#7=Rh1<}{!VU?j(M1+Y=_L6#h%^B8Tz7-g*D2+=gfD!sB19q{LM=tZVE768r&k` zY7LeNyNnKA!Fu>p`yP^kgzEfTt%PIh8!tnxF}<0?_>zZ@2{xU2CDloG%0u}b>lyOV zBAb0lq6D+|@F-FjZgx*|#@vl~_|E}Js+4k;d?{vald;;kBjCI4E|NQNLqtep0TwHc z*|rIn<94~Dip@Di?{o`9~9qM+?ORnG#um43y=3G8|ufm za^>dsLyW2JdkAN&%8E5v#|+*xt<#JYb@~SihmJiuNjbhZ@fRqRKx1>CVA9f{stIg$ z_<7y(O2&IuJbrqN-|D6#%W};v-pAoZkV@EFCJi(j;w>+6$Uk_o4nd|F99#b$~y=Q$6yS&7NBzz6ZJb>Xl*hA$<*YCbX0R zuACp-zW632pryfLIK&uDrOmSZ&(DH&aEo&8rcVGP+6bi+W*6?K-IYK~chaJ4$RU%I zaP?N(O9oRgKe%rFz|#(EQQO^s*uVwBcw8Ce*lNZ2a#%MjaIC(xP_=}7r3=zj7_Cmv z`ba7mTzIa8dtvY)xab(U^CclM zBq7NCs9%k_G<=#G0`k1v66g^=og}Kkk!@*|hX?CwSsWmDOFwv+^8H2F`h(0euyA$5 z2741Wr!Mge&JvDY1rp-10qanM&v8QET!DY)D3r20yVH}mO^lShs%FnLkI)TsBsQ(c z0G1;aDHSwplXaJNgP#R3nV6~U<|qK4A1}u2VToGZ;~^X36RQ|s#5oMB;wVDY9376X z1`i%sm~A=f=Y8zew3F7i3t7*CAd|}C_U}lYy&y+roItehO1V+B26 zV?jh*TwYGj)V>8{YN?Tv zoB`V)udr~&C=KX3qDQBopx~VNrGgyvr-uhOxLY%WBgtF7$2f9Ee0HD%GIVB%nKC@| zZLQKSkCL)Iwaa+**jZhs7VJjOxndvA8((myI9m8MPoxYLPoGJ$D@X{#5)1nI zPdIx{+Q-CDb&xL%pBA+4=BVH*K&KTSoZxQN4vL}aoc_eC;Q}h3cOHSu4NsC@WTw~J zz4zP7_0_)C7N+Tvs*OPc&vOGDqZ8ioTZ9q^Lp9=FpWdb(e3u?~{j$;`;`b|3OAAc? zmTLh#J|{tc{i~HfBJuUJXVy6A1_0PNzj-G1?Xm|0{U%7bQNcGq{^=1US)o3LAjSir zPC;B8n&IQm9w9$&?S5Zi;)6JAKv*K*|0MdQ{-29~-vGS?_@@pZ7XUxS@G=OhX-lIE*bOspb+FWEv`YyU+^pZ2`f;{`P}(Yuc8!g* z;&>#2g299e%E4>$b@Ac3`jRVIm{~s`qzyI7O{^Tj0e^$D);DN%pJ&&aA(QGHlg73v zS1iOicgUb6Ky?Sa)D!gf(%IUmP$({tBCKkFxZwfI<&Eh?`FMncq2X{UnC*3)!2v#T zesOILb3s|5nWms9w*n#NCC=1@xSEVAuPDz7CjsaX{-NQf)9rNga0X<30>Zp_7zogA z8ti86l-T5ew7j~S&0p_9_X=EB6|XkBYY&`HA=7lj0iWG^GLWOWI!Yr_ev#vSh%v@f z!kF%6febWhDV6X>OSzt(mSm46njvS9_~5%b8P;kHim9zjqeZm9dll?vs{k z6a=PY(#$#dWol#^|o?nA_rH5E;Ep8c2^)4r`{_!p}x zqD<$@j1*e7zf~Lz%uZ6q`y1rc9lDFM}YnOuN z+(Rxyho`KgR{p7@{4^ci@mXeELVV|A)M{=kCzlp#x&{EGIlT}AVv6}XIVq`X;0^=b z1}db^b{p~9G8+i$Pc#qwBpxvljjm&U2EgGB5qaVv;QECIhi) z`pL@@fzST}>FGo@iYtff8+}aWL?zl%WB3?DIm3}$Gts#G{jb4Cm*L3VKa(No=vZnD`zos`?`CHgd23!dV>Kn zS!H=7;tm5(!YK6hqwZCkte`_!HJc(b9@D+rLSr-6_7dvgVkC1ufh5=upx}b9*3&b<6 zNuqC`_>C36jcLz8V|e(9s3@DnVmOTdAP6wh{!RLc&zIO*qY3=3Fi79EqbS)_bf$zJ zJ79oT;OUS|r%%Si0WtUIHa=n_+uxZxknVf?ylCd2;`|$gEMA=u6|B~d4-0+Sh0LQI zvOIMBc{{VNcT)&`o(3Y=%|p$n%~V^SAy^**j-oT2exZi40 zWk2+w$WtW_G=gt8TC8mK^qu!=b%`VQ<30`fLH-XHpyptUjL26h6A~LQWMQzhG)NS( zNTkR5WBO+nw4Z~SN;8v}9k)T05th}}#oZ$m7Z-z$-ET1Xy}g+DGp@?VIZ#HK+5!4kI`C|hoP&=aqZ1l)({MA%luDpp)>AilA!juQ$y2w0k~m0<5n}lwg1~WE7M$ zU}tArTien;FE1}ACno{|LYpE{iP-n=-;C@%{~&q5(}I!Vm2z^np{Px|DsG4)yGTf` zCcmZ*xYD1kt1Y%C`iB4Ef7$-B2WN`T8_6GYN!-|FKq+)qZAVdQdwkE;N!|YGOG*^wIh>{yk4{eY3BFN1g zm&wGBp8K%o(#3gsbwFWA2w3d0^4H5rmUKxRW(#$45DAh049fBp5%``{G}?66DP7vK z;h)3$*DxRwAz~6qDOtU?50#1t+>W{K(2%Fq?cD-lWH~7$DDC-vU1qfXw3k+XmO@{Y zc_6vAfNXB1L2! zya@A+7#3QaelQd=uFkJ_h->7=vjQrH`JAfQZazuolTUMPDLI$j$K2i6|NAjrmA-xZ2BHTdiN4}rBYo>zSyAKR$-c!8&C|8o z={u*wOw;+hISol=X;BeIH%+$$8H>i!o05Rw{)~u$TeGoVzF)KtC*-gAL%Me%@IF@f z^F4h3tu~l~{R0V`>bS4#xytIb$ydagIVEnD!&~6SHkF)Ddz&e| z(ho_dYcHD}yoa(hahd}u6;Rtlen7F))a2yF#RZGw87A*wPY)j-U#>*y6mq!hO5F^+A?8Mb~ zsJv0e%7lEl{~faz9xf68!R!YawV8baMn>co6p+f|N>52a{{RHf%;;zt6&01WcRP#W zpLg4y`sqT8)amwzM~2jgp^?I=QMe)H>o0wT1bxs5B+b zv<}>%g6}bqxa#8SNQG!)>@9s7cLU2!M$Fb>h8k9wLP5XYZ@1Y%4RC?_p(!U>F)&iK z(!qB8X(h79k`65NEX7diPmZi`Er(loR< z7de9CXM0E)mmvp#t1G6@0)PBMtArF1DT8U2wj+iJ5C2(jpQrXF&+P89T+SDWeF3xj zdmsYVUa-lhNRTsBLrO|>kNu{;F*eu-Ad?c$YpcxI6Oie+p|4#mVX^p|E$U~E|85>+ znKBo+3kWm-9~wNX^p~j4-DspSrgG5)(@%r-VO#2#hV6ESNON5mNh>%DHO{17FPOxM zK?rc1wT_c7YU(Q%N8XHL>0(Kw}|8W(#ce4UEP(ene*@!JXwPqU)J-zSVCzuij z#t#D=@Ivqw7njfz209=XJ-pxAnJBinq-2;)B1str1R|x}Wgvx~6`CRaU&OML`7cua z_dBmUFXoT+tWnMxj|eE!EJ}~t&ru&Mv43PzfaiZ8@c+Q$e-poeZ^*cU-Yb8IibSxX zAp!&GVE|ytN{>S7NcgqjWaA73vR;Y=ZlDM>2#-04yb(HdIw{$jT0e?bPY;@^ej(Sk zS)`y;K_Wv7>8Sr?Jz4*y#e@f>3;uel7|jj<)9^nG@qgsMRq*PB9x5yLH$3|%#RYYm zX>du~EH${e5iUrDxIIvC9#az2l29OSA%6^{EM)#8Tv*ERZ0#X4qtLrLC>4a?Dsp|N z(heiGl*zV6ewvj_i6MiYon+{;KF}UA0zGetd^wzP%t%FhNBklwG5ePcirfCC=VS6Y zLsFp?AZok3n;8k$7|nsgwt#Y_545qHWZ=~h6H&KY&9I<^Y7$SMHJOI6=-i;kO$;Iw z;Vrna916=k5^%#8IOyOhaE|iG@OX=+Bw*4cXAEB@?z% z+J>ix1Akex7cCyeU)*LcFX*4aOWTU_8t7qT(wl@V6x2Y6Ik3e5t43?r2%FH9HWlHJ0##Eb24a5kLFGha_0b`rgERZBEM!fJ%T z@sg0x*r;+6nA%j!}I+1K6_v1yw_QO&6SzVnn~75 zX5HV(F&$*Wdr_ha!zGk&;rG)BeA8_OP)czbQj+&?Vd#YxVaYez0TNX|Ky z*e(WMYQFsw-QR6Ds}p@bAc9|iqIW$(kF@M%6wKJgt{Gy;Y5s|e|C`0*;(NHaC1#D7 z(??bE!q;m@v<=0W(1IynfE07KD|h?`CXX?TE@wyw-RUuG8XSkts&z)Pc|2o_-ADh=vYifQeZ!1`T zXAbGt;8y^E1>Q`!A2Hr9g>OctF5d#~(VigL?K<~{)fzjk2AqUyplUd7D{}ASiohc3 zI!$FP#kU~xFnm4!!2U%VLB(>W0T*}yE--o18q!@>X<~-OJGywq zbJX)V8x14G6Tm0`RaE0C`|LuozudmD!4T?5X1kP_Na0a9Pwffg!Oi0f$tj6(Vflw4X_FATFBKtA z7un7MpP-3vM+_Sc(QXZ#bE8^(6J%_xLnoCHoE8MNLDS!?+N9(>`XNIOR;w z!e2~$$K=dgUv64Wpy1XE=F>jS!9H z-4t1IK9qreZe9OCV~{O8_21xY4VS@acFj*Ju*q=L!->Pt_uf;IoM|;rHtE&(iLfYffzRQT2My^mP0fzw*5|0ph3?9!wNjf{UCR4d+gQfLa!7Cx38OB-mV7*<2Uc9aTy$8#YV9&32(kF|!f>epYl4!gNYb?49%yWDZ3nZmB!9Q3^Fm!{lCx3A=BE_pO# zEp0a@D&JC!WeU0-FYiTmYdF@N_SwKEZ#AAhzsXHcLQR%m76F~?Qd+`@nfAW05I1L) zuGiZPXJuMjL|5<41wF*-U&JPo#d1e<006{FNri#1uvgPCY)Bm;DFED94Kgx=aOPcM zMu#G#HX7RFk>}Bkw)bu)%!jMPol?-g;)3 znQDw}K_9ycQ?9u%Oq>LQ_ z^jpm;Rlest0RcPp7-AsU`YnU>k=C%)j{_bin*IBV z2B2Y`q=AQ1Nx8JJfg0!%MG`h5K@V?#burj42$aMSn3xzVo^v8fEvrJ(;RfcuIIG=h zULwB1IQasqr0yD9KB1`XSR>3xxjIMaAhqpy{_HI*pttp{Tf=Djua;8szM9U<{_mEq zwIB-Di;nEp*`zu5kK(2Dqfa<~BF-x6sdfD&Z}LzV9Sy5tKXIda$SG<{u@G|590r+L z)*j3>{T3KceMz|uImOqJF9FebUy=ZrS_2evaIStg6T+OEn z8Ak*T@6rgPS{|K9m_qCRV`9xtZ1NS?EB2Nj%w{$^>K-I5$9I_u&$BFe7_DwFtWY!d zmHe#)%4}&rWrLc}rxm4h&vwB!s}Hd0nWPQtUy43`T`rt0LJ8=*!oHvK`Shiy-!a_T zApj#!oAqJGbVa>uFaI)#Kne2uZfxe=Q}cZA6S>V<7J6Cj@$$-&E&+Lu=iSj}Y{6Uu zGR;PVQGUx_Q{c~HSTnKfL=;rWCQCSZk8gLq`|ke&Qn?0w=Z8q(B$2h0#_wqx7i7tY zyQZD-`y_xzgV7l0S2N%D*)6uVgAD{9plO(rE4_ZJD2Y7wj!_q#uWKLYqi@tV0gcz8 zLH;#E5!Nrr^?9?=r0};Rrz%~TudzDohTE>*f*;30JWovDl20It#@f))8|(D4Dj3$> zn8_CumWD3U!atV5txzu zc}!@%mqt(|t>WymI&R2Br#Y;%_x`@6(`i=<#F7tu7&Q=HsaB!>A6T&>38L2JQo~v6 zUQhE+%KHGsmpBQB_cx=o7Vp($T{VoB!iSB(zu>P+o9-RN4JR9@S@vqK2S!fZOdM2f zHCmYsWl^DX8C>wp7-85OGi3SEf30h&>Okq_J&S*RJ4OHG*QZvVPP;Z@P<_W1n-RBK zd{D>Gv4q|T?)$&1hB#2d*wuyCp!#vSl(wANlT(L?-KhlOrf=``hXk-V6ysoQ`ejyu z=D=BAfE*~OT%XotVv{`V3YUw{YjMXkLjh9^xNuq>^DjzYEgNKuo8BP`yrj_D&7yM;?Z2TD~bIrKQDZwW(c0a;3pBBZ6k`oD=Y1b4{r`C52@U!|QE3VSwU_Q;V&EZoS z;^Shu*MDc&|wDQlckT;B-=Unt?PVJWM)L(TY+kE2YHg%aJ!*?q8&)3 zghWqiB5Ltz=YfjV%1Qm{y@dIV!CBmyco-{Osh zLMLip@wcgB`<49bfyov9=u%GOlZN5&hlt#IdgOojL4Hxe;SEL*R5kO9GMq*I<~flf z`{47giatVr+hXN*^6fEWgL>>eeehQT`eD@tjox0|3iDbkwz3~7DIe3x1X)8xT)49# zOS$lzqM~HvR7PM!eDLjfsq}%T6LCkqDkf><^mibpiYk`mjff8mmCAIfydo&_M_}gr ztH)srGHX=QN9=Llzq=pU%YLT^4~E4!%_9pZWVe+i658ZX$2ua@V%95R+!U7KF@))~J5#F(pTUx$3H~WP7^OhZj{Iiq%qDQlbP1vuvw}mTUSgqX?V-|-qsmQoc z$EOOGUtVV+(0mn`D!O^^XZ*%rj(lEto48zxZ?F>*acMTrkq~BL9SXxC`afTB4Px}6 zu)emVjsxq`!82T@lpi?&OS_^fI~asVLr2}Kl!SxFZTFrOx2GqAM>VnqH(Uxw}4RZS6|9R9d`< z<3;bv&s^zlugKaB zVpnJQ*BHx~Z`AZSaDg^bXjvp;x&?k9?l1#MK}-j$p8!QbDk-@^2>ovtMHipS@+BL% z;V90D<-=}P-Y~|ITe>&zoJOSdnFBi_dj^z&hEP!+!-^yHI|hW1S-|V5>b#|G_ZhX8 z`B^bA%G^y=NG-@ybFw~O^%|J_!y%#B%{RX>+6}Qe8 zoBXg5qp0copcy19Bm1}YeEo7Xy4!u_7nK+SJ~n=@r4%Qn9mI0856Ail4JqPKzj znVWhF{VqzMjFHPm@-~oYp7KVK+jiKyya6bgwC!El)05tSGLB>{21hxs?;eZeWfOJ0 zBR(yhyi)#XM)EW9J-2;lseMh>n+?cZ;Qax433twGpE?hI9;Yt9J_}T0suv%|5^>ijHS&3=C=k6ThBP|2I0J2Sv7W zC6Xb~(ITlUsK~K45}F(tJ+zD*J*W~AwhNzuehxhwNx%gw$ zV|sJ`WaK?#w>y_6Mau&sAoUqU-go~SJ&+b8G%?-H4( z_4oWtlM5Q3XhwQBZ10l-nX4~@kn-bos}n$^gTrvw2&6bt>#4?(e91P$|1Y${^XJ7~ zwd8nP^tp|6&1#!@r9NMMX3{2HCZ-e9ieF zR`R4>r@aLphVP2=Ln(1ki8rET?oGSGkxk|RHOSM%K}Oojs|)iH#rmqp24z?i|Bz+ySqv| zdRDv94a-LsYz$wlW1GT)-LiW{c(c?-f5Fqa-BOe*chm>t3%RZQou1x3l!=B~4cC<| zM~~;3Y|ZtYxZ!PDtDOlnI0{~R3-%Xy7TD{!3Mjc56hzY$9xHP>&#DHOXdfhHllgcS zpa3~e|7-A8UfGg9z}5#thSvI}S%05ovl*47#(ij^+Mu-kEAX;;wN#XRg2`gcjc>iNtC!g?DP4r} z;B4^J6pv%8O$HpMpCJXSrhtS{Snf7S#uPgTooY?bogxDi|N+ zYE{3Xm76b|UZAxyG1U*E3R=Kpi9?^}aDL`?s(DW6l+tVT7YIJhEan6vZP=;@l+?3| zYBd@$5S-L6>@{8oPtFi-zw$d#p|8MvCC>7o0Oih}#>s~wI?HnJLhy}Q^9+_$o3!nOn{(StI8Mg#i` zV|-2KTQDZ1_E%O}2rN~3l^;bGIgVJudo0D`hjd#K} z%Zc1OM4mcn4XrwY=CdP5nz^02>%sP1TkUzxB^h}#v-y?NWE2e9AqjC&dw&ay%|Dab zQQUDleHa)Tl8d9DWcAmdoa;=B-7MIPya6c=Ae5u z+$$>!5`NN;m(et2;dOG<3C?b*;$GK)Tb+F=72cvv9}DxMakqYeWf6Uxc=sl=ki7o} zj)c)1alk4yy1ME^EFkFY;L@e~{`~UqlpuAzI{Pf?JAW1!*6*d^Zjx}e?cOmQyqC-O zu)!%@nD22{p2ym7peu37?wb+R?`v^w#aj1U)9RjS^kHgKyIc|du9MT4`mPlvFJn2M zj0wbK2J8k2ZxBu<{+64dp>OF31q^!xh=xoVV0oMRO*E@2RHW}wfsx|8I!l6_(Ct27 z>-)KvimFM}&#ONmnI1Q7j8XRneuC#^;TuVsI1;?*2}(3rIc*SmIuE9|RfL+VLK!D< zDk`dkhR!yxn}}eS1%t zexK8x+k47Aal3pT81HP9TAW&g;Lk!g?{j)6KHlE&rCQryw4K2d&s1!~330qs>10kp zBW-yz{0>rnaleeEgzO^whA@#lF9+gwd^b+rI?c+=Jm~q`u_^QyM&uA}t~s^?dwzij zWqpOEW#@s~g?>b$1OX0`{ztt}D-)i^x9P@c^>*io$@Ov_t2)9Xa!Rrp*;2XZ@+I2t zEd1%W!obI5_`Vx;EY2^dIWg9-$R<3foGMrw`G3`YWnIcutbu+EEX<>A+>kGNLsl*d zM)6CVLj%yi${M9ZAN7VCl|>Np;P3D!S*v3OtpXw@ufc#KKn|rGOGIo@I&6p8Q$2Sp zK6d{DS%3YUzD$)M{V884lmlz^eYzLM2O4(RxlH^@OiDVBvVhX^kr5FXfWJyHDJdmI zMRWVc|1E;MKt=HM)9N4P#ce7 z8@w0!Kj+6NXb$V*Im1#C;^5Z7u6Q2@7CYsV{70YPbtj;)pV1`^L;B+f|M)7!zcwh| z4h4?^fF%a5eaioVx07=r8P}++lz)9aXvy9?9R45CeFDjZKMt-K8Tz72HBwZ>mM;Wd zwQJ>-Kd+pK6%$aUdIkERR6I-atM(6V|LGmN^%@8Gr~Nm{u922D@rs{;5dJ{`UU&X~ zU*?%fe zu(GfD2ZygAXIZ25Lym$_epUW_danzqP2A9Xkym2ztB$FWRut?&7vppUNJ%JO@x#C0 z(rNtv9q{+i$E5iOG2EA(vj+_*PNsP<3vamocoXQR;cH(Ca%cCgw$LuDw=w=*WGczr zQW2D?q3Lu#+UVx7yso#MFByd8txnwSNSGZ`VXj1*eU%)?sPp~`?>`X1MhP@uG~=pV zYCn;THBe#S;sfhQKSmNS0oQ?y@%x>Mftnr@Ct_4n*gF_1ZW~$~pA2aSWT|U-8~5@x z9`)=#ZX6)f4rW+9wbnbKx6ky){1&>mxkvIvmzupVKfLf#nlg6)k6n4Vyz)ziI<5v& z8!0!i`X8qD1gP(rC8@*X>?Bu}N{^>pypv<^5AD2lT=2hDo!8xh9ZNOXf?h}PX$!(W zDk-Py$Spl5ew8qe59dM=kKJ$sk(qvJ0AC^!{O_uQG|Bb>`i?lgiQDc+hALi6xel}B z^Tpd>5)?>_%Pf;tu7YiFB5S-gsXeAjX+7R?K+!_y`ANi-{ zZE8UyE!so5U$kjyQBsIirYr_KL&Oa@K5!aAwzX+|I;pz8PT}jcy(o#|%tYTf|cgOB( zX0`(6sPD)PtEl@`b+n5cJ9sr;@IN)uZ_S$Bp}GHsrcT~7Fw1E?Z1sW8d@gYd%@R=I zZ-ayx%S7AM|6j==+V49|e$__if_~K%6jY?0{TMv3FPeAIO_3fF)rzRO{kyutMeNpU z|3fSUfJIJ#kHLpP^3Ft|^!6`@M2!lX$;jBap0? z6GE-UcwmM2bijQ}P2G9dTVPfYu-5@BY5i{M3-VfXjE(CaK#cEeS}a9&-a@--i+U7= z{3cnfIlP1~9_u%&#&{jo6Wr?mq^B6K=e@;yvI!i@7V^{xTJqX8Y&I-zAj^5w2hFAdx5>J9CMTrb>DDqY0wYId*1lGi$K9KAX;tnooZ zt&4(-mQ#bQP$6)(KX~34nM)cL4v3C=UN05xSJ7(=m>*(Ll$YE#M0dDeeN;Fe?O5Zqd8e=*kfG0o%VNFY$;J=OM6_x3FS1Q`)X zomAhu=&NVc^(z{FPKj1hFAYAZ8MchfTOZk-sGi%$dh1OgC%pW2yd9XGu8CRe7|>Q) z)Jh4|sT-|Fx zSy*nxz`>`zKB)r1zYjx`jj(1nJAp6&gW>|6|BU_q@Ky|3e)`$%k6TD$)h{}-Nb)2R<|P`cVuiIP&hz2F!#r%nO_Yw3fclgs>Y?jmqnb(L za(l#9d2dwJk4{Df9SQiLIEadV#D*?7KQM&1Z~69G%YBdA$af9MV2yFObWP|Duz_DX z)o;aqN;9*Nl00Np%HE#vT1vZ^ML)tB$<)_BfVVY$11;FH*WeHfgaXe6zjBG-~&(R3YKf;(8pCJoSTW1^#5sxE#^Xb94vYum3F8(o91mQ|%hwNNCEeGB-b z^K5(OalM8rl_Z7Q6AN(WUJNI3se}V9-o&EDGV{^@*k9MpwS`+_II-{wQdk2MSWYqABa~xTNeVN*GU^JtOUmL`rG%% z>v)Y<<#Rgazr5y`ppcU8a@e}WCx^N)WLhuQMasc82w4tC97e`Mdq?s3G*2`Cj_?kt z-I&)&GcVQhn-3}bEdKq`lct+1OSMMYGC6u;T|)3Q6Ot;^Ae052Ip3L)Ei{fn3tk}E ze!Tj7GXANM5>MBwnaD1dm-LsmSQ{7f!&T8$zcLS=E+7;Akh{D2RfahCv1s&-m1}#F z(e{Q3Mzx2FyK^JAa2<@SV1r&B_yy>0LpW zGpXU4a=(g{#YUOEZ0!6?SsfdA&&yNIWn8x>GT$Hg$bbN#v0cHir|U@p0Dx3Qiso=W zm&scfS(disN|1Q*yGg~k7P?q zy+mjx_~BR<#I6iAbK7DB14jtteYPE6*JY)RDj&V-d+hlg;Ow}b8ea`O4;!|o3;G30 z)dh-V1syI)1%4cj&p-Gq{__44G{{ZKKE~J1p}NVF$H?G8=yPqZYVIq`{TtP%19{HR z<9NAp$0PAWKl?5YM~}WR<9)Qg_OV)rHBbTYD+veE9A6eeQa0Ng>yk2}3Ki}yWNhqq&CkywvPt%6KA zp{fATZs%YnFUNTq0RtSGi0{gwdg??;>bk)1GXnqhV2eUh9pN`0PcL_;PV)gM`K#0I z2RR4;_0P-4hO8Z>Ig622mMeBo0#T&Ggd`*@k5|VlE%30gMxb99R`H8+$ImA=*Yk>H z2>gokoz|MH=#%7c)V)wH&6HXlp3l7TrKR>(Zu9DZe{}!eeM{tE@ zpU;|Ta1tXvpgmFO{!4CfL8IT1Mzvc2xm8=Ls=CVl9&_$MeMYJ7YBQqtJ8f}3qpGS3 z4iJe(+IN|aPCEOei5nJm031!Y*X$`Za%J%=3RFp_eNEZ@0SA?YWNBY{kjcl-GkcT* zRbx{o&(~_vAipoiTnKyE)q3Y}lQd}j0va;f!FvWhn>{JG|^(Wo)d zq^NhGd6P{kfT;v#7V(23?vcK=HUkAG5ig1yA0Um%>Q0x)C}nT_GW(h+EsJ{~ZD7@eX% zq+f!ps)0ESE7`SdoO{}VRl$K+lOFO36v>-XXLLB<=Iw7jZ`U|W+R4G$F7ft6z1u29 zoYabvDpz{{b+QD*$W>0D@WA#1m<}t8Tz->hoq=K4{Tehk!P+AmsUlCW-b>P)QMh8Q zY$&{#w*H;}bG*C@H<0yE_*!DI&Fgol_uzEOF#HX&bkx}_xuo>)P?#`%SzSbNHtnv!hbEj$6pK>Ptr59l;?B4k;|AxY25Ww~SG)CCMN zl?;v0VQpHNU;Jczr=k5m(B_?ZeIN7VW{Y^}4^=>#(kU^hrF=L+B)ZLe9gxeFTTGs> zvcy{RSR^@_TKQh56rd1)obQ>&{xe3UY`hq;A=|Zia<}YY1c|)PwpFVYz(!dxa$G{n#^Di*LW|lNINDH_krhf%BnKDb3zuoFY zK{9@tO{?f@o}vK<3dg!&WCpVKBSB2JC^i0;U#N91ml3whkA1f8d7|slPJXD<)IXIk zkoSsWyst0C-vIREFdkyQLAtflyoO{muSrnInfVnF)URjd(Jg~NbuYDzhD8QPoW>5e zVm&qOl?kZMk5)db<*_WILU3HRPm^HDbsEj@Z!14nShh#z?9M_nY*Sx)a<7qa5Y+N{ zPg+(EF~2x3rt`l2)om*v4V#vivORVkcA1{H9u2D07K^7pvJ)o$+j-T*%Y{H{7+rZv z!u8|f15Fjq55JeCQPnm(VW#6CFYUDXY7#PSiV=|S7W=~GO=H172~Brtq?rnPw0`04 z5?}q%sJNcqwwh*1OSWOWoq(O(vGe!(Xt)C6i6>@(mkWc)faQ$lrY&#*Pe7a>FN^305_Y;4RSi{s+O_gCdMNSl$P&Vi*s4j914t#-Mp84hlhZ&;zbwp5x0 zB_`%p-@e+|73k=y$kOel5bk~3Bh=Nk;%zq2Ipcc%RH%KEV;8kegIDGIGqWD~xTm86 z7oS<5z6Lror090P=%nr*?58&zY}?U|4B?#hfNs`Gs{w#7a%KC_pwGULr1^05AQ6G( z`Lvo7J-}DWibpMLL}nE4EdqyHZOXmW5=xk%nwtJ=Q)4U;?LICc(n>=!u)E$aN@8hy z>w_wH0`XaVJY~)ER*{qyu7iexlJcvdRgiP5C=6eu73qi;zN8YSE_hyjpRWWKCbnEz zlOs@Yan1DgZLcu$@bEy7gqft7gIS-&4Tzzu@jL=KA2F@-NQxECyFKrd_oG9y77tVv zny&^7ilDizaa3>Ubk-+N^;rsrs7oK0-y*3WZ2xvPGcb7HskRZjE;^gmef(gl9$zAZ z%MxtLudrJ0nq%%?$tR^(T9(U*gj9xr77Z5q97^rC&bEYOOPyNN#Q;Fy_8b|3+&-gL zzWnT6#EV0oZI^fb?3fg<@z*k4C(W9^^+-!>*4_cVcTF1orS{L_wz)GCWX*5IS@uVg zz2^rU9Ndt!G#x-0TSp9qOFh@BMK4lXnw+z@FF9<`x(M~JR)ZE6=&fBMA~~Bb!LsVQ znlJzS_&J|xZMJU`TZxG2>EW+BC(WbK%}~X$4@Do#$=z3NFU|Nn06=k?3-AW;E26|q zG~fhu&Q_n3Rybsm_;XF#ygV*##NlwV+xvNYH$73@wo31_Bf=t4m#`?MKo!TAm#Ho1 z*{U|Z%_AHhl6}1!wJ4OM>o8|%_Is~Vgo8@NH+Rzs54F$lbZr{n<<9a>SobBw7aNW? ztY?Wjitn};bUcH9d`pANJ9`XfX!Tp42rT&CP?Z=S+~RL{Bx-?$146gyr~EPGd0(N> z;!hnM!xo?r!Osr-9@e?sCC4jp>=86E@|%J(Ut+fBOKQ&4_w(CA;0^Q3gRL{@A>=$8Znvjr7c9iAz8%iT$|_FTR19sdHJ~dI{BYBc0^mTd=9i-9jU1q zDRHJU(4VN89IGesXo_0PSyjU$wV9MMZ8dE)-;B6E*_d&2YGB+yG_9>NaUK9;l%?xMuzK?eK(elFMQkR^KVLF7N0pnR|&-6L`-9C7tF zya>`}XI9j#C>x_JZO!mTn%S>i4_jX55Yy1V>aH;s>zy5qbKIQ`w;^{R^bVco$5-l| z2UT428odR?Shc^T|Gn$KOs`H=_oA#>QwZx%%+1Edo6bT0NNtaX8YS~RyV==LiILlU zukf@}UgTPfo0=!p64DVpSoQ?1+OLgIs(rm|eo%g>Sxv|A95~rq%pi>KF@%Zq+4tvX zRP&NQYUJ9VA!N^-66K#%D~&1kIkcd^;&pJ)G2Y!Y5YH<8jkrb1&4BY%zd;NaKM|l+ zZ=}hh(Be~`x3YU;Tb8vPB2v!}dW*w+AE63$fE<#8TA3U%(Cb30{Jg~&(roZdwii)6 z$<;{8s5G6@hy>(Ve&AjXh7sscMd*j->3smmN_6NldNoDA7^VK)jjD0KBmg$1DaZKG& zUFM-*H9aOlN&vUoE#0Eb1t8Dws%*d8bCa%3SwQR9~w7yq|NeXt7y1$xWX8z8-C)aka(ztcsGqxm{Qo{vh;@cknP z(=TgmqO>GD8fkZNv((;k#S02QvVU9k+%A9QB{4eh)d#bx9f#9;42#z~!=|7a%Re`< z?efwVxkn0+6A`91wabHB!0pBF$5R-SH8h7~+g=*LvVURUfybnK9Xlh>#BFE>IdSa@ zvQJ9DZMa(u&hL4TCfAuHP>R)U8Q(niI57b?pzZ9Q8T73Px?ZQ?AT?3+n70QToX z5YB5Z=z~b5a|fM+%#7_|@C{n4Urho zwzeEpgmWO}`?EXmL*yq5vR&{Bq?XsOZXB*Nd_kFS?caOKj~2T?o4GnPh3gQOng6n9 z&){N*4hZmkIk_Jp2A#~CS!umr;f3989oFP9XfI1(XXmw2ChRV8OCeXpfN8pXzcI6= z82V#WBkpw=``%Zmm+L zA5}fL)g<@UY1J!8P=L$sx9E&*GUL4s&!)Z`G7`3wK9W?r(?#(eiJ4kYj$q zD5=OYv^sWa4lnE`YsE)d-*PvaWA?JKAZ!}$;w(ny>e`s85FcI#41HYt=UY6 z{%}|KGX=w?w+RoA)McJOOeIgxA4R56Do?q+-_xhxL*WBH3{3~u?Xa%a`>q<0&v#ib z&q+G@Ll=ok-rq|-)B~LUWbZ6)ua%?Q>3Zi#&;gB>>|%6bQP0=ghh$0QR}SN?cM3HJ z_F7BWL`42cyDGzWMLC{e6k>y3&LzNQ!S)pJ%PfpkVT%Qsv;7 zsgCR6A6;M0*@+&KX*RI0vy5 zErvnZ`_9_yla=^GE8^0icYwrncgjzzV&X-Iy`$EaLs8v|W&prK(wJs4MLtU`Q#pn{ z8~|g8hs4I7xY_wO<4?DocoqX|iF%C-Ta0_6>fqAk=2Ypf`&2=6J;$UFRxDUX8&f~U znb$wJv}&MaY>c6JYi5g(dL?Y`IEoidf~-Z%J#8pisM%LIen((R?~pY(gs#%Dgrm%A zh#TF$U)U`5yIG*523OtjJrntOQgOD3UxMXY8KVeKP*h~%`thwGM?)u^!Y?zsQcMz` z;JFjb<-_+~+173V{n(3p7Y6*H>6d1__~PTmeg#lAHI_<3oj9_Li!JK<8iW^HqhdkB zOhH_b5|+Kb?t3G}>u{fnIcjFFZwk!!mbEje*XgMK1Ul)kMWo(A1e?%sEgyb)4uOl8 zF4yKovS`bW*mucn%Re|YH;gHxr1Fv5G>;RZpkRpMKIFHM9_MG)cxvE#dS?6_ycauV z!&I~xjUvA4^Bg&IXY2QOAeHAcFV~wt^S@utcKjSon*_pwmcs5Uhr=S<)8E%HZO6Fm z#<-2cm@oNt*p6QO#`Sv=@(0>17-iT87F2wxzp+nN+8fL2u$`^w2oPe-9`w>;A-19N z(bQuEmTEY9j-Rh+o8^?)Zn3Z5R3i7eslx!oKKG9(OKNng$0L3$xw8*jEoZmhYd8ts zzF5?mGcfXFMn<2rN%93*7K+TW9&c0LbN?MMJaDcA@QMY4ev*Xmy?=n*X9|_@LT3&FHNgp$TyQz4k zB&{0);PahAFP<^TlhmI%E;o|Z>R)iK;l+E4A|}#bYzzm-u1)-vB9mrMX(sur=1_Y6M9*7I zu{wcEvF%Bai2XzDWVZ6S(VL1AYZdtQ+wWFIByd;Pb7sGqz*K3;Qk!~~1PrAITaZew z{kg#vtveX`&#~pUG>|k#rT|cU@^tvcynX9;f; zH7deCvgAr5+sF+ADNV|g)xdzyy(1|}Wv=@4{am+r`kRkF0|eH+6BG%=AU7p`$EVBZ z78Em|@ejJydvc6!j99`1^76X3Z`L+9P2u6;59tHi5ko^m`}+Er?Oab*g`v5LvG(+g zjPkdCBcIX@kq0kJP#MQdN5Vxg&Ch@$Tc7Zc5o_+-zfV5Zj!q@M?C+jmUMAE@cCye3 zyq#qQ$V@!HndzVSINpXQhanNO)#R#^siQ@f*j|}E&AMC}eS@-z`4Yua5&F>Cx%07q zfL8GSd&tY&Zmz-8{sx<&@5;u-#+B}~bzTk7X8CN&sEZF+G+#paoF#QJt|21yD4ntG zwvqO@)ZgC&O(boY?}dCWvCWMf4ZEKbKOn**A_5y*O_-*|rlk!Thqmwc?p&avqEaUZ zX+h^^q>Mlyxs2XQ?aBFrw?=vv=S@}G*PrkhGbuWBC4oHNKLK6+Irh#&j0^aW7%V*c|U-Z8}u8R2_pgH2XO?8`n0^!a4- z8tE+&MbCUY3+f}F?As11;P0d|caNj*MY7Q%*$!fh`!en83eRA~dgeNnr92VvgjnUz zTGW7;OL&xEGM3_}Q-~!nWy`2U!!s@5k_*wY7d`-(uQ`Iy(XUl|lkQ~GtzPaodxKNT zYkbx9?rFTIZL?*e`kB-Jhp&_9b*awol81rnlNE1b@@2)B(LT9->|OOv(f${Z%7=Wr zN%oTH$S&Fp(ng**AYc58tC%gaZ9)UIZb+T-L3^YlJAv2>4Os`&kPk7xJh1nxOj&b_ zvJ5A|OLJ|V*w2_w)4#{kyXiMu`2K4e%ut^{RdU&yq9VA5H51I2PCZxvo7SgY$c6rC zp>@w^X#{dARBGx%Eug{d+?=u2hkyw`wa=fU;^L<8t8h;AKSB57>g%sS>u$BUolJ2> zw*ABqd*O!pM90W@u)l8%jg5zFu#W8{&55Ba)u_g|Fg;VdYaeG z*HHSu-h(Ms{~7@X{P#Zc*PrEo4}&iH{@pBM{PzX&fW7~CBJM&WLiM3$hBAcE$*_P3 z-AznP{LeXN@H{*{jcH$RKpX1OhhK$>@Bg3vm+1cc+^_vk_`OMNrJPuYt|I@w^7y1u8bq#gRB&~Bty4SP5o~Ip+ z;br@H6yDAZ>1E^f5F?R0J*)e|)CLq{YUq`!O8T%|x*j$Jqk3 zY{722NdXWZK);u2dmxYNh9V}@I&^!L>(fd+dc~#gu@BPB3}O<o=;{ojx-e0eD^Q_Pp9q?3m{&td*(ZWa8cgaeN3Ot#D@iptRr+jc< zO5ZIhFS8;PbS0}@7qg0Nilk0>j5Xr21b(>KUvgYIMCF=wsSrTb$GB@PYbWcG1~UOZ ztz~_5sb5X!D8U=Tfi$W=j~YrpEbW$-TkYz$pnkzMe5aiFdjUL9cViz7!^Z@b>3vv) zaBmK$X_;m(9c`!Lpghm%|^~yPxdC8-;lJ{wyvDgueoy2 zs|fN$op~9!m_gXyq=;;DhLrPL z#T&HQBXv$t$VrA|UgY`t+y#tC_R@U$G3jcAG1%8N0hVArKV^-_{vlRNyq%Z!4S+zZ zjaG?7E8A!y5b%fXklRux1$r{y$B*SAE6KqxJVlVOs8Jp~5d-on)7vICLM#z5X|>>} zvDD=tG(Ggq*SZeNoSkTS-X6HBeHi0>7kfN_{z4juo;J`u%IY3&vQPM*8zl9!@)6j z&2xg42zvOk^tKiHI}u0PsrePVzMdkRPyrVoPwi29xUDXUSLoIhx_a_t;@zo7`Z*R1zrE1HqN2FCxS|2g zc|*_NueNp!yh;8LLoEf^yGK9HeUTtmMM8V5&97M9u1-S5LLt}ZDlQ^|+rH|QVgQZP zJflPP-zajHVi2zX^mGcWqrKDL+|2%^pqq+De1Y>5CQG#l1Usb8Kb{ox_Ovfa7?jlH zZj5CBJ>oOq#t?>wlMHET?~h9tG2>Nd5+mvS3v;~#wPRD=zb{1yxu_FfgUhR9JnyE4 z6Kn9s?^9xOvtrggo5hV$(a;g)y+`Ij0k3oQGVk{isI;owWzx!h8MJ7C*R{<)rQbGt zBBT3f&AJ;$&60sc9Rt zcl^@FpZOg>Z?9cxUI9EA7$VXI>p8issvBRxqjDCecwu9s&L|^~Y>L4z_t7yiBL*XdVaTU%4_l~D<={N|zw;J2M$JPQrQL(%m7rd*$dl*mPi;+= zS&n9%^vZjJcJ-YBxW+j&nsNx5IIn$^C{YX!Gp;O>D@&62O85lLfdU4NC2=gLTb(!+ zN_3`TRIg?{+M5o4KUX1PG=hvh-M?hxO2*^YJK-7W{Idk+0(~2+FR$;?@>kxcG`py*f97&qg}Y;a-yE?YBRhj|bZ{D+db_cro_1S2It z`<7iin{j~S=wP{?u1WQE2bbd>Zl#BlFossnUnTG9kKa+|)yo%wW}BOK0(unEGBPv{ zQK_j1D=VLxUS&Cm8XIR5n;1LP zLBR!i{FSyRKd!V3qQ^I?ZF)R!PD(;S^aM*8Mzjb1rfdvo4RavR`cN5 zU&#^6+T|qc@^=;#`2Q^;FhJAOyS{DD2`gqbsLss)`1StrLY0n8v&yok%=k72qaeB1 zda=ZyVY{fmS%d!hUB$}{iz1>w!3bBPJ{~FI@wD4?l4x-sC$*PEW~0@k+=%eVVl|(2 zNTXkcus({ghqz>Fe@WXA8ssW0OVX#RQq{-u2VcciyzHLNKI-0-xv<@keH#tR!uyeW z0yI66N1eg~`1G-`u;}osGoV!pY+YBET zx2JEDGGKRo1!Z6R+Zut;C-BVr%4t$EJ$(7AfQI2a>Yst)dV>QDanRICp~-Y5DDC!C z+5`}U^3L0RE-th>ZDZb*xNuwnm$%~#W)r9)Y>TDrHB;{}Yj<$De?EPA7yMj>nAnSU z#POhg3EE=1|FpOHkuX@o&NXHaz}Jcz@k^NcPgn{77;4RGzQt_zyw%=u=z)*I6A-A{ ziWs8zkPEuNC>hXr#+ojG&u~3U#3rOwMh!(J4QtgD-zh6jiL!vABR+5V@d**2_~Ve@ z+7+J=07&F$FKdDhZZZxBkRmJFLz!<>qd-L>vzgn%@a3drtq7asU^DcMdcEuXa%z>F zNi?IUwjm(sao^OUEPK&C;#UJ4@=so8X4;ty0h{C-mUr}+vlW<%dge5bkiigrK!*-a9 zz-4dAIXfgK=faZ(^#MZmpat+V zHI$0ABYHjAGuTeZ*VCymA}&8WjAW*h(RSulWkjGSd849gAK-QGSAC)E9Fz#=g3@i0 zx7&`&W_qsA`R_JN`5)ueL5-Q*n}fe1FA+(2y~@9B_d?c)7gH#}Q&SR1l_+pSP{Qr6 z!J-VuxBXf6K@869XS%*kM>57XA1<2{g{*L7q3Vh_&uiHxz7sqX!OHFTD6mOB|I*7n zt(#sruz@C|tquvXU|ck9>{wd65!;A%grZE;@cL zzN2FT4>_w_(|x$EbV-3sU>&K};z^v}=`WQo`PLwhwP8JaGNG>$?}sLWM*^%x!ok#V zg7m)tZT8di)}7LDqv{T(xT0)r7X(CsiVA<07WmYX$v3rqJ#|6JRhcKghptnhi7rlo z%PQNl+>u3n;led$ayxJ57&R;w*gmr~KDe~43^h)-uj9#%8#jA7ns%gfI9csW%yaWa zDSxO(KoU$g=RI+K*pWvh3k;#j#h zgwSyznF-NNo8U3k$&wpQG@!K_53IPk-J=G4k`1O$$#*q;8cAY`nnxEDB-c(1hQ`!2~Y$Wa@3B*gN z132GoHBDk<^gZvT7OopyHMw>q%#v)^$si?AW53?{qzTREW56(qQpFwc7+kgDHq9Zl$}G+D!fP#0fI&gIhRFMBpOv>$v+t=26O&S~vB!k-56R!;aSC<-YG z<lsYr5YztQM>%P+nT8Cs(`hAm&v)Xay zpli3eModFGPow~{DPzQVm+DlXBH3KcM7;emzuM%KC`bg_U15`*`U_~jPgwpA8f0aa z@}xX*7X)ZDj7%Jvx^;Z3M$V2%!YU)J#1q=IF&dMrgpV`fa5B!Hd z4~SguA49Py>!bOhekPZLHMNO##!V&QRog=UgvN5fGpC~1V1<6OgN3;&!24@kVCNBM zDB&30c6|)er?H3p&+zWd5*BeoFy?mx<$tOTgSQtt9;LZc{3xumO70dgQ8S(Z ziq_|QAZ-s^M&s5+K!29ri&%ay-rKC71A!>NY8L9N&E`Fvl$oz`|%hs|0i)E3cLNQnQ2f*_|69b zyZY<9yumr{TL1tprj_G?r|*-k;#Z?96ct*6$oEKA`H5k^T((mgt1PmBeuamki&QAS z!Tp+TUnGIoy{5Hh)k)vD*Mdr^dCx~yD^^T^k;shU~gW`ZetA|Os}9~!qY8S}4V5*DG@x4B~mAF3ZHrbzCe`tKQJ5v77~g_fxE zzwZnB>Jk*J+2<*d<(nQZB?b^hONTebN!9XDmtY$oAv4o@?M7w?OXTeBJY_?Ww{1&m zzSYqUhI2uBeQ&38dTXPqGZPg?vE z7*F2+qh8J+ev9rO>{(e=VIYTHAN#GJSCGo|2*Qnu`S3CqvWD_Xd0|S8d|ZV+0Iqx0 z1%A*%LibRKJx5{KR$sLFRsIMVxGI3=dHb-$krG33O7CuyC;sq<@cFGEbshJY;CjE= z4gb3ZL=5-%1YI{qZ-2!@w$X$%@6|h#CzENPX?4NgE+_!evC%P+W8x&+F)w^`ubsl8 zVn;N73^@Ls&q)9|knSy-EX9E72Gn}U_|rHv%Z;zjAsp`vM|@(12;YCjI34s$z+0YJWQC=u2V4 zYn@_kpSEwuSOJm9!|vQdq)XIy6Y5(u4JBBja=lINuIA}Jj>l?cZU=o$ReIcR>u+&k z=syg=x1O#}pe5ippQ)pxEm^>y`a>O+`=!+HhBkkvDg>u`<(dtc6@Ns<9hsv1$$r!V zeto%nN+Bkrp>u}*pz7p=B5qxBstsfh`a`N15f8Ltb~q_6RhQzG!^Oklw$IN{4`{8YCLk5C*Vu=QzaD@)Ze&A`%>(>oDow6wOFfIK%J04`T|W?R75Fi zMMLfMi7Dis_3?fpy6fYz4>isfK6iMM;^?Gm_Bs@m{;lm>LpXWkXB-e}{Go%Q0Ggi~ zUq1ET5%x${tKz19jSYe!`$8&1r&$S&3f^RI3)d)6hq-(?AlmsHzC?>HOg$nuCBG;9yc_5&4579r+C~Ysvz}xm0~KRc-9XA5Jmb$ z!Yv?~V|qqLYQ*xC=X)6Xd8>EfAF)^<@5jkOA$Ao@(k8ML3+G$A5ybeAE9@fP z-6T35Y~;(r@1M55&~)eex&-!qMkckau=@$E!9Y`W+_eh@Q^zNRi1K~8v}7UYLx1bt z2z78d3*ThNM1g_y-RG9uNQ~n!X1k?b6Xk`aj%WIU{i+`JkLjU-e5jIk5#hAwIlW?- zL_%*uRM#9O8R+(xth3aw5Ep%WaH# z3}Pqe%l_8)k%FO=`Td~ zyxrOT6EMfQbeACehI#GS6V}RFfnv&YgmZEUPHYYCpI2e{7p7)CJ#?rk7k6~~uf9;D zd?_z0%hR!>8t_^H z23KDHCiXKWMO7v*4l<3voZRyZ3z~)wEYYFfy5rT3r8=I=%H;+c{fp3Tsg?(dbkuyk z+u+^v(BlPM7+MJ%RN{cV%3kq(djo(tDK?bfT{mIB(g>VD@+lzW&YTbvc0bbA6jYjA zlkIZ;#%=%PDB-E2Ets&w`ry~7zkfa2VU(b?rNt|8 zqvZ>_r;%%x1Fi&*F^}zR(A#s+yTK8uV7WN+R#P{Wj178BS=gyC5*b0QyWvP|l*kd{ znVMF`Tnp7l@V_vi6aCAEyx+;TTdgj;#bmdiKXIFTasSV}WE3nONlDIUW0$FR+LRH% zlzDg?KnoA$fgoB5Fpy&vACp?Bc2tw(Q76zy@7MgdfQX$h?Le79qbYr*ls<)ma zqgt&?WMK!#*#)neq(JR^m9{}VI<6%eVf(XvA^J7ucUP0|WhkMIUml-bwnJ3pmY6tA ziO#X^cp_>R*1^WPda3a~O@#sP3Zv?(OHv zEe+tKPd zy!RGqc}6D4&v?u~jEa2f$&>QB<`!-547&NIj74u~xlDn=b4epzC|3}4T7Ch&e9?%f zzh1|ZOleMDmN8@C}Pen^?}9*R+%S^Q*}PiYgO`g z+*I>Jb#_y^0C3Ut^sEhoNzkCz;!W`$pZlS4!T4Hk;N**IsKA1N&qcdkC}u43CYS62 zx^qdUNIF2->;?+#_E3=<$jzp_UeLL)%S9z=j6$`{{5}CQj?Bh+?sXBAS zx$Tkk2s52z)g(l^kb;2Hx^GW{d$`pfyD>+&P@)Q|Re_l2=Ji`Dxnmw2V)0!hRT!kk z?Vj}pPwNeYjo3U+hnb-fvDgeB4?lVoWAQgjuA?kmnT>?aR(q~ZJ8OsMl-unscSS=3 z3rYj5k0C4><7tARg<>rbikG@j%vHJWyV!Au^Z`!y2m{K@#i5UFXDl zTFcF1$O4&ElW5{9fnChH+DhwAxVZD3+S1uIHL7QLeSLix7#Ix(FK=n#Mo&b<#PtpE z%_>*Ay1K2Wb@6G0+waciI#V<(wAfnbJ}MG6i3;GC*fzA(3MX76eoQDRC~#evUQRTc zy2ucKA;{>cqO;X+-h1*ZT=02L zT<_VsJ|TY|9l;Da4YA41DgF?j-$_14)OV#zW_rrJ#o#*UUu~iPrzGQeb|mz2{b3+~ ztG1p>!NL(Z1<8Lc9vu7Nd)CQFk>qVYwC36Kb@t)$mu{eRo744t3=)t>IMxXS0VWmIGEy>>!*bDx)w$>WmVTp%u|&P22SO^4Q z*uI~xUq*%Xgv-{l+AF39yWq8X{-}a+HElLh+}3byHmW-^gP~=I;YxIsvWM>wIb4rA zSz_Ra%wskySo~>*&0-z+um9`N(PfByUvig#=bu&a7M!d9ySKjs*&Sd->_9UAtAqbf z-E}*hluLQZBjNCXRkG!}85-p9vnN=fBFLNML)eq0tc=-lCV17!z-UIA~` zec-?RftJWFt5p{O`b6(?V3gEp%)laFsk+KH^t~cVm-G)lzW%aI!Jj1r!tHU}9Wc+W zJWN-y*@3X$GvvXexa}OvOOg%_FEZKwX|+GxQu++Ot8tQm1Gjd#B!j#8fW`+5t|l5W z)Au=>_OtHKF1)xF3@mPZI~Hlf3-!+VWIgjJNJNb*#t3oM_CB4sj!`WUxqki@j1TB>Jn4X)%4uzL_=yL|CK7wI#;=S+j*h4S5XnC zcjE#RbH9Snd5kip|L}1WegOzzrAn~B>G7HT(mo9jFI-D%C|1Sg-VIioD~pVI-pik} zgcBKOav{7fu6-vebOi^3jlBsP1_5vPC$Dm*wKI|q-$3@RH>o3(mAjxODR9}yA3_g& zx{lYsfJKCbKR%KUvXn>rh zhcCbT2`#8?`}los)q(4%VCB#svuufH!lIc1<+V?X#n7rxNs;h#s~Glq-=2B+9v-{F zYWR?p#48ex)`WuvY#Z&;>aC-GCusS&o`&9%;4oMS_vRF)G6DO3OJy4lyxVHkeG2## zsq&hvln8&-$*U*_GJ=|h2;X^B?na(bn2EBtTO}Bm4(XJu%R_Pp7O?)#}k*pqv|_X66Y_rAU&s*lZkIGw)-;)|AAL# zZkh6w2D-^ZBJBdXMNIsXa;5NFxAse7Tu<}ne36cs9C&=B8`5N@C&Ry++@)*i&G8I= zD)fHo!csm8+)M34WapYY4X!IiA*BUw?%6hh*E9c_agP2kj5SzolQXRTUnupUYfBo= zTrK8X@hFubiB7GEo13ffWYvdiPA)g4;7eL`9D~}87VV@nJ{`$_$9(qYDt@4of~y{#K3ZmPx;1H8A@{%SX5i4yFB5_ARtHQ~x&kl${DUY0Wf0PP*5 zB_pj)$-w(-Om21nAg4?kt;eF~L|yFHKaryQO|MRVRfE-l}x3&C!wY zDu9>|4#(n_o$#4;@70#6O~2K0#vkgO0*N{tyPnuXV~Djj>f>R6n&=j zorh>QWc}?Y{E}?MJnORV~cVDoy-dxZ4^)|bm$Vt{J$Vd3Vc-*D>N8xc;UW=u| zCK-VlDAVe3=LulcjF6D#Cl}im*MN{|m@GQMmxC~p7qSq)kDwheEKE*$KKncWr0mZH^8l0lCZX3V;55IK2b2Bo%$(6K|y9i2A5)TyrZa_T?7fPI`eW84DPK^Kl z2kz_#yRu#jm~vNO7tZ6DGi%u07RuUGzjgR<>~W9s-2>0iz+4p(KG~^tE7HmH&Y6X z+uSP1JjJW0^y|wxH%b0`x^rI17{9!AzGW!2e$sN}o%Jc&*iA^!=hV`bP0h(DD5ftg zh@*!?!hz}Zu$|VcSQk#`^yj^g8*_E0zbyNhQsl@>t(Iso>sSpQ+1kcy21b<+4lq)A zW=E5+-3ugAOMOB-q*Q~mRhw_UX68Smn-5U;NKV=jV)sYMiq^StW`?nkGU%kf0KVM0ChTh+`tZ>oH+rtg zn(q*_Aa;$@Xvph&bvU1$DpGucP+JjI880gx^SXelOQP5x;y=U{bwVq;!G z-G$iW;H$sjfiMieiL$PIsYl{_=S)}t`;tRPJj>SpIf@+DtL5pZKQ<1MkuMYK%)2_X zuEM{q70{J4i(H({TXE{38ZMohZhP|&PAJ!fBloY3hm%demIP&2rgCK6wl9wxH49~0 zbF#E}nMpuh5{y(?6vj42g|ldhP3sBerHlOaDgn{N*G8llU091kDj#*uOyWoG7>ibO z#)k*fJU9(Ze#T~HA4M7w1hQo8#`?kZvs~{XYj5C^n@PMCxC%#cKp$r^oHv&FV#{^4 ztG6W1`o2egXSL+|!K~kNXFAP=^@Bd+@sh;Yzoj=6p8^cOf=b552K??uJ|lfBBc0_i zGmUKe?4veq<9_^=&15TLR@ihY^V1inC*17x_#OuhEHfq@uB^O(Q>cRI(9qHs{YImC zlHv^BsQc(8(CN;p@C`W$M}@*;aK~M$>#VBPmW+DBiBvka2J*Kz7NykzXu(D_+}g@_ zIz!gG{K>wqGXGPqrM6CI{S8@GRH7Ee`4F^Nb&~PZ;9+~=)HM=B$B~yD2HXx$2kDx; zsA*O@y_hQ?%DXOz_)~#9&k#**DJ>ZpO|~noA{uSW?UIYiLKaap z{(^90!V@e(ghnvQ=ChTzt}$?XuCnH?=XKf8x3&o))%0$S98pxAayyvsk`H|Ly0dMn z@ZHc8&4rmMp~7TjJ(^i*&01HHux zsj?vi1dGbe`j=6y(y-m@(2U zowvzKy{j=D-x?6%1{=3pTUe-JHEo5m5LeYo9*mpu>UC`UZ{pBI+nJwp{ z50M@KAiy|S4tSH;o>?gOB7Nuwn}*k15=m1$eIG;1ux6y9y@Nh@H(rFbObhHHx>To* zW`p7f@VN4YiK_;#XmzA-QecM5HWH^8t9x7@%oG<^Mcem~K)~kRHYs&&^=1=^Kc^pg zYHl%)C0Ld@Y>i?Ty=r3hGMEL;#%#yF&SQKNGas_CfFp8N+r*nq@@0hO!_@*j(OwKU zySz2kG$(7PiXT!<(R#!I7cgOml2RLq`ch!q)m4f_`K&-oC2c+O{m^j_M&fb3nGY8f zyS^>#v|5S9;8?2Zk2~*alvtcw#C*6okb&RXugA-6+~IN0I0G@i*5oE|jbEgv())Iz?`+a&k% zn7R7?O)u3{rJ$|?hDs68y9~5n)v##M3Ba_sy3Fh>rQ9FnL1<@Y6e4bOj5lh#vzGJi`5-f{rZzO7u6kp@VI)vH%C?&PwV)h#W%F>09dgmYQju$ON3zPxK`X((OV_=)xRU3|n( z0IYA!H2|^ind{k^eFtcA_wf&9n4GsusY+V;4eI7}%F+-V+=|Y|4hoMSh`6()k8asJ zz8?qG5Lc<)OJ`If-1T>-%~m;=>0jg5(vJk%F9z`d75G>1ABM9vNYFe+^lB%H1F@vn zM%EWDulZ`tkE2vdlJ%GIZoEAAp@dE9m(D!yW2t^;!EyW!t?aIMqy6eZ0k#evan24` z0d-69oiGoS41qRF5HOLSGO0_&t% zvfJ}(C0NL`beIEd54Yog7@0oN^V3}x3yU-8Tq-^vNHD%|#wE7VLZ;BzfgP>nF$<+# z?q)`+3pck3-Naqd)z#5&iPDQ{>mGb3#!M*=9r28O=i2rNX6?(B53Xa0sQXxDKVSLn z!XpJF=;+X}egE7t-%m_9!|TUt;g~}kczfXTQhfSe5*ErwoRJ9+#cMA90%1QPyoGrU{w@#&$#{u26If$jqrm{??=8i%2TM;}3&r{$w zb;CuyUc$>ILRKT)k5C?5(XXFdrO5m%@GI#fQWJ$0Xmz+rxgM+^c>ksau+uCNP~g}Z zVNSa|RLfxF=TRuKVhJFa9b+tAUf>X(Bj>(~%l6jJn)=FvxJiq7--ubj5?aLR4XeBQ z@<^`_GQS)wOz~THFY_u$PQnr-hFg--8Sl=A3if!HgeBk!R*b6aaW5Or@ixfygkPm( z@`IZ{v24d&VVS4}$M<YcsF zs7{qa);P09sw|PxY<2eIxz3}s&kyxMN1Njci#Ud~CPB~FI)`Jm7KKs==}XU9=s29c zz*jR9(s4h?xXw17iXA+(fKVXQb}e!6tzUz(X}*V83N*K!yV-~xk|k9#<10^_boYu~ zUhG^Hw>~TjwW`^0VF1@dEfdJd%O9Ilp#E*x(fMYbowR4COQ85;UOU$z_O3^c94jHS zDG#qw63EZySWsnZhxw3Y-oo3{rLt4$Z}Sf4wm#|bEES7W%?JpS8s@a+OB?6?>1?k9 zT&OAzJq5?L(c$m^)U&DYNhq4gOlA=h(nmIWH9jk$eSHr`IG(bxWkLJ)O$u?YPd?@` z{#%rsm(->XsKKZb-eYeR+#hzf8g)23@Bm(?&~(xfn9?B~3in9kU0%lbk2O5=cZ{N@ z2&oU~!6L_hwB!{<7BlwA7V+>_FI+DbfN;6UN~;9~R@O=d{nCjuscG-ptvZcpW+Pw} zeiZi_pbrgJ<>06tedp~B^41cY9Vs4MCf3v^KP;}?!n$0VLVOna5|X6^}O?nf9Q<}?0d#{g>g8f zaeDrkaB<=QcrzQ(p!3k!C%|SY(~A>_v#lnZA;}*3|;+dx}mk zkq+^kD5T3 z^lchHeRQ`xm5jN0espwlbUrdPo272^jC;C@;=$f$nY(w>uGB2%=S#@oaq9NCc3rvR zBnQbAP}Q(xGSDG9kZC|vfPMbCtCxsLXtSE2`&Zl2yF2{Na;_9kJIjq5FRgfLiarGg z?d4qaoLlt)58HX?90Si~x=*pWdMWNp97W4-a+|4LBLef&tEr~B65sBw7i)Fdf|M+% z+SUOllqPHhpD1_E`+M4Gx(oK(-X(ep7HNh?#&)K`Cqz`*Umk&nygp!{s4gwl87U-V zRhd{)3GWVi7#u733EUCPsbZ;lJz~UA((!OL`kwciZ9Y^I&!i9fLK*LCFG{X6Or#T*q`?Eo0}J zHk7fknRDFVV)n0O5k5Pzy}@8pzCd-Vmaekma5gksTGm>R7vJ_RsbP+xoL;IuetTB? zB|TC}ZHLE3_rgXZya)fdr6ok*cQ#{(XEU6`F08BM-11Y$lk~LHB62 z9zDIn_u?Y=kl0VoZYn?Zc>nl9XK8(4CbT>c&tEFiUtp~A$5$gha&o${o ze$%bd!g%sDpf>!eSOdJ*eR5(zH4tq)DpOVYTq{I`b>OD_O`)QJ!4#g&h%7C!9h7_{+L`B#{&u3M7{m`f*2`MTp$mEr8$ zZd{Q9^fi^8onI;pb$50?Lqud({t5ck7egl~C>XkUczwDqA|ld#HRs4@*U^X5Vr(l8 zHxJ}P(snP`j*mhD0G{ic>oz)fb>^F=8^5^m26m)-T}uUj^OYPxwDh`>z$^KThj;i^djZw5)%KpE1LM2j9w} z0cbw>qrj^3jywmGP>`EBjd=oC5tban0(kD3?t6>T zdYr1?Lu>2eJ6sxhjT@Sd7g)0yo{bP)uJAJ)zZBLZq@1+2AO7>}QUv@sv6&KX&e}!A zPM>TZe*$+8=(_(17zvL4VHeQWT@4G_M})tk(SC%l47Y{PdpzNMPgR9IEYm;?gkRR8 z^S(MOG=i4lk)*J#)?;-PZ;J7qC8;gURn9C z5j>5Lfw}DU3=Ub-HJ%RYv*iyeRNS08(|HH2Ne*Sb_HCxxHab#ooOF6>cU!c{`Aqwt z9>=yAFelD(RW7y`E|*s+9v#BRqAo;xKC-OF1oOW+O>niJ&$h2loF;4ZG_|Y6s;bRs zV)7e#m4po)}`AfAaS-;p?_=%;zH)+JXOK@o@(n)NpyxY{;mZ|<#_*=Nv>F6Po` zRV;#d``|$4KcP7$qlT8hp}xTkQ4h5G%Li}uqVYzZ+QsonGCg2Wa_7+;X&TfMDfNs0 zQfKYJj2FxVK6zhlk^7lpSVDq^rsnK?txX^ne0-a}zJ3?q;AEb*fA@1iYgT?PYpssE zo{Va}rzKnEd+gNRbVRNN7J+=j{na^1Cd$ReUn4?!dgU=9DjuV$DPp60;sUM%BxGWk z4i1^)a-2{QS1&cu@ywWWo1xNPot>SI#_-s(yDJ3`^v9@ZXtGjLjgJv@%ma=#>$z7T z7yCKs7T{`Ppk4iUOwL!%Uc@$Y)Q+7zP=0<%OqcI&&xxgmt19EsPd(Prf2m)*l}=1b zl9slXPNp2p>sgOvLN$8xuK!QCbmhJ%oJYUwag-}b#4q>o&0Oj>!5|I3kH`>?JJt%} zOW)#sk={VCdATJOQ>WST;IvUCPQ{pZ<9OavP!$-Nwil#|gRcM?oi4mkjEtE6&?`?- zp{gqHpy@(Mb-W+%Z}B`dbpk0Oe+wj4T~im2UT?q5j&tt4O)%f9z*p^b*ta--J~Lgh zT;O6mI-0=G0{}E9UK69Q5K2DU?9a%s3Z!C1l_4y+p7HX!oSD0gOgRL{awROMayY8M zB4}Aq7X)&RS604}Y6MHvSb>T)_Vp44_0@he{wSP z1ksnEOArj|Bc|r0x+$$bk$uFK40BhpkwR9wjC%~VY`eU;ajEloY_2sa7^&0tR7k7$ z?h7lVZfR`M7&>x0V!ep|_sO%;mJiw_Ui=Np!}3)#J*RFGe8Xu?ctDLxF{$>ECE3g9 z{0r^`?S>}!`GrGdx`$C>t`+xoFqgcFxli?5uqHN*bvLwMGQ!AVZ{Z}R3db|Ri`c|O0#eens3;>- zau*jD0|NtM;*{;BcQo8pIzK;5EpiYe&B{D3*{mk^mJdN@S{mKf70IV2hy&=D@E%PH z2!AKVJB5(&aJV3A+LH6*rIe)PJNE-kNlD38X03(b8&{pA!j1cMsi7;32lk^PT@l6* z<#Qo;+8hoYcT40i4Y%rh!Fu?a=15%rqqeuVJNRf9mqJGf>QzCw>7 zVPRBzv@WuOJXrTT7C@{#3PN$rs}HQjNlM`p34Lp;S?`#GR|z zSH+UeN72d=`^@x(PYA!c-IK^bgOZ0;bUDJWzxu2%K!uuc8tpMdItcI+8s}$e)W&^* zz}PsuU|nPXgt_N~{nA?t$^F-?5;|o%??c&2vR$@q&i~@VR@=|NAJy02+1*3tudXTS zHkXGz^vjMmZ3`-0&QNx_&7gUupgJ@Ay<T*4rrRN1&b24SBH7ab?z}X-}Ic5 z7<_GMOflda;%nIbt)j=(hG=6wduo;!#+mM=sd0e8a5~UY<6eemz#~8qKWe>PepyFU z7V2=HK2fhfg}c=L6sLZ+=iemikClEeA}~~MS0iBcMfi$iS~fQOD_Uh**9V6hRnkhs zCUz5$`6frp&^=s|8VoWuQ6v`r0@mjCZm=yJpX7Z0ey>+scWLkE#y2P!YgTg{!Fbe% z0PqZHc!IQ&#AZrM*V&V@_$aEJq!U-Kh)T15+#*aWX)EAGr7|&?im1`l`1`O^($e^iH7p;wDUw5!i^Y)T9Z9a$@_TF{qFr&V2=kcDbq01nxHeYY_0O})#FF1-<=)1d-!;tG-Cd72`)Rd z8RDNB4A_|~T)g0WDKn#gz{?FX>k?GN6Gc`FRHs@dhy^YccP4=S86wR6gE8IX0zF%D z?hiA+5%Xk_YH%s)Ng}0%@(Tsx!Xp108Zn^HK9NA&^MeR#V~}%As^J-*tPHo>B@ZRv zI(atDxydNY>)%3U{)ujzD~(9k(DdKOpEqm!8<%IN$p|&RXwhFbPu0^th(Q9J;5ca@ z|KtGrR@!lybZjAUvv{>gKM+$Zdl`T%oYwo@e2=(FGAiPnVOw@$Rt&0Wh_xDAk5$sj z#>P*c6Bu6|-^*Or+3%N`n!`VP2R6Z;3}bZi7PxUu(m=yq{x|&3k($2QBvkCnjJ6x! zNxooGC$K-fzbIACbE&s(OgOcUC>IoyGrl?qANI;t`6*!d{?U7n7`arG4QpAisLkE177 zRa}!mN6YNsJ(Lm}Qi(w%I4IgbJj^166iVSx>BX0bm0=i}MpARuasPKjq}-e#N^zmZ z&#V2$;)s?YKxMa|t+S^~c)l*1=s;aFA@8hAc=3nktJC(ybT^Ae&K(}F%O=6Qp)g#t z#|=760K6bt5tfS%z`_)Uh(-tW!acg->>dpb7meJ@tv~}TqVf+Iha51U!>D5xyFHT< z`8mOo$L4qNHc1s6A$fX!W=g29jF)>V;lt5Z-O+Joy8nVZB2Iq~L1wl+8SRc`Onv4kpaRYscM> z6LpkCwxt8gjSNe|v$LCr)jC2c0#*+{SonfX$Zv5<3hDUam1ka$@mm1v!G6p?5LDaK zI5R0_SFo-4Mk)-d8ZT3T5(BHXB2cc^&(S_IzhZrD_7^KthBT0GS^%cN?dqD~so}B~ z#M#K<@+w^(C%d-AnQs^HKgp#!{5t|ea_D!ryWjKW>p?{Ko%-na;s$ZgNV@O7?V)?i z4XL`;&ES+?56%(J(Qm|s^zWkE$eZF+w2OOas4cg3GVCmwDoOt6%VXB; z(&l}%IXTy6&_AdrWv)?B9>KmHqx1^xusLrQ74Ogj9&{?#_fEcYA1Yi}>n+Xwiu76k zA{A5Au`dYDE}29U%1PMm4|F?OeJzVpv42hNk)hhUs&ZSp*O&<`Xb?%elVd&p8n&pC ze?yleHh@g=;*oxXxNtWu01MpV0z+#hRqxRvuU=2dYa`ejCRqO2Y`Qt<*#h@Ecu)HF zI_*c{6DWWE|0(UQ!=l{w_hH;_r3932L=dFAOF_CphHj)`=%G;okq!xIk&@1#Lqs}; z?(Q03=ow4E8Fbq$vHEY(oKlgoq)~cClme+?Ih9xI3>nLIZbCFfx zv6`Qw#t#ck5q~BtMa~kqzNd`E=*YMvNSi=sgoZ}%(oh)nws)eOm%pp^5Ep-kz^PJ} zkzwa!J96ras=TSV0Ty=qQnU1xk>Xur$o&1D9Ij^FtR#NF<#!)e1+3+Zl9e;#Ta?Nl zes4h3;CQAd*9kbSf4+k!s_$vDR51*REQ$qQ-OWa$He2szElXaP){|vgCLD1+Nf^F=nH1YVzU}uOb{{d6~mb{TwPzZ~N=;|L&n#C|THO0fl{mu_;*ys1Cy7?B~Hhli1e+NbQX^dU^ zG31gR8|6)I(B%JC;Qzk@-USa;UFw+D`B@!<&YtzZ!dQ*#9S1P-+g2-BnZN1v*KOw-@L@{9CD9i{TeIpn!TGfoyx64) zXYf0>AXnIQaB#4trG+%Sy;Bq&9laxh0uvp*t6CCbBxe~3gFSssLJQT_4zEyF87`LS z2;g($^717b5FoMI+??DLqz;I2u-t5TrXN(>y7g(zek_+hcQfvZyvM|&3|+&qp_y21w>{_6D23=Hlrg_IYdRs~>fy1{`raw?Oj&+O?$)`>^Hq_h zTqcPC9Tv=t=6{e;pdqybH(I7mw_Ze%jd??r)uop%Z#X)$`o@dw;^L9=FEp2~zBV=~ z5~(L59cR-8F(p^$+YySWs1a7$Z=FeQxw&|R*;urWM>{ho_Onil`O*LnX93kLzDDtI7y`}nRAsC1yUMpr$!v02v%vk z?^b9iXUfZ}UMMh0k{h~={w1ZOT6f6}Sf7oN%{-*l--G0*5`EDORi+P+5vwdUO#U~7 z>~n>H7aDeesvJp58IG?LE}DtBtDG_CW752$eJ}hytNI(qXG8yByy~b=OaE1hCKCUU zvQgqWmcNq4QuyWFS0-+>qAa<3hefF=7}$NfCde=S#7KQ=?QL<=XNVt+!O>NK&kaU( zWc{Yze{8b)+Gfw5KV68sjRcE2HmeA;S@>yS*v(mZ&}|gMtD4Z{4a6)|ShhQR`A{h2 zuwdADH-mXuQiSqz@xRk~<)8;BDvY;4f;G01j1QN<^-WyhAZl4$So7(%%j2dNGCZl8 zWb-ykox{~2q-P?0N~Fm;O_Ql2>3Cbq`>>;|qAmie7!Y&NG1%;4)z$IEd1~wx=IR_( z2W#i(o)2Mge4i)tJm0`UzcO2H%p(IWyeXpK(HR_u@Tj%ql<)w*wvCKtrh*uZjd zxIebeX7TUdvES7qvCmf4Hoz$&Fvf8RjOnOKk;r_y>+ez z8Ozon>GI!30JReLZ=E~t#>AW-@s^e~v>6~K5q4=}ujTXggIL3h0@(2#Q~oXss_yV0t$t#LY&Da^=8PVuk>mI0UZ7bic|C++EPu@iLThBbbP zAQJz|=t_EXqN1t&;Vs-2JeCqw@&%PbR=L6XGCm~^-|KomC(@!gbi8tFwEG34X0t~l zXR_a<2O~YV49k&bBzZ7Tk%m}}LRbFB{=e6#t9{F)BxGxSk!hs|g_ed!qyq^#-v5e` z2{hQi;s1+T44KD3dEVtvfccF}o~c^%=m2qabb$51ia>Pzv&Xg-(>i1*ena2>b?$-@ zpCkO>SX!*)rJL%vyhGH9uUT%<6pr@l!s zs=B_{1t4?5tIe4r9eY+4FBdq8HEz8H#6<}qwR~OgB&*hP9AE7HA2^!iZ-G2Q2-wFK z=mU-hhPJwW6EDkSGye1fl%8MijlI*=85@QMryZPp>?)tq%uNw|Y<^6#bpR_Lnd$#Z2>2`HK%8)C9A8A97Yl_IT~5 zu9hR_BT}2mkv$fRJcYLg9qzu?r1-{CtK}~8Xh(=HaNgX9V}Im0Ea2g z$;A~HI}*z+`H6^!K(#aF?NqaCCu+-d3XJpRU}=CHou?ofKUi+XZb?!BfVZVl-YNos!lI4 zR@-T6YYc6fbqq-P4kd=$D6ciO#a~Zdj9$7#V3A%Bp8CdQ*@cwt(`76qI*K3zt950v}A4%C}8ImU)Glz_lx&cQbj@Uyek{>I8i>Y z0PtsD__t8N){v#?M|!;IwO!A=w`)1~S=wUVbv|OtgmjMMc|pJMv6ou#IC|nMob45; z>~;$hcU^!ILaYCHErPlkm=vWpX6 zQSpD>T|%UnsE)f=4FqYkRTDEtdpI?h9pnO1^wBqq)5wsN9V;%5zrdRB)g$G9+Hmr> z%yG&*kUjyJ5QdHI?bD*7jIR9Q07&fZ3x!}Y*1P#x?5``X!?axrm&2yw6)k+LWq2H~ z+wVQMy%CtBR9FCc;!g+;BZC34;YTXDq^Xy1FUjqd^++w65 z>T6*4YzeW#b7G!-cflv)XAM*B+!c{_yLPy_=q_jmeozp&H{?Ohoj3|3KhL zmOk*pP%(X^O`mmbS>&|m8P&yZWW6xQho?FM6 z^Tm3U$tkAR%La|Bl>}T<+VbM~Iat4~e0m93X?`m6Uj!q!kntjEo-B+`FPr5%w4w93 zjg5_4#xuHoe0&_(v?-g?rS%wGH=3+^{gK8Hy1ofe&y@AOHBPG`!Lluw?_32_gMxw( zLqK;^VFwu8sHlhS87gw!!Bo|HLp|eFKAh{8&wuofnE}{4O%O20#)gQRufMCdK;~5( zgE{QmW~fCQQhxgh`bW+mN7G`F|lN~cNGrt;Sl8455y7SE)i}&zz554SyX#M2o5-;cN_6*T{ ziy04z-opYWvDLXjwz;&U%_=-Tb&V?MoKF2~JsyLCJGZsyq!APp0&6 zc4wicrarIQz*azqBhG8HnEkVOnF<_F(5jpZU_)o1DrZg^+ zDiie=Tu)efoHwTNB|k8x!N+7TIJrnDEb%xZSGK&RL%(ej?RK*IM#IU;8D>F19Uzs- zV0O_K<;2I^R=B%AS)!r(nr^nqWhLkstey~{c7Kyrce7JB)Qap+9l+E1#2(_<>__h>(vrfL4|KQ1cvSLa!C3d z@_sn+O$(n8-;H5h07ZqUP!y83L<>eIcD3!gnw>S@n9Nx3UYF?b+Z~j&AR9jW#?C&P zTiDrs#)s}o2>AkQV*+@_|0X!^BvCfHsU_6SDd~8fzA=}GmtJ2n=;&a>8J>dbc+Ucx zCjX~jG%O+SKU8WZwt$C;$rnOe+CZ5cOLwK)8@`Gh`9;oWj5g@Q?Ra#E<}XBY!Hr4v za?Hs64&?L*=GTovDAaYoaE#geQJ2ct-b z_`&_?MnCVk0}xQgiv6q22__~}$m-W|JbAv&zS!Q;y@5jbQeZ_4$bs*^c3Y?*r?To% z%h}&_B7$e?jQgX;UKm7F=~TZVGzCen?|yQ0m-5FjC=wM-ZJk6%VS}!WjQDrl^xn*4 z4ha+b-}x(MrwzcZcGXLsD>%-O$YI`Bn!Fr+7p#xh0}tJTGh+UYyL6q21-XL51ex=G zy>XTiSM&gVa&DKA<~8G~o3iD!oos_&!hTmPMeLcpAYT!4z{9_~Kn*5l$$ruEAHe(V z0grZbw9A6Vyw8oB4b1D^B1`f?oc(*}i03Aa#>>*LN;~U3EN90nJSl}wSNosp>xGcw zENc|_T=u3;^n!3>^1QHIFZ#Vx6|X$Jrzs6{$96=VtTl#vrk{x6fPF+w$GZ)7)!OI1 z`~;&W+;>(ucDGiz$VBxyt2!9R2PhlcUCZUg$}WaWq_JGSmP1T=>U|0y(!#s- zOYq8E;B9H9jdAMHhrSJ(K0|5YH|t==fQNT|J__GAWGi+5MRl)u-sl+mnY$Uk4*4q; z-A6@UC`KO;^;3DbGzcsHDx8ExDLSD+xO5zGQteM?KjZDDH~M@OsmKx(NxH>|$>Hu` zG0)p)u88DsnBA$9#J6m)p5*;385B{gufBBX%Ii5-2`;$^o2>PaQ*@^O@h!r@_;@bu zyz9EO_BimN`0AY5!opj?%pf>rh)zs0?9+ZFJ31;(oRpl7=?m#7{)8B9&{!4nyU@+% z(-3|8nqU3S!s}Pi^6J;M<-o|r=Jmx!p%r0fE(~)WP;SJX(tyo6|J%6*fRD)zUv%ahmsgfX1&r&$5^H zlZ`Li>vnDYT1+VJc5(<+^R4xme%zoT&bY415;csEVjwpLiU%ABodkxU_|^QgS5Jyj z&gN;StCB882cEn3*Oo{t&+LvY7qz(B0+^bbLv--7kO7#Z>ytU}oRyl;jyQ@tA6|FBUiN5&orLDXm! z?{Fvse0#`&<+LoIS*lf(RmIaXEE>chX1jHD8_~cHy+XjivPPS1u5JMtPSIk&E@*ZJ zy6}IP2qH4Vud=x2eHz7T%@Ri1yIn9mvcEWawkF|uh*3*b{b?i+4#XCDd8vt?aIWjaFI8b$?og!1r=uHuJ!QJZ$1CC9RP+=c&4SyYtz_^&ahg z@`Ob>$0h#`h`9_(RP!SR2Px8~E%xE1g8dhtDCU`wZ7PuV{riB6=DvuaP|2m+E2823 zh&wv2YkBh@oz%n4PO~%$yjydRv`3v^oGLl`3!Wz%%snEh%~d#5mI{i0^rWc9cy%r( zI^!;H;_@1LvbzM)brUVqd(5ch+ZzpX}>(DbN+^XQQ7{;0R(U_aE{v(B5I>TzSJ}ls{_!0K|o;gjFN+c zKcF~0Y?Quls&hSt@0V0hqnS`^nNdUJ&?tPvT(0$2Lt7E4XZ&z=i{H5UT=u7 z!I<|^+^+!-J#fH3oqxa|zku$q+^}Ci|2;ztIOx9^GG=nk(cUBsBbGZzfWQYhVE;Qh z28Ia$fxm@Dwiy8@{gGq#U(Wq&(%P?w`P)bO|L<|UdGQBp4l5A{_UlxK)!hC0xbrzP zLUryk6(~4fy1Mk~@rTMQ!v2gVH1+4Bl*c|_4pKZTec$AIO^lb__XQFVgkM{~2e_uq z2=4*Ug@j!2to%4Sv=-`b5m(1@89_14@=cC$X-L%L^2Zz=pDu)$Ynwv(qnUqX{T9gz zs-#D8W`?q|(lApB5SZNhD<3`x)`ZQokB}ZaviWI9E%h}=UX^+-M%i83@18l@YN(r{ zn*R&rZAJUpplMinwyWkMb>n=G#6M(!T?!^@5LW~?%8jZ#ot@huF}|nv`8!LS=HJf% zPu7>)xFWY@WKg>R7l+9Bwc+f^i{|&&tU6xR0CFXw^4TX2ZZUc|dD)@K0KWBZzy;lH z;vYU)Z{I!lH^6d$3zYgKpNX0{I!I=hWP2U?se5Q7(R?nJ%q!uGfr2TR9(+#7&bx)#=6Gx;)x1;{J#IQUjo+chT+YS%OdZa78_8{Biwc*K^_k)C4SO zy21Im1VjL#wMMie0fVsPz2{a|+(}J7m#o_5v_e9c>ENWKq!ajb^#?!?xo$5nCP)xD zldCE|bX$pf&P5$?y)%@k%7<)+>l`Zaa8Y~mGC~@aehfj0ydsX{fm>8zSq^5H+J7IA zh9&=93Q=_vuzL|>B`Ko)GpbN}qP|vFpja80^y&h&<@9Ef=A|H&AY{Fy%+xzIgqS2w zx}v2OwZ2=tVQ%o+ZQw_EGAyEi^jWH*psTh*_TY4NBa!X3azpTcGRqEreg-Ch0?f&L z|E_@>VA}-jgKBwBC8rvU8i1EDB{enF_k5r#z45$*%5UZXd9fP$#Pijamr@Yuqn|wJ zLD}Ugu$Gx_^)Uz_1i83ePoNUQaT|EK6&TNc{3D@9qo?sg^PfEIHH4p4wd?%ZegCO~ zBtPw^3>;;&U;*ZTxFiA>Pv(HlV>P~ej|O)ID!St~&YS&%0}M<|-Fmm8 zhfkQ9dwY5c_@!mdmF^r%?1@>kB|*W4+L{&fF|ev8?QyYxM|aVCjPpO)Qoy-zGb0rJ z7dPiwe{f)et?Go(v-O#?^VSbIAh>`ti)hbF(@Z6T-P3fp+x(m3-PRaMZA=CsAzd^H zZ@sI`BAzBcUtktW0i(RR@kmQc14eUYbt5AqIk|46Uu-@~*HCVGsI{}VUiqsz0Ii{u zBl@oTv2nNpZ1r1m-n^u?v9&X6ot<{PZ|vpgD=pjXOfNY!VRaz=TP^L=b-EO$Xo<^S zQA4kldimswiF|c8*p?XEuI~irpWLgwXL4?3tgAjrWbk#Q_$;eE`GxW6fJ5 z=)y)}Gvvz206bat1q8ok4$rQRu;52kX_1ZXE6IIP6cS|TvK73=A_7moysv#I?)S@B zunQOqp1={Q;jvdTdQt!CVu%D>41u43=v;laFAK0=L{SJcX{}s&?ps?mT7Wq$?6(eJjST|rg{Eed9xya{oQWr(tXzhzQzU`PABEWwe&D- z7Mg>$xTX02(~c0Xu!iO7;%0CP|H_)PNO8Avt5?05o}iIi;cuJ)@E^c#c|RXpf^`qn zB`eTid6Y&j$;$vRH5j(Y6WvTHw<9h*@y>|<%E8%=#{Dh2!SpW8`t&rJkMi`zp0hfi z^e#EK(f3?=uwn|-vwxyw7@vzts3QbQ>Z#i!j`lZR0?CG(4?6>k!*;51bIBD>lb6YA z0Z^}FQR+<$dH@cUG(ge%q^1mTafZgL^t@oA(~^KGbA|J_!fU~9pH;q zCe{4bkoDW$ySM1XVIIifNC&mM*Fhw;e?p*zdy@Y<^#Yiqf$|HMEHqO;xh+cqn%x~Q zZy}cZdlx~9pZETT+0oj%EQ@WyL412W(QS4fP%z?ejN=;S6FE%VE9_DJw7>!r7*L*R z&7FU|JY<9!OFd-+1OT8keEut;sDcYnS-`uWEn(+*ahJEw0s!5;1DXXW|ANu}>LK`b zgNWa*%awO(_HWV`7|bOx9bQzZ%J#t3U>Te zG4~gbe%9vpR_?K)^wP%oa!m9hi(~MWlp9~^2cF1K57K9Kl|Q^T@B7l+KI6jbM-TJa zlu4!tdS_RV+-5GXZ+w8dy4^?Y zC%eZV2dLooUwP7w@qU&43f<97*h9a-TAdqK(*HI;>jP9vAxnrGuwPsYkx@Ss@E3?z_hWwX}254 z^R-#8O#w#Cjwt)PWVKy^@T(8Km_i_X5f6F>iIVctB9$W5&~{(aQ9S&w^=?MP#p-2_ z!3>*%ZCN{}a6Vc?!Tp}y<#lj2aW9nM?lzyDR$G78n@61j^}EPCAl1FxG?Ut&Oh=!& zG+J&d?(Vc3CGrv!GMAMS?(9%ME>5N2nN6E&KvjzfF-^WRYcpkrU*>%0cuKy_%Au)@ zROR?!2zEN<;#+S+0DUM<%nH`8xk>@gs}>2HEiWH#SViGCBBwJoi|277xd zbyOOE_*WFc-Id*vBXcgbmz^(_X1Y?0i>0FZOVsbNJ!#MljV@G;W{pNhGylvGY z>QYiQE!X>jb)kNKawxELy|FxNm2fAgDJY)wNfA;Bd9jT}Zbm%>yE*Gi`WRvXr?Z2X zn?&r)H!-%_T1IYxVmc#`+py-3B3s;m9C>rHD_j`Ng}8erR4XBl{ha`fI969u_~E?< zl>(8hVo=-ERFD01QIUOh5Zuj6%WGnLa@SsWde&aIM5hEg$0ymgO#jS8lJFXAhiq(+xookS|d(1`gz5@HY$&) zTh!C#(7c5zT@I01*C3u;#owI8!E7~SNz!58kOV1LM3P}bkTtupJkNZOvQVlr9)at|hq zc10MSV>Lp&Ylx`vbqRlLeMZ+8GZz%|l_B!_oM(9&X`{^h`vXc2W59K9W0m(|YlGs> z&!N;oXei`~+fL0(21#H6+-QVhJuJ0E2^}pfzk;iu7^zO!oBG)dkCQe`^K`Unr{aRT z3jhBCg*p)8!pz^{PQ&x<7kda<*8!=TybDqgaB1^ zwUDSsWhTSTCszhHNjv_zSWR8Zeacu%55=W^Toj(0pO$pU-JdP6zp+V(`CYoBy{c)e zFeb!PF*}edjaz_Ys0(c8)z=li9m(@j$?pj?{msLmrp)jC)jcERZuQ^CPvfdSBDX93 z?1_CUB-;BEqn`neHgx~1vEqp|*FyJ+%4;YE)%{q_=Pwm@ph+$R0m)NsspMj~LvfU! zD<@H%jzU)GsnUsOJTVGdR)v1^xVL*SkG}zo_ch38d!LOM^0-Dg0`i*G^dzpneDNWv zpS7y3mVcGLmjxW@}_l8WS$!6Q{F;1&uOM9RADM(@vMgxn7Cw zrta7PEcw(V(sh_kaZ#R{DP;djplI4je`58lew%;}Uvj2?7Ho6l@{aysi8HC4F32M2 z1*&3{TL-I~$CXP4>Zkk867^Ga3XykyPMbe(R*_HurgNB4JSQSO>^6MBol zS_<27IL7>kK6CZ3Q4?1S;l;J00+wSsvZqJ5<-&pwgssGh+{(7~hk-uxyZD_p?oZPr zC+40HezGA8@$H0r11eNbU%Hfj# zeupZ_PeXl-(uxq5Mbmtg2p`*CXlTY1(MhfzczhN^$SQwAy5ALFpQYkK=nAB#zglxO zg)!r4>&0bRBZ)G$KX#H4e2F#Z8Svd5=3e$3cgaIdVsbdrOQYj}W^T9K;#w3GPa4od zC*m&5cIvXVyZh|xV|18t-f+cz1@h6z>DBMdVy{sy73Qw>=UOki+?ngd!4`F<+x^m+ zzBFnl5)kjxF~=npDl0!SqS5 znFYMUhh^~hOFL>h?BA5dqvK^sm_59w{DCd3MULdnC)D!{)>8>~E$cZ>!yHP!^DX5s z-675o4&gFzt=dZ%-q9 z_R))%zAZNFgmEci7t1`tCo*y7DEn6QJ=XSzKbN8xCyab5&Gaex+k|R8f}90)zDC#e zfW=c$-f=I(_PoNz+OPpl5vTSQgui_aoQ=uo)fE2h2~94}j@y2AOKWQb9e)w6Y1vp@ zYQKnGB|pD-BB#^APS@bO?eq!;a`w&ZN0J4R@XnAlIww#~D_m!`uoUm=jyk21DO}Wv z)n*Sc^AgGWTuT4$<>f$tH?B!RqNeG)Wu%JNhbt3zx4lnRF*sI*at}>8X<;o%0I8ny zV3d+ST|bK5yP;q?S6<|Ku;~z`vL-oyZ1P9PesOfO?u-mj1LBTegK?OYX~)x$(9q?j zr2%ymNI*kFLuqNLQs|-&762^uiN$L&YierhL6QzMKvpGC_Fn-Em7ALz5GyEy(SBi6 z6k!!02Uka@n>`GWnORtVp?Z`z25)n$5*6qkUWu|5=jVqQd+xu@`2~r^gp%Lq<|k(!;}JS`#o7*Xr%EOmP3=I8ka|0jZy=s)Hv4*=m!+<8Pqf zesQBH3wn~0ZFaqkzgzso%qZXT$Y2G(-@A7Y6?MK_Z~JH`$IixzBvrsaB6wlNMK&7B^ZZAo6F}mt1DM z+;(*!_%=c2+a~5b=djyl+fdzWGB(y^vS};hB2dBjHVqBUQ{3Ro%xQxNW8%y~EftmL zXo$nZ`U1h0;i`FCJ&r^L((w0fCJgu?pFcmQjOuVT~g@6Mq!iYJBn1S&JOoaKklkW1;6<*?sPfP1{r z?NSm7(6cwRew5q;O`fSM?X!lSlobw%Gse|ZW7 zSxIN8{OITy=GIqJ`_BI0HgW}$`VAn?nY*7$8Yr+v6G|-YLz_qkG?H4oOQZi5s!xKCAf|@_L&o6N?{1#eCS_aS+TaV3M5mn)?_cL#|JF1}Z{Z z6zD#7nT13~f|ElY+;1H#(oLrCd-?KZR(5umS$Wkt10y3NAK$ncr!ghwZGCc=f7t7j zWviTLV;w50p!;}aWi%m{Uc;+)lALqmiyY|siR;+I$`cIr&zxTqNIXk(-`)8d^xMP_ zwdRBMD;A!Jx$&=rC*7Gh$g_`oRpi%2GcpbSnbKMPT4&>$lpC=6du8RPtN6qH_f&^{ z_jOXvr!Bg!gsR`BrRoD!w1(EfJqY|;O1;kyQc}o){=`k{?Bo>BW@yi@$LAB$XAXSU zDMr8Q9V;VKjWiCTf9WHrn@UIx!kE|Q@{JC= zCqpE`FVD?IUeVKh#YW(!uQz%*9Vr-;%zZNonfF3&GGtsJ$s#QGhDb&fR&xl*c(Cg% zk3;lbAx;v){PiZh$;o?n$4=UcbgOx}xdB&JR&FkyIMASw#qWW@oBiox&2}7t;;K42 z*1XA%ke}nR@{(M2IJ|L$nk`obi1|pi-OO!ZWWk0X_WTiQVr*TlI01Us2)W(k!*r7G z5yTYNdfbtc8}F7Ant{DTTnsbo8Z!%XMZL`sC+%fDkym@Gi*w^oLT1g^9rn8K2C-^E zO-gvIc6&pxbekqsqN^TnDESCBpdRy*hV%6ou`x5#1^`dlW5$`NFh;JEWM~U?JmPRE z?~U6Y=gwKamqVa>6`bjf)DE!XzPenOp9gk>cs61Ot2wInccpt@n%%^OdQc*U_2RrE zpixkVnO5OAaA~zeL9ww^waA9s_VTEQ>s5F1{B<@Du!R-B9yZ_@h1MG=g`h?4*Lv@L zuCXeyW2^*Ld*rb2;q>}m>X5C_xr~vOpD{c?|se3Qixd5%31HUZQaQ`t6@D7KF=;$FBo^5e+3iptk0J zlx=sTYiOv9mSJ_#PL2yNHt@4+8IjNFTS-16A?^HvHmIL}0WE%Aq_BFPzuYw> zYfvZyt=_22evQ^0gUX!LdzdYQ938?r-GVBBZC^1>cQulDDlHX~Sm(~SsXs}-leyM( zqV*|-VfN-YH(iudnbO%smbrjK%yipWf6H+0q9SSjRT}i72gd@2pTdtYCl6wWUJ~Xi zUFrzA?^f)wV4GR9PKzV875b{gvitQUmEDae$5oF>3|aj)?=+rnvM2`}RdEr;n~D`T zK1;hW*V$L;o2iLRDAx0`AJp|vUUy~>C&IaQ(h{>DOb)&D((aBZzC5iT#z{AHby2Jf zaccbPI{Q6~VY$JNKPK8Pl_O0Wt!&jimA$7Ul4L{z#&K)}b*oQ>?O7%%`S>8;W09dz z149`f>+0dTTRjDp;B8k*djKbS@6x)18ULxo=#luR@+~Q*44u3Fjb&P}hioA<`meuF)1@5|A&YaEnz0uNn^q;YSAs*VQ=LMS&{_{mc~Wcas*y)g7@RUijBn3#w;LC+W!$>T^aVeF2ZkrDgW z`nz8{h20~5-=+-nX+K&h#NdigL=Rg$$Y2GZ zZGLL=uv&}~6BUV2_uT(>uih*4+5DxsGA_a1gxi=BnmXjkHglWM=T z@uw6Z--ym;+KL!kakSf65hUIq@y;+ke?QmLG*bt4Am;O9uo@&=Yu&r;qQyBb~-pH}g%0veHENmCNQy+xz}` zkUqn_>C<8CB6bGXuZqAt0ELP&eEgDEknAfsv+?EB?1kK;%z`n28NHVbyYkJ_-d4*O z!)`+kW`^-B_T5enknskauD~#hh%!I#)bC}eL!s*P9i9s#D zoFrqkRXhTAsUfFv6UN89MzKWHKU7uJz}C)8rtQXkQ(l_|@*va<{OGL&yvCYYcj;Ao zs=34Wk@C3H5#ULK#g*OH5ldjoWav?;`x>gv33>Mna`})Zpx6WkHD;DyqcIhz9G$8v zFzxa-W46CazkwWSkp(rm59IRcW3+ETkd^X#>~MG9@AI(R!sf6eZ2@;_i%;k5?f3is zN`3fw7rs+`edx1uq_W70pGWJIscQ#>s*s8w{ua$GlBd)E>M>y`)_wo#>gwTaDxV|O z4l#Rn;{9Jj0nV}<9~L|@2YHjLEU$is_8Qh7tdcrn!2YQD;ru2-jYE?B_@Zx}1zx>U zYPQo<7$ep9z(_IzL%!M5_;p|=geTwiYDa82&;z`8b}~!UL{)#1_w2>AqUs5zfJKrx zrH|L-JA78^A37y#>Zvm~!2~IsbSdX6>!Y~_dhrXR2PRH({O28sYB!O&IF@`L%(a+9 zObS6v5}kHfY+=6R7mZSrjZ}rc3>m9WM03oy3nG1LeULAPeTH@3FOQ$x(XX?beN=E$ z$T3JhwBYne1EPOdGEZ5I^m_T!u(1G^AM9~~!y%FY5)Nj<4bB1r63k4>ja8@VsrnF5Kw zYbn@3Ko3%KLl!+0)5%>-T(z4TlSxNsf8fpET*vKfQr&P++apsFm%m)qglpWRKc&F( z-un?UVeEJnXJg>S;oiazYb6k74u>1 zgmZj(eFFETZ#tSTPS}LTsJ^D3eE#%M+*W!%ou}DJDJG<;;ZTg}gg-`1J7r5!#=l1r z-__@CZEO0u`hggcPgxSzlE1b{B1pKFSb-K-fE+tjX*{rR!$mNt^{vG(9Y?e(w{8>K8B|KXwlr*1=!fi^`a6Bqi_oBaBzM#jl18QP7gwgh}JE!PkD_~L_t%J>|1&z%O zGLl7a|61YM`56v2<@|pC(LyvL55^{Ft&O7a|GsvBqoJ@v~9{?<_1QcN>m@cZf|0|zD)v1v3MtOQS9cDH*HUWX! zlHEyQL0ce9kH6u4A5^W+*l;p(yu0_i)aCr81}ssGwbPv=cJ<^7nSk647t1yHz%N~w zr**hX2tGBZRA0PryIZftOc>gBetr(9*!DAw#pjrJ=)}aD&ruP}$n!$#F6Df_eBP`0*jT+92Wm#f{9$KiZpHQW z^KS6`n-!6-)HcMaK8CCF+wL77&eTHeKq<844T zFg#&f)n~nkxaX52$y?CYXLV1QvZT0JiSFZZ5dZBk!{ zn_lqQ%TCu5YbN2z1D>%iGe!V?u&`)6!*{Ap0ZwoizMTqmY;0_j*T+noZW)1ZeIFjg zuOw_X`rzwP-YDH%Cm0hakE)f7OXtz<#wPTv1_lN`dh`ew9AMt3ee7X_mdwn|u|2C0 zU3&MalljabX>~Z*44C(%GAC`TCT)RNJ*z-K8!bZ{E#pAN4&35R@~g}e-MYL5wc0`Z z+{@>{wBlw~x>A(*$X9eo2~(Gu(7#1RSJwurb2%isif{+cru&FEzltZ)SA_1FrY=;% z;g#DV$>9Ji3^a^0@p~Y$4G$0Jo+VnsNbV!V2sA01=)=1Q#M1WFr)7EP_>zwNB~Ww_g8 diff --git a/contributing-docs/testing/dag_testing.rst b/contributing-docs/testing/dag_testing.rst index 7e311171ce019..0bf506c2f321a 100644 --- a/contributing-docs/testing/dag_testing.rst +++ b/contributing-docs/testing/dag_testing.rst @@ -20,31 +20,22 @@ DAG Testing =========== To ease and speed up the process of developing DAGs, you can use -py:class:`~airflow.executors.debug_executor.DebugExecutor`, which is a single process executor -for debugging purposes. Using this executor, you can run and debug DAGs from your IDE. +py:meth:`~airflow.models.dag.DAG.test`, which will run a dag in a single process. To set up the IDE: 1. Add ``main`` block at the end of your DAG file to make it runnable. -It will run a backfill job: .. code-block:: python if __name__ == "__main__": - dag.clear() - dag.run() + dag.test() -2. Set up ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in the run configuration of your IDE. - Make sure to also set up all environment variables required by your DAG. - 3. Run and debug the DAG file. -Additionally, ``DebugExecutor`` can be used in a fail-fast mode that will make -all other running or scheduled tasks fail immediately. To enable this option, set -``AIRFLOW__DEBUG__FAIL_FAST=True`` or adjust ``fail_fast`` option in your ``airflow.cfg``. -Also, with the Airflow CLI command ``airflow dags test``, you can execute one complete run of a DAG: +You can also run the dag in the same manner with the Airflow CLI command ``airflow dags test``: .. code-block:: bash diff --git a/dev/tests_common/test_utils/system_tests.py b/dev/tests_common/test_utils/system_tests.py index 578ee6cc04d43..6558ae2d1e4cf 100644 --- a/dev/tests_common/test_utils/system_tests.py +++ b/dev/tests_common/test_utils/system_tests.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -def get_test_run(dag): +def get_test_run(dag, **test_kwargs): def callback(context: Context): ti = context["dag_run"].get_task_instances() if not ti: @@ -60,7 +60,10 @@ def test_run(): dag.on_success_callback = add_callback(dag.on_success_callback, callback) # If the env variable ``_AIRFLOW__SYSTEM_TEST_USE_EXECUTOR`` is set, then use an executor to run the # DAG - dag_run = dag.test(use_executor=os.environ.get("_AIRFLOW__SYSTEM_TEST_USE_EXECUTOR") == "1") + dag_run = dag.test( + use_executor=os.environ.get("_AIRFLOW__SYSTEM_TEST_USE_EXECUTOR") == "1", + **test_kwargs, + ) assert ( dag_run.state == DagRunState.SUCCESS ), "The system test failed, please look at the logs to find out the underlying failed task(s)" diff --git a/dev/tests_common/test_utils/system_tests_class.py b/dev/tests_common/test_utils/system_tests_class.py index 836782b8584c9..5abdca96bee06 100644 --- a/dev/tests_common/test_utils/system_tests_class.py +++ b/dev/tests_common/test_utils/system_tests_class.py @@ -28,7 +28,6 @@ from airflow.configuration import AIRFLOW_HOME, AirflowConfigParser, get_airflow_config from airflow.exceptions import AirflowException -from airflow.models.dagbag import DagBag from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER from dev.tests_common.test_utils.logging_command_executor import get_executor @@ -131,31 +130,6 @@ def _print_all_log_files(): with open(filepath) as f: print(f.read()) - def run_dag(self, dag_id: str, dag_folder: str = DEFAULT_DAG_FOLDER) -> None: - """ - Runs example dag by its ID. - - :param dag_id: id of a DAG to be run - :param dag_folder: directory where to look for the specific DAG. Relative to AIRFLOW_HOME. - """ - self.log.info("Looking for DAG: %s in %s", dag_id, dag_folder) - dag_bag = DagBag(dag_folder=dag_folder, include_examples=False) - dag = dag_bag.get_dag(dag_id) - if dag is None: - raise AirflowException( - f"The Dag {dag_id} could not be found. It's either an import problem, wrong dag_id or DAG is " - "not in provided dag_folder.The content of " - f"the {dag_folder} folder is {os.listdir(dag_folder)}" - ) - - self.log.info("Attempting to run DAG: %s", dag_id) - dag.clear() - try: - dag.run(ignore_first_depends_on_past=True, verbose=True) - except Exception: - self._print_all_log_files() - raise - @staticmethod def create_dummy_file(filename, dir_path="/tmp"): os.makedirs(dir_path, exist_ok=True) diff --git a/docs/apache-airflow/core-concepts/debug.rst b/docs/apache-airflow/core-concepts/debug.rst index 9ab7819b8b307..d58c490854527 100644 --- a/docs/apache-airflow/core-concepts/debug.rst +++ b/docs/apache-airflow/core-concepts/debug.rst @@ -122,18 +122,9 @@ For more information on setting the configuration, see :doc:`../../howto/set-con 1. Add ``main`` block at the end of your DAG file to make it runnable. -It will run a backfill job: - .. code-block:: python if __name__ == "__main__": - from airflow.utils.state import State - - dag.clear() - dag.run() - - -2. Setup ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in run configuration of your IDE. In - this step you should also setup all environment variables required by your DAG. + dag.test() -3. Run / debug the DAG file. +2. Run / debug the DAG file. diff --git a/providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py b/providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py index 33abc67b639d2..3c008ee5ca262 100644 --- a/providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py +++ b/providers/src/airflow/providers/google/marketing_platform/example_dags/example_display_video.py @@ -92,7 +92,7 @@ "example_display_video_misc", start_date=START_DATE, catchup=False, -) as dag2: +) as dag_example_display_video_misc: # [START howto_google_display_video_upload_multiple_entity_read_files_to_big_query] upload_erf_to_bq = GCSToBigQueryOperator( task_id="upload_erf_to_bq", @@ -125,7 +125,7 @@ "example_display_video_sdf", start_date=START_DATE, catchup=False, -) as dag3: +) as dag_example_display_video_sdf: # [START howto_google_display_video_create_sdf_download_task_operator] create_sdf_download_task = GoogleDisplayVideo360CreateSDFDownloadTaskOperator( task_id="create_sdf_download_task", body_request=CREATE_SDF_DOWNLOAD_TASK_BODY_REQUEST diff --git a/providers/tests/google/cloud/operators/test_dataprep_system.py b/providers/tests/google/cloud/operators/test_dataprep_system.py index 96f47fa3e365d..dad77ac4ff806 100644 --- a/providers/tests/google/cloud/operators/test_dataprep_system.py +++ b/providers/tests/google/cloud/operators/test_dataprep_system.py @@ -26,7 +26,8 @@ from airflow.utils.session import create_session from dev.tests_common.test_utils.db import clear_db_connections -from dev.tests_common.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest +from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest +from dev.tests_common.test_utils.system_tests import get_test_run TOKEN = os.environ.get("DATAPREP_TOKEN") EXTRA = {"token": TOKEN} @@ -52,4 +53,7 @@ def teardown_method(self): clear_db_connections() def test_run_example_dag(self): - self.run_dag(dag_id="example_dataprep", dag_folder=CLOUD_DAG_FOLDER) + from providers.tests.system.google.cloud.dataprep.example_dataprep import dag + + run = get_test_run(dag) + run() diff --git a/providers/tests/google/cloud/operators/test_datastore_system.py b/providers/tests/google/cloud/operators/test_datastore_system.py index a98215a5317a3..8807288358bae 100644 --- a/providers/tests/google/cloud/operators/test_datastore_system.py +++ b/providers/tests/google/cloud/operators/test_datastore_system.py @@ -44,8 +44,8 @@ def teardown_method(self): @provide_gcp_context(GCP_DATASTORE_KEY) def test_run_example_dag(self): - self.run_dag("example_gcp_datastore", CLOUD_DAG_FOLDER) + self.run_dag("example_gcp_datastore", CLOUD_DAG_FOLDER) # this dag does not exist? @provide_gcp_context(GCP_DATASTORE_KEY) def test_run_example_dag_operations(self): - self.run_dag("example_gcp_datastore_operations", CLOUD_DAG_FOLDER) + self.run_dag("example_gcp_datastore_operations", CLOUD_DAG_FOLDER) # this dag does not exist? diff --git a/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py index ba24a0c34da25..9cb0a9be5264e 100644 --- a/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py @@ -25,13 +25,14 @@ from airflow.exceptions import AirflowException from airflow.models import Connection +from airflow.providers.google.cloud.example_dags import example_facebook_ads_to_gcs from airflow.utils.process_utils import patch_environ from dev.tests_common.test_utils.gcp_system_helpers import ( - CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context, ) +from dev.tests_common.test_utils.system_tests import get_test_run from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") @@ -71,4 +72,5 @@ class TestFacebookAdsToGcsExampleDagsSystem(GoogleSystemTest): @provide_gcp_context(GCP_BIGQUERY_KEY) @provide_facebook_connection(FACEBOOK_CREDENTIALS_PATH) def test_dag_example(self): - self.run_dag("example_facebook_ads_to_gcs", CLOUD_DAG_FOLDER) + run = get_test_run(example_facebook_ads_to_gcs.dag) + run() diff --git a/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py index afd0856fad244..d556f2d86e2f2 100644 --- a/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py @@ -20,12 +20,14 @@ import pytest +from airflow.providers.google.cloud.example_dags import example_salesforce_to_gcs + from dev.tests_common.test_utils.gcp_system_helpers import ( - CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context, ) from dev.tests_common.test_utils.salesforce_system_helpers import provide_salesforce_connection +from dev.tests_common.test_utils.system_tests import get_test_run from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") @@ -42,4 +44,5 @@ class TestSalesforceIntoGCSExample(GoogleSystemTest): @provide_gcp_context(GCP_BIGQUERY_KEY) @provide_salesforce_connection(SALESFORCE_CREDENTIALS_PATH) def test_run_example_dag_salesforce_to_gcs_operator(self): - self.run_dag("example_salesforce_to_gcs", CLOUD_DAG_FOLDER) + run = get_test_run(example_salesforce_to_gcs.dag) + run() diff --git a/providers/tests/google/marketing_platform/operators/test_display_video_system.py b/providers/tests/google/marketing_platform/operators/test_display_video_system.py index 78f5d4ee021f9..49f44948abfdf 100644 --- a/providers/tests/google/marketing_platform/operators/test_display_video_system.py +++ b/providers/tests/google/marketing_platform/operators/test_display_video_system.py @@ -19,13 +19,18 @@ import pytest from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook -from airflow.providers.google.marketing_platform.example_dags.example_display_video import BUCKET +from airflow.providers.google.marketing_platform.example_dags.example_display_video import ( + BUCKET, + dag_example_display_video_misc, + dag_example_display_video_sdf, +) from dev.tests_common.test_utils.gcp_system_helpers import ( MARKETING_DAG_FOLDER, GoogleSystemTest, provide_gcp_context, ) +from dev.tests_common.test_utils.system_tests import get_test_run from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GMP_KEY # Requires the following scope: @@ -50,12 +55,14 @@ def teardown_method(self): @provide_gcp_context(GMP_KEY, scopes=SCOPES) def test_run_example_dag(self): - self.run_dag("example_display_video", MARKETING_DAG_FOLDER) + self.run_dag("example_display_video", MARKETING_DAG_FOLDER) # this dag does not exist? @provide_gcp_context(GMP_KEY, scopes=SCOPES) def test_run_example_dag_misc(self): - self.run_dag("example_display_video_misc", MARKETING_DAG_FOLDER) + run = get_test_run(dag_example_display_video_misc) + run() @provide_gcp_context(GMP_KEY, scopes=SCOPES) def test_run_example_dag_sdf(self): - self.run_dag("example_display_video_sdf", MARKETING_DAG_FOLDER) + run = get_test_run(dag_example_display_video_sdf) + run() diff --git a/providers/tests/system/google/cloud/dataprep/example_dataprep.py b/providers/tests/system/google/cloud/dataprep/example_dataprep.py index 9f603f43fb1b3..cdc736a41c66b 100644 --- a/providers/tests/system/google/cloud/dataprep/example_dataprep.py +++ b/providers/tests/system/google/cloud/dataprep/example_dataprep.py @@ -313,6 +313,7 @@ def delete_connection(connection_id: str) -> None: # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() + from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py index 00364794f1d53..f0c9a18c1a567 100644 --- a/tests/cli/commands/test_dag_command.py +++ b/tests/cli/commands/test_dag_command.py @@ -322,7 +322,7 @@ def test_cli_backfill_ignore_first_depends_on_past(self, mock_run): We just check we call dag.run() right. The behaviour of that kwarg is tested in test_jobs """ - dag_id = "test_dagrun_states_deadlock" + dag_id = "example_bash_operator" run_date = DEFAULT_DATE + timedelta(days=1) args = [ "dags", diff --git a/tests/core/test_example_dags_system.py b/tests/core/test_example_dags_system.py index c60b7325b125d..bd34d9bb15919 100644 --- a/tests/core/test_example_dags_system.py +++ b/tests/core/test_example_dags_system.py @@ -17,16 +17,128 @@ # under the License. from __future__ import annotations +from datetime import timedelta + +import pendulum import pytest +from sqlalchemy import select + +from airflow.models import DagRun +from airflow.operators.python import PythonOperator +from airflow.utils.module_loading import import_string +from airflow.utils.state import DagRunState +from airflow.utils.trigger_rule import TriggerRule +from dev.tests_common.test_utils.system_tests import get_test_run from dev.tests_common.test_utils.system_tests_class import SystemTest +def fail(): + raise ValueError + + +def get_dag_success(dag_maker): + with dag_maker( + dag_id="test_dagrun_states_success", + schedule=timedelta(days=1), + ) as dag: + dag4_task1 = PythonOperator( + task_id="test_dagrun_fail", + python_callable=fail, + ) + dag4_task2 = PythonOperator( + task_id="test_dagrun_succeed", trigger_rule=TriggerRule.ALL_FAILED, python_callable=print + ) + dag4_task2.set_upstream(dag4_task1) + return dag + + +def get_dag_fail(dag_maker): + with dag_maker( + dag_id="test_dagrun_states_fail", + schedule=timedelta(days=1), + ) as dag: + dag3_task1 = PythonOperator(task_id="to_fail", python_callable=fail) + dag3_task2 = PythonOperator(task_id="to_succeed", python_callable=print) + dag3_task2.set_upstream(dag3_task1) + return dag + + +def get_dag_fail_root(dag_maker): + with dag_maker( + dag_id="test_dagrun_states_root_fail", + schedule=timedelta(days=1), + ) as dag: + PythonOperator(task_id="test_dagrun_succeed", python_callable=print) + PythonOperator( + task_id="test_dagrun_fail", + python_callable=fail, + ) + return dag + + @pytest.mark.system("core") class TestExampleDagsSystem(SystemTest): @pytest.mark.parametrize( - "dag_id", + "module", ["example_bash_operator", "example_branch_operator", "tutorial_dag", "example_dag_decorator"], ) - def test_dag_example(self, dag_id): - self.run_dag(dag_id=dag_id) + def test_dag_example(self, module): + test_run = import_string(f"airflow.example_dags.{module}.test_run") + test_run() + + @pytest.mark.parametrize( + "factory, expected", + [ + (get_dag_fail, "failed"), + (get_dag_fail_root, "failed"), + (get_dag_success, "success"), + ], + ) + def test_dag_run_final_state(self, factory, expected, dag_maker, session): + """ + These tests are migrated tests that were added in PR #1289 + which was fixing issue #1225. + + I would be very surprised if these things were not covered elsewhere already + but, just in case, I'm migrating them to system tests. + """ + dag = factory(dag_maker) + run = get_test_run(dag) + with pytest.raises(AssertionError, match="The system test failed"): + run() + dr = session.scalar(select(DagRun)) + assert dr.state == "failed" + + def test_dag_root_task_start_date_future(self, dag_maker, session): + """ + These tests are migrated tests that were added in PR #1289 + which was fixing issue #1225. + + This one tests what happens when there's a dag with a root task with future start date. + + The dag should run, but no TI should be created for the task where start date in future. + """ + exec_date = pendulum.datetime(2021, 1, 1) + fut_start_date = pendulum.datetime(2021, 2, 1) + with dag_maker( + dag_id="dagrun_states_root_future", + schedule=timedelta(days=1), + catchup=False, + ) as dag: + PythonOperator( + task_id="current", + python_callable=lambda: print("hello"), + ) + PythonOperator( + task_id="future", + python_callable=lambda: print("hello"), + start_date=fut_start_date, + ) + run = get_test_run(dag, execution_date=exec_date) + run() + dr = session.scalar(select(DagRun)) + tis = dr.task_instances + assert dr.state == DagRunState.SUCCESS + assert len(tis) == 1 + assert tis[0].task_id == "current" diff --git a/tests/dags/test_future_start_date.py b/tests/dags/test_future_start_date.py new file mode 100644 index 0000000000000..dadfbff600f60 --- /dev/null +++ b/tests/dags/test_future_start_date.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from datetime import timedelta + +import pendulum + +from airflow.models.dag import DAG +from airflow.operators.empty import EmptyOperator +from airflow.operators.python import PythonOperator + +exec_date = pendulum.datetime(2021, 1, 1) +fut_start_date = pendulum.datetime(2021, 2, 1) +with DAG( + dag_id="test_dagrun_states_root_future", + schedule=timedelta(days=1), + catchup=True, + start_date=exec_date, +) as dag: + EmptyOperator(task_id="current") + PythonOperator( + task_id="future", + python_callable=lambda: print("hello"), + start_date=fut_start_date, + ) diff --git a/tests/dags/test_issue_1225.py b/tests/dags/test_issue_1225.py deleted file mode 100644 index 96a3ad156269e..0000000000000 --- a/tests/dags/test_issue_1225.py +++ /dev/null @@ -1,149 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -DAG designed to test what happens when a DAG with pooled tasks is run -by a BackfillJob. -Addresses issue #1225. -""" - -from __future__ import annotations - -from datetime import datetime, timedelta - -from airflow.models.dag import DAG -from airflow.operators.empty import EmptyOperator -from airflow.operators.python import PythonOperator -from airflow.utils.trigger_rule import TriggerRule - -DEFAULT_DATE = datetime(2016, 1, 1) -default_args = dict(start_date=DEFAULT_DATE, owner="airflow") - - -def fail(): - raise ValueError("Expected failure.") - - -# DAG tests backfill with pooled tasks -# Previously backfill would queue the task but never run it -dag1 = DAG( - dag_id="test_backfill_pooled_task_dag", - schedule=timedelta(days=1), - default_args=default_args, -) -dag1_task1 = EmptyOperator( - task_id="test_backfill_pooled_task", - dag=dag1, - pool="test_backfill_pooled_task_pool", -) - -# dag2 has been moved to test_prev_dagrun_dep.py - -# DAG tests that a Dag run that doesn't complete is marked failed -dag3 = DAG( - dag_id="test_dagrun_states_fail", - schedule=timedelta(days=1), - default_args=default_args, -) -dag3_task1 = PythonOperator(task_id="test_dagrun_fail", dag=dag3, python_callable=fail) -dag3_task2 = EmptyOperator( - task_id="test_dagrun_succeed", - dag=dag3, -) -dag3_task2.set_upstream(dag3_task1) - -# DAG tests that a Dag run that completes but has a failure is marked success -dag4 = DAG( - dag_id="test_dagrun_states_success", - schedule=timedelta(days=1), - default_args=default_args, -) -dag4_task1 = PythonOperator( - task_id="test_dagrun_fail", - dag=dag4, - python_callable=fail, -) -dag4_task2 = EmptyOperator(task_id="test_dagrun_succeed", dag=dag4, trigger_rule=TriggerRule.ALL_FAILED) -dag4_task2.set_upstream(dag4_task1) - -# DAG tests that a Dag run that completes but has a root failure is marked fail -dag5 = DAG( - dag_id="test_dagrun_states_root_fail", - schedule=timedelta(days=1), - default_args=default_args, -) -dag5_task1 = EmptyOperator( - task_id="test_dagrun_succeed", - dag=dag5, -) -dag5_task2 = PythonOperator( - task_id="test_dagrun_fail", - dag=dag5, - python_callable=fail, -) - -# DAG tests that a Dag run that is deadlocked with no states is failed -dag6 = DAG( - dag_id="test_dagrun_states_deadlock", - schedule=timedelta(days=1), - default_args=default_args, -) -dag6_task1 = EmptyOperator( - task_id="test_depends_on_past", - depends_on_past=True, - dag=dag6, -) -dag6_task2 = EmptyOperator( - task_id="test_depends_on_past_2", - depends_on_past=True, - dag=dag6, -) -dag6_task2.set_upstream(dag6_task1) - - -# DAG tests that a Dag run that doesn't complete but has a root failure is marked running -dag8 = DAG( - dag_id="test_dagrun_states_root_fail_unfinished", - schedule=timedelta(days=1), - default_args=default_args, -) -dag8_task1 = EmptyOperator( - task_id="test_dagrun_unfinished", # The test will unset the task instance state after - # running this test - dag=dag8, -) -dag8_task2 = PythonOperator( - task_id="test_dagrun_fail", - dag=dag8, - python_callable=fail, -) - -# DAG tests that a Dag run that completes but has a root in the future is marked as success -dag9 = DAG( - dag_id="test_dagrun_states_root_future", - schedule=timedelta(days=1), - default_args=default_args, -) -dag9_task1 = EmptyOperator( - task_id="current", - dag=dag9, -) -dag9_task2 = EmptyOperator( - task_id="future", - dag=dag9, - start_date=DEFAULT_DATE + timedelta(days=1), -) diff --git a/tests/jobs/test_backfill_job.py b/tests/jobs/test_backfill_job.py index f1bb6f17d05c4..dead9be86230b 100644 --- a/tests/jobs/test_backfill_job.py +++ b/tests/jobs/test_backfill_job.py @@ -32,7 +32,6 @@ from airflow.cli import cli_parser from airflow.exceptions import ( AirflowException, - AirflowTaskTimeout, BackfillUnfinished, DagConcurrencyLimitReached, NoAvailablePoolSlot, @@ -54,7 +53,6 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import DagRunState, State, TaskInstanceState -from airflow.utils.timeout import timeout from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from tests.listeners import dag_listener @@ -1070,34 +1068,6 @@ def test_backfill_ordered_concurrent_execute(self, dag_maker, mock_executor): ], ] - def test_backfill_pooled_tasks(self): - """ - Test that queued tasks are executed by BackfillJobRunner - """ - session = settings.Session() - pool = Pool(pool="test_backfill_pooled_task_pool", slots=1, include_deferred=False) - session.add(pool) - session.commit() - session.close() - - dag = self.dagbag.get_dag("test_backfill_pooled_task_dag") - dag.clear() - - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag, start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - - # run with timeout because this creates an infinite loop if not - # caught - try: - with timeout(seconds=20): - run_job(job=job, execute_callable=job_runner._execute) - except AirflowTaskTimeout: - logger.info("Timeout while waiting for task to complete") - run_id = f"backfill__{DEFAULT_DATE.isoformat()}" - ti = TI(task=dag.get_task("test_backfill_pooled_task"), run_id=run_id) - ti.refresh_from_db() - assert ti.state == State.SUCCESS - @pytest.mark.parametrize("ignore_depends_on_past", [True, False]) def test_backfill_depends_on_past_works_independently_on_ignore_depends_on_past( self, ignore_depends_on_past, mock_executor diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index a5748ebaeef9c..639a3528fad8c 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -43,7 +43,8 @@ from airflow.callbacks.database_callback_sink import DatabaseCallbackSink from airflow.callbacks.pipe_callback_sink import PipeCallbackSink from airflow.dag_processing.manager import DagFileProcessorAgent -from airflow.exceptions import AirflowException, RemovedInAirflow3Warning +from airflow.decorators import task +from airflow.exceptions import AirflowException from airflow.executors.base_executor import BaseExecutor from airflow.executors.executor_constants import MOCK_EXECUTOR from airflow.executors.executor_loader import ExecutorLoader @@ -2828,128 +2829,33 @@ def test_do_not_schedule_removed_task(self, dag_maker): assert [] == res - @provide_session - def evaluate_dagrun( - self, - dag_id, - expected_task_states, # dict of task_id: state - dagrun_state, - run_kwargs=None, - advance_execution_date=False, - session=None, - ): - """ - Helper for testing DagRun states with simple two-task DAGs. - This is hackish: a dag run is created but its tasks are - run by a backfill. - """ - - # todo: AIP-78 remove along with DAG.run() - # this only tests the backfill job runner, not the scheduler - - if run_kwargs is None: - run_kwargs = {} - - dag = self.dagbag.get_dag(dag_id) - dagrun_info = dag.next_dagrun_info(None) - assert dagrun_info is not None - data_interval = dag.infer_automated_data_interval(DEFAULT_LOGICAL_DATE) - triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = dag.create_dagrun( - run_type=DagRunType.SCHEDULED, - execution_date=dagrun_info.logical_date, - state=None, - session=session, - data_interval=data_interval, - **triggered_by_kwargs, - ) - - if advance_execution_date: - # run a second time to schedule a dagrun after the start_date - dr = dag.create_dagrun( - run_type=DagRunType.SCHEDULED, - execution_date=dr.data_interval_end, - state=None, - session=session, - data_interval=data_interval, - **triggered_by_kwargs, - ) - ex_date = dr.execution_date - - for tid, state in expected_task_states.items(): - if state == State.FAILED: - self.null_exec.mock_task_fail(dag_id, tid, dr.run_id) - - try: - dag = DagBag().get_dag(dag.dag_id) - # This needs a _REAL_ dag, not the serialized version - assert not isinstance(dag, SerializedDAG) - # TODO: Can this be replaced with `self.run_scheduler_until_dagrun_terminal. `dag.run` isn't - # great to use here as it uses BackfillJobRunner! - for _ in _mock_executor(self.null_exec): - dag.run(start_date=ex_date, end_date=ex_date, **run_kwargs) - except AirflowException: - pass - - # load dagrun - dr = DagRun.find(dag_id=dag_id, execution_date=ex_date, session=session) - dr = dr[0] - dr.dag = dag - - assert dr.state == dagrun_state - - # test tasks - for task_id, expected_state in expected_task_states.items(): - ti = dr.get_task_instance(task_id) - assert ti.state == expected_state - - def test_dagrun_fail(self): + @pytest.mark.parametrize( + "ti_states, run_state", + [ + (["failed", "success"], "failed"), + (["success", "success"], "success"), + ], + ) + def test_dagrun_state_correct(self, ti_states, run_state, dag_maker, session): """ DagRuns with one failed and one incomplete root task -> FAILED """ - # todo: AIP-78 remove along with DAG.run() - # this only tests the backfill job runner, not the scheduler - with pytest.warns(RemovedInAirflow3Warning): - self.evaluate_dagrun( - dag_id="test_dagrun_states_fail", - expected_task_states={ - "test_dagrun_fail": State.FAILED, - "test_dagrun_succeed": State.UPSTREAM_FAILED, - }, - dagrun_state=State.FAILED, - ) + with dag_maker(): - def test_dagrun_success(self): - """ - DagRuns with one failed and one successful root task -> SUCCESS - """ - # todo: AIP-78 remove along with DAG.run() - # this only tests the backfill job runner, not the scheduler - with pytest.warns(RemovedInAirflow3Warning): - self.evaluate_dagrun( - dag_id="test_dagrun_states_success", - expected_task_states={ - "test_dagrun_fail": State.FAILED, - "test_dagrun_succeed": State.SUCCESS, - }, - dagrun_state=State.SUCCESS, - ) + @task + def my_task(): ... - def test_dagrun_root_fail(self): - """ - DagRuns with one successful and one failed root task -> FAILED - """ - # todo: AIP-78 remove along with DAG.run() - # this only tests the backfill job runner, not the scheduler - with pytest.warns(RemovedInAirflow3Warning): - self.evaluate_dagrun( - dag_id="test_dagrun_states_root_fail", - expected_task_states={ - "test_dagrun_succeed": State.SUCCESS, - "test_dagrun_fail": State.FAILED, - }, - dagrun_state=State.FAILED, - ) + for _ in ti_states: + my_task() + dr = dag_maker.create_dagrun(state="running", triggered_by=DagRunTriggeredByType.TIMETABLE) + for idx, state in enumerate(ti_states): + dr.task_instances[idx].state = state + session.commit() + scheduler_job = Job(executor=self.null_exec) + self.job_runner = SchedulerJobRunner(job=scheduler_job) + self.job_runner.processor_agent = mock.MagicMock() + self.job_runner._do_scheduling(session) + assert session.query(DagRun).one().state == run_state def test_dagrun_root_after_dagrun_unfinished(self, mock_executor): """ @@ -2963,57 +2869,15 @@ def test_dagrun_root_after_dagrun_unfinished(self, mock_executor): dag.sync_to_db() scheduler_job = Job() - self.job_runner = SchedulerJobRunner(job=scheduler_job, num_runs=1, subdir=dag.fileloc) + self.job_runner = SchedulerJobRunner(job=scheduler_job, num_runs=2, subdir=dag.fileloc) run_job(scheduler_job, execute_callable=self.job_runner._execute) - first_run = DagRun.find(dag_id=dag_id, execution_date=DEFAULT_DATE)[0] + first_run = DagRun.find(dag_id=dag_id)[0] ti_ids = [(ti.task_id, ti.state) for ti in first_run.get_task_instances()] assert ti_ids == [("current", State.SUCCESS)] assert first_run.state in [State.SUCCESS, State.RUNNING] - def test_dagrun_deadlock_ignore_depends_on_past_advance_ex_date(self): - """ - DagRun is marked a success if ignore_first_depends_on_past=True - - Test that an otherwise-deadlocked dagrun is marked as a success - if ignore_first_depends_on_past=True and the dagrun execution_date - is after the start_date. - """ - # todo: AIP-78 remove along with DAG.run() - # this only tests the backfill job runner, not the scheduler - with pytest.warns(RemovedInAirflow3Warning): - self.evaluate_dagrun( - dag_id="test_dagrun_states_deadlock", - expected_task_states={ - "test_depends_on_past": State.SUCCESS, - "test_depends_on_past_2": State.SUCCESS, - }, - dagrun_state=State.SUCCESS, - advance_execution_date=True, - run_kwargs=dict(ignore_first_depends_on_past=True), - ) - - def test_dagrun_deadlock_ignore_depends_on_past(self): - """ - Test that ignore_first_depends_on_past doesn't affect results - (this is the same test as - test_dagrun_deadlock_ignore_depends_on_past_advance_ex_date except - that start_date == execution_date so depends_on_past is irrelevant). - """ - # todo: AIP-78 remove along with DAG.run() - # this only tests the backfill job runner, not the scheduler - with pytest.warns(RemovedInAirflow3Warning): - self.evaluate_dagrun( - dag_id="test_dagrun_states_deadlock", - expected_task_states={ - "test_depends_on_past": State.SUCCESS, - "test_depends_on_past_2": State.SUCCESS, - }, - dagrun_state=State.SUCCESS, - run_kwargs=dict(ignore_first_depends_on_past=True), - ) - @pytest.mark.parametrize( "configs", [ @@ -3136,9 +3000,14 @@ def test_scheduler_multiprocessing(self, configs): Test that the scheduler can successfully queue multiple dags in parallel """ with conf_vars(configs): - dag_ids = ["test_start_date_scheduling", "test_dagrun_states_success"] + dag_ids = [ + "test_start_date_scheduling", + "test_task_start_date_scheduling", + ] for dag_id in dag_ids: dag = self.dagbag.get_dag(dag_id) + if not dag: + raise ValueError(f"could not find dag {dag_id}") dag.clear() scheduler_job = Job( diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 997ef06329fde..67dc699fc3c8b 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -25,7 +25,6 @@ import re import weakref from datetime import timedelta -from importlib import reload from pathlib import Path from typing import TYPE_CHECKING from unittest import mock @@ -45,12 +44,8 @@ AirflowException, DuplicateTaskIdFound, ParamValidationError, - RemovedInAirflow3Warning, UnknownExecutorException, ) -from airflow.executors import executor_loader -from airflow.executors.local_executor import LocalExecutor -from airflow.executors.sequential_executor import SequentialExecutor from airflow.models.asset import ( AssetAliasModel, AssetDagRunQueue, @@ -2740,20 +2735,6 @@ def test_asset_expression(self, session: Session) -> None: ] } - @mock.patch("airflow.models.dag.run_job") - def test_dag_executors(self, run_job_mock): - # todo: AIP-78 remove along with DAG.run() - # this only tests the backfill job runner, not the scheduler - with pytest.warns(RemovedInAirflow3Warning): - dag = DAG(dag_id="test", schedule=None) - reload(executor_loader) - with conf_vars({("core", "executor"): "SequentialExecutor"}): - dag.run() - assert isinstance(run_job_mock.call_args_list[0].kwargs["job"].executor, SequentialExecutor) - - dag.run(local=True) - assert isinstance(run_job_mock.call_args_list[1].kwargs["job"].executor, LocalExecutor) - class TestQueries: def setup_method(self) -> None: diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index b161020d1fb91..fbdd500661d22 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -23,7 +23,6 @@ from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.types import NOTSET -from dev.tests_common.test_utils.config import conf_vars from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test @@ -146,7 +145,6 @@ def test_xcom_not_iterable(self, dag_maker): @pytest.mark.system("core") class TestXComArgRuntime: - @conf_vars({("core", "executor"): "DebugExecutor"}) def test_xcom_pass_to_op(self, dag_maker): with dag_maker(dag_id="test_xcom_pass_to_op") as dag: operator = PythonOperator( @@ -161,9 +159,8 @@ def test_xcom_pass_to_op(self, dag_maker): task_id="assert_is_value_1", ) operator >> operator2 - dag.run() + dag.test() - @conf_vars({("core", "executor"): "DebugExecutor"}) def test_xcom_push_and_pass(self, dag_maker): def push_xcom_value(key, value, **context): ti = context["task_instance"] @@ -182,7 +179,7 @@ def push_xcom_value(key, value, **context): op_args=[xarg], ) op1 >> op2 - dag.run() + dag.test() @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode From 3bcc005cab3222cd85a942b4c5b666421f50d63c Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:40:10 -0700 Subject: [PATCH 059/199] Add test for behavior for paused backfill (#42837) ----- Co-authored-by: Kaxil Naik --- tests/jobs/test_scheduler_job.py | 90 +++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 639a3528fad8c..78c3acdce0de6 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -53,7 +53,7 @@ from airflow.jobs.local_task_job_runner import LocalTaskJobRunner from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.models.asset import AssetDagRunQueue, AssetEvent, AssetModel -from airflow.models.backfill import _create_backfill +from airflow.models.backfill import Backfill, _create_backfill from airflow.models.dag import DAG, DagModel from airflow.models.dagbag import DagBag from airflow.models.dagrun import DagRun @@ -4960,6 +4960,94 @@ def _running_counts(): assert session.scalar(select(func.count()).select_from(DagRun)) == 46 assert session.scalar(select(func.count()).where(DagRun.dag_id == dag1_dag_id)) == 36 + @pytest.mark.parametrize( + "pause_it, expected_running", + [ + (True, 0), + (False, 3), + ], + ) + def test_backfill_runs_not_started_when_backfill_paused( + self, pause_it, expected_running, dag_maker, session + ): + """ + When backfill is paused, will not start. + """ + dag1_dag_id = "test_dag1" + with dag_maker( + dag_id=dag1_dag_id, + start_date=DEFAULT_DATE, + schedule=timedelta(days=1), + max_active_runs=1, + ): + EmptyOperator(task_id="mytask") + + def _running_counts(): + dag1_non_b_running = ( + session.query(func.count(DagRun.id)) + .filter( + DagRun.dag_id == dag1_dag_id, + DagRun.state == State.RUNNING, + DagRun.run_type != DagRunType.BACKFILL_JOB, + ) + .scalar() + ) + dag1_b_running = ( + session.query(func.count(DagRun.id)) + .filter( + DagRun.dag_id == dag1_dag_id, + DagRun.state == State.RUNNING, + DagRun.run_type == DagRunType.BACKFILL_JOB, + ) + .scalar() + ) + total_running_count = ( + session.query(func.count(DagRun.id)).filter(DagRun.state == State.RUNNING).scalar() + ) + return dag1_non_b_running, dag1_b_running, total_running_count + + scheduler_job = Job() + self.job_runner = SchedulerJobRunner(job=scheduler_job, subdir=os.devnull) + scheduler_job.executor = MockExecutor(do_update=False) + self.job_runner.processor_agent = mock.MagicMock(spec=DagFileProcessorAgent) + + from_date = pendulum.parse("2021-01-01") + to_date = pendulum.parse("2021-01-06") + b = _create_backfill( + dag_id=dag1_dag_id, + from_date=from_date, + to_date=to_date, + max_active_runs=3, + reverse=False, + dag_run_conf={}, + ) + dag1_non_b_running, dag1_b_running, total_running = _running_counts() + + # initial state -- nothing is running + assert dag1_non_b_running == 0 + assert dag1_b_running == 0 + assert total_running == 0 + assert session.query(func.count(DagRun.id)).scalar() == 6 + assert session.scalar(select(func.count()).where(DagRun.dag_id == dag1_dag_id)) == 6 + + if pause_it: + b = session.get(Backfill, b.id) + b.is_paused = True + + session.commit() + + # now let's run scheduler once + self.job_runner._start_queued_dagruns(session) + session.flush() + + assert DagRun.DEFAULT_DAGRUNS_TO_EXAMINE == 20 + dag1_non_b_running, dag1_b_running, total_running = _running_counts() + assert dag1_non_b_running == 0 + assert dag1_b_running == expected_running + assert total_running == expected_running + assert session.scalar(select(func.count()).select_from(DagRun)) == 6 + assert session.scalar(select(func.count()).where(DagRun.dag_id == dag1_dag_id)) == 6 + def test_start_queued_dagruns_do_follow_execution_date_order(self, dag_maker): session = settings.Session() with dag_maker("test_dag1", max_active_runs=1): From bc5b57e2efd547b422deecbf5072b2a376b0db22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20=C5=A0im=C3=A1nek?= Date: Fri, 11 Oct 2024 00:21:50 +0200 Subject: [PATCH 060/199] Use url_from_endpoint inside HttpHook. (#42785) --- providers/src/airflow/providers/http/hooks/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index a53dd7566cefc..05b432626b8e3 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -169,7 +169,7 @@ def run( session = self.get_conn(headers) - url = _url_from_endpoint(self.base_url, endpoint) + url = self.url_from_endpoint(endpoint) if self.tcp_keep_alive: keep_alive_adapter = TCPKeepAliveAdapter( From bf39566541ba8c54c724b821f9d6302ea1891adb Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 10 Oct 2024 23:50:36 +0100 Subject: [PATCH 061/199] Fix typo in Breeze (#42919) A couple of minor typos due to newlines --- dev/breeze/doc/images/output_testing.svg | 4 ++-- dev/breeze/doc/images/output_testing.txt | 2 +- dev/breeze/doc/images/output_testing_non-db-tests.svg | 2 +- dev/breeze/doc/images/output_testing_non-db-tests.txt | 2 +- dev/breeze/doc/images/output_testing_tests.svg | 2 +- dev/breeze/doc/images/output_testing_tests.txt | 2 +- dev/breeze/src/airflow_breeze/commands/testing_commands.py | 6 +++--- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/breeze/doc/images/output_testing.svg b/dev/breeze/doc/images/output_testing.svg index 228a97056c95a..c85e5a60979a7 100644 --- a/dev/breeze/doc/images/output_testing.svg +++ b/dev/breeze/doc/images/output_testing.svg @@ -133,7 +133,7 @@ ╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │tests               Run the specified unit tests. This is a low level testing command that allows you to run      │ │various kind of tests subset with a number of options. You can also use dedicated commands    │ -│suchus db_tests, non_db_tests, integration_tests for more opinionated test suite execution.   │ +│such as db_tests, non_db_tests, integration_tests for more opinionated test suite execution.  │ │integration-tests   Run the specified integration tests.                                                          │ │helm-tests          Run Helm chart tests.                                                                         │ │docker-compose-testsRun docker-compose tests.                                                                     │ @@ -142,7 +142,7 @@ │db-tests    Run all (default) or specified DB-bound unit tests. This is a dedicated command that only runs DB     │ │tests and it runs them in parallel via splitting tests by test types into separate containers with    │ │separate database started for each container.                                                         │ -│non-db-testsRun all (default) or specified Non-DB unit tests. This is a dedicated command that onlyruns Non-DB    │ +│non-db-testsRun all (default) or specified Non-DB unit tests. This is a dedicated command that only runs Non-DB   │ │tests and it runs them in parallel via pytest-xdist in single container, with `none` backend set.     │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_testing.txt b/dev/breeze/doc/images/output_testing.txt index 3f7450936bae4..5f088aae9e13e 100644 --- a/dev/breeze/doc/images/output_testing.txt +++ b/dev/breeze/doc/images/output_testing.txt @@ -1 +1 @@ -d81f2195fbceabe4fc7d17bc3b83c5b1 +1447b50a7f4c843b01e92358a6b2e072 diff --git a/dev/breeze/doc/images/output_testing_non-db-tests.svg b/dev/breeze/doc/images/output_testing_non-db-tests.svg index 28fc44c028300..da43621dd6931 100644 --- a/dev/breeze/doc/images/output_testing_non-db-tests.svg +++ b/dev/breeze/doc/images/output_testing_non-db-tests.svg @@ -369,7 +369,7 @@ Usage:breeze testing non-db-tests[OPTIONS] -Run all (default) or specified Non-DB unit tests. This is a dedicated command that onlyruns Non-DB tests and it runs  +Run all (default) or specified Non-DB unit tests. This is a dedicated command that only runs Non-DB tests and it runs  them in parallel via pytest-xdist in single container, with `none` backend set. ╭─ Select test types to run ───────────────────────────────────────────────────────────────────────────────────────────╮ diff --git a/dev/breeze/doc/images/output_testing_non-db-tests.txt b/dev/breeze/doc/images/output_testing_non-db-tests.txt index 8be059e551195..1080b5a377717 100644 --- a/dev/breeze/doc/images/output_testing_non-db-tests.txt +++ b/dev/breeze/doc/images/output_testing_non-db-tests.txt @@ -1 +1 @@ -3120758ec7ae00c88305cca093595aca +2cc222da8b9f31b93b527220b76b48a2 diff --git a/dev/breeze/doc/images/output_testing_tests.svg b/dev/breeze/doc/images/output_testing_tests.svg index a5da2b4d53ff5..c20e2ef16b243 100644 --- a/dev/breeze/doc/images/output_testing_tests.svg +++ b/dev/breeze/doc/images/output_testing_tests.svg @@ -457,7 +457,7 @@ Usage:breeze testing tests[OPTIONS] [EXTRA_PYTEST_ARGS]... Run the specified unit tests. This is a low level testing command that allows you to run various kind of tests subset  -with a number of options. You can also use dedicated commands suchus db_tests, non_db_tests, integration_tests for  +with a number of options. You can also use dedicated commands such as db_tests, non_db_tests, integration_tests for  more opinionated test suite execution. ╭─ Select test types to run (tests can also be selected by command args individually) ─────────────────────────────────╮ diff --git a/dev/breeze/doc/images/output_testing_tests.txt b/dev/breeze/doc/images/output_testing_tests.txt index 7e33718d66c69..d19f6cf5abdae 100644 --- a/dev/breeze/doc/images/output_testing_tests.txt +++ b/dev/breeze/doc/images/output_testing_tests.txt @@ -1 +1 @@ -2be847f8a1f4e2b80c6da1e4433203aa +15002aa129ce25039921f800fb1cf744 diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 61b260bea09ae..c3bb4e9193204 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -499,8 +499,8 @@ def _verify_parallelism_parameters( @group_for_testing.command( name="tests", help="Run the specified unit tests. This is a low level testing command that allows you to run " - "various kind of tests subset with a number of options. You can also use dedicated commands such" - "us db_tests, non_db_tests, integration_tests for more opinionated test suite execution.", + "various kind of tests subset with a number of options. You can also use dedicated commands such " + "as db_tests, non_db_tests, integration_tests for more opinionated test suite execution.", context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, @@ -624,7 +624,7 @@ def command_for_db_tests(**kwargs): @group_for_testing.command( name="non-db-tests", - help="Run all (default) or specified Non-DB unit tests. This is a dedicated command that only" + help="Run all (default) or specified Non-DB unit tests. This is a dedicated command that only " "runs Non-DB tests and it runs them in parallel via pytest-xdist in single container, " "with `none` backend set.", context_settings=dict( From 4a85104d80bca9becf2bd7311aa432e46d47b952 Mon Sep 17 00:00:00 2001 From: Kim Date: Fri, 11 Oct 2024 00:26:51 -0600 Subject: [PATCH 062/199] Docs: Add templating info to TaskFlow tutorial (#42887) --- .../sql/tutorial_taskflow_template.sql | 23 ++++ .../tutorial_taskflow_templates.py | 107 ++++++++++++++++++ docs/apache-airflow/tutorial/taskflow.rst | 56 +++++++++ 3 files changed, 186 insertions(+) create mode 100644 airflow/example_dags/sql/tutorial_taskflow_template.sql create mode 100644 airflow/example_dags/tutorial_taskflow_templates.py diff --git a/airflow/example_dags/sql/tutorial_taskflow_template.sql b/airflow/example_dags/sql/tutorial_taskflow_template.sql new file mode 100644 index 0000000000000..375c39eac610b --- /dev/null +++ b/airflow/example_dags/sql/tutorial_taskflow_template.sql @@ -0,0 +1,23 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +select * from test_data +where 1=1 + and run_id = '{{ run_id }}' + and something_else = '{{ params.foobar }}' diff --git a/airflow/example_dags/tutorial_taskflow_templates.py b/airflow/example_dags/tutorial_taskflow_templates.py new file mode 100644 index 0000000000000..925f60524b5ea --- /dev/null +++ b/airflow/example_dags/tutorial_taskflow_templates.py @@ -0,0 +1,107 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +# [START tutorial] +# [START import_module] +import pendulum + +from airflow.decorators import dag, task +from airflow.operators.python import get_current_context + +# [END import_module] + + +# [START instantiate_dag] +@dag( + schedule="@daily", + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + catchup=False, + tags=["example"], + params={"foobar": "param_from_dag", "other_param": "from_dag"}, +) +def tutorial_taskflow_templates(): + """ + ### TaskFlow API Tutorial Documentation + This is a simple data pipeline example which demonstrates the use of + the templates in the TaskFlow API. + Documentation that goes along with the Airflow TaskFlow API tutorial is + located + [here](https://airflow.apache.org/docs/apache-airflow/stable/tutorial_taskflow_api.html) + """ + # [END instantiate_dag] + + # [START template_test] + @task( + # Causes variables that end with `.sql` to be read and templates + # within to be rendered. + templates_exts=[".sql"], + ) + def template_test(sql, test_var, data_interval_end): + context = get_current_context() + + # Will print... + # select * from test_data + # where 1=1 + # and run_id = 'scheduled__2024-10-09T00:00:00+00:00' + # and something_else = 'param_from_task' + print(f"sql: {sql}") + + # Will print `scheduled__2024-10-09T00:00:00+00:00` + print(f"test_var: {test_var}") + + # Will print `2024-10-10 00:00:00+00:00`. + # Note how we didn't pass this value when calling the task. Instead + # it was passed by the decorator from the context + print(f"data_interval_end: {data_interval_end}") + + # Will print... + # run_id: scheduled__2024-10-09T00:00:00+00:00; params.other_param: from_dag + template_str = "run_id: {{ run_id }}; params.other_param: {{ params.other_param }}" + rendered_template = context["task"].render_template( + template_str, + context, + ) + print(f"rendered template: {rendered_template}") + + # Will print the full context dict + print(f"context: {context}") + + # [END template_test] + + # [START main_flow] + template_test.override( + # Will be merged with the dict defined in the dag + # and override existing parameters. + # + # Must be passed into the decorator's parameters + # through `.override()` not into the actual task + # function + params={"foobar": "param_from_task"}, + )( + sql="sql/test.sql", + test_var="{{ run_id }}", + ) + # [END main_flow] + + +# [START dag_invocation] +tutorial_taskflow_templates() +# [END dag_invocation] + +# [END tutorial] diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index e15e1c78045c2..37d0d93f59994 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -629,6 +629,62 @@ method. Current context is accessible only during the task execution. The context is not accessible during ``pre_execute`` or ``post_execute``. Calling this method outside execution context will raise an error. +Using templates in decorated tasks +---------------------------------------------- + +Arguments passed to your decorated function are automatically templated. + +You can also use the ``templates_exts`` parameter to template entire files. + +.. code-block:: python + + @task(templates_exts=[".sql"]) + def template_test(sql): + print(f"sql: {sql}") + + + template_test(sql="sql/test.sql") + +This will read the content of ``sql/test.sql`` and replace all template variables. You can also pass a list of files and all of them will be templated. + +You can pass additional parameters to the template engine through `the params parameter `_. + +However, the ``params`` parameter must be passed to the decorator and not to your function directly, such as ``@task(templates_exts=['.sql'], params={'my_param'})`` and can then be used with ``{{ params.my_param }}`` in your templated files and function parameters. + +Alternatively, you can also pass it using the ``.override()`` method: + +.. code-block:: python + + @task() + def template_test(input_var): + print(f"input_var: {input_var}") + + + template_test.override(params={"my_param": "wow"})( + input_var="my param is: {{ params.my_param }}", + ) + +Finally, you can also manually render templates: + +.. code-block:: python + + @task(params={"my_param": "wow"}) + def template_test(): + template_str = "run_id: {{ run_id }}; params.my_param: {{ params.my_param }}" + + context = get_current_context() + rendered_template = context["task"].render_template( + template_str, + context, + ) + +Here is a full example that demonstrates everything above: + +.. exampleinclude:: /../../airflow/example_dags/tutorial_taskflow_templates.py + :language: python + :start-after: [START tutorial] + :end-before: [END tutorial] + Conditionally skipping tasks ---------------------------- From 24a35fa5ba61f151e4e307196b4eaac23db2ea46 Mon Sep 17 00:00:00 2001 From: Rasnar Date: Fri, 11 Oct 2024 08:37:14 +0200 Subject: [PATCH 063/199] Add possibility to override the conn type for Druid (#42793) * Add possibility to override the conn type for Druid Minor fix, which allows to use the schema which are specified in theschema rather than `http` as default. In the same time it doesn't changethe logic as any conn_type can be selected. Intuitevely it's expectedthat anything specified in `schema` field will actually take precedencein the building the desired url. * Add druid endpoint connection from another PR * Fix missing scheme in test * Set schema to None where it's unused Even though we don't need it directly set, by default the mock will set it to an internal object, thus we need to override it to None. --------- Co-authored-by: Oleg Auckenthaler --- .../providers/apache/druid/hooks/druid.py | 5 ++- .../tests/apache/druid/hooks/test_druid.py | 44 ++++++++++++++++--- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/providers/src/airflow/providers/apache/druid/hooks/druid.py b/providers/src/airflow/providers/apache/druid/hooks/druid.py index ca315b3a2c751..c865adef412fb 100644 --- a/providers/src/airflow/providers/apache/druid/hooks/druid.py +++ b/providers/src/airflow/providers/apache/druid/hooks/druid.py @@ -86,7 +86,10 @@ def get_conn_url(self, ingestion_type: IngestionType = IngestionType.BATCH) -> s """Get Druid connection url.""" host = self.conn.host port = self.conn.port - conn_type = self.conn.conn_type or "http" + if self.conn.schema: + conn_type = self.conn.schema + else: + conn_type = self.conn.conn_type or "http" if ingestion_type == IngestionType.BATCH: endpoint = self.conn.extra_dejson.get("endpoint", "") else: diff --git a/providers/tests/apache/druid/hooks/test_druid.py b/providers/tests/apache/druid/hooks/test_druid.py index 9befbf37f02a6..f01175942be4a 100644 --- a/providers/tests/apache/druid/hooks/test_druid.py +++ b/providers/tests/apache/druid/hooks/test_druid.py @@ -42,9 +42,14 @@ class TestDRuidhook(DruidHook): self.is_sql_based_ingestion = False def get_conn_url(self, ingestion_type: IngestionType = IngestionType.BATCH): + if self.conn.schema: + conn_type = self.conn.schema + else: + conn_type = "http" + if ingestion_type == IngestionType.MSQ: - return "http://druid-overlord:8081/druid/v2/sql/task" - return "http://druid-overlord:8081/druid/indexer/v1/task" + return f"{conn_type}://druid-overlord:8081/druid/v2/sql/task" + return f"{conn_type}://druid-overlord:8081/druid/indexer/v1/task" self.db_hook = TestDRuidhook() @@ -257,7 +262,8 @@ def get_conn_url(self, ingestion_type: IngestionType = IngestionType.BATCH): def test_conn_property(self, mock_get_connection): get_conn_value = MagicMock() get_conn_value.host = "test_host" - get_conn_value.conn_type = "https" + get_conn_value.conn_type = "http" + get_conn_value.schema = None get_conn_value.port = "1" get_conn_value.extra_dejson = {"endpoint": "ingest"} mock_get_connection.return_value = get_conn_value @@ -268,8 +274,22 @@ def test_conn_property(self, mock_get_connection): def test_get_conn_url(self, mock_get_connection): get_conn_value = MagicMock() get_conn_value.host = "test_host" - get_conn_value.conn_type = "https" + get_conn_value.conn_type = "http" + get_conn_value.schema = None + get_conn_value.port = "1" + get_conn_value.extra_dejson = {"endpoint": "ingest"} + mock_get_connection.return_value = get_conn_value + hook = DruidHook(timeout=1, max_ingestion_time=5) + assert hook.get_conn_url() == "http://test_host:1/ingest" + + @patch("airflow.providers.apache.druid.hooks.druid.DruidHook.get_connection") + def test_get_conn_url_with_schema(self, mock_get_connection): + get_conn_value = MagicMock() + get_conn_value.host = "test_host" + get_conn_value.conn_type = "http" + get_conn_value.schema = None get_conn_value.port = "1" + get_conn_value.schema = "https" get_conn_value.extra_dejson = {"endpoint": "ingest"} mock_get_connection.return_value = get_conn_value hook = DruidHook(timeout=1, max_ingestion_time=5) @@ -279,8 +299,21 @@ def test_get_conn_url(self, mock_get_connection): def test_get_conn_url_with_ingestion_type(self, mock_get_connection): get_conn_value = MagicMock() get_conn_value.host = "test_host" - get_conn_value.conn_type = "https" + get_conn_value.conn_type = "http" + get_conn_value.schema = None + get_conn_value.port = "1" + get_conn_value.extra_dejson = {"endpoint": "ingest", "msq_endpoint": "sql_ingest"} + mock_get_connection.return_value = get_conn_value + hook = DruidHook(timeout=1, max_ingestion_time=5) + assert hook.get_conn_url(IngestionType.MSQ) == "http://test_host:1/sql_ingest" + + @patch("airflow.providers.apache.druid.hooks.druid.DruidHook.get_connection") + def test_get_conn_url_with_ingestion_type_and_schema(self, mock_get_connection): + get_conn_value = MagicMock() + get_conn_value.host = "test_host" + get_conn_value.conn_type = "http" get_conn_value.port = "1" + get_conn_value.schema = "https" get_conn_value.extra_dejson = {"endpoint": "ingest", "msq_endpoint": "sql_ingest"} mock_get_connection.return_value = get_conn_value hook = DruidHook(timeout=1, max_ingestion_time=5) @@ -343,6 +376,7 @@ def setup_method(self): self.conn = conn = MagicMock() self.conn.host = "host" self.conn.port = "1000" + self.conn.schema = None self.conn.conn_type = "druid" self.conn.extra_dejson = {"endpoint": "druid/v2/sql"} self.conn.cursor.return_value = self.cur From baadbbf0cffdb01514919044af1853926235982b Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Fri, 11 Oct 2024 12:27:09 +0530 Subject: [PATCH 064/199] fix: HttpSensorTrigger to include `method` when serializing (#42925) --- .../airflow/providers/http/triggers/http.py | 1 + providers/tests/http/triggers/test_http.py | 33 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/http/triggers/http.py b/providers/src/airflow/providers/http/triggers/http.py index 59a484b081e04..543cd323b80a3 100644 --- a/providers/src/airflow/providers/http/triggers/http.py +++ b/providers/src/airflow/providers/http/triggers/http.py @@ -167,6 +167,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: { "endpoint": self.endpoint, "data": self.data, + "method": self.method, "headers": self.headers, "extra_options": self.extra_options, "http_conn_id": self.http_conn_id, diff --git a/providers/tests/http/triggers/test_http.py b/providers/tests/http/triggers/test_http.py index a4f2559876fae..8af78149b24a7 100644 --- a/providers/tests/http/triggers/test_http.py +++ b/providers/tests/http/triggers/test_http.py @@ -30,7 +30,7 @@ from requests.structures import CaseInsensitiveDict from yarl import URL -from airflow.providers.http.triggers.http import HttpTrigger +from airflow.providers.http.triggers.http import HttpSensorTrigger, HttpTrigger from airflow.triggers.base import TriggerEvent HTTP_PATH = "airflow.providers.http.triggers.http.{}" @@ -56,6 +56,18 @@ def trigger(): ) +@pytest.fixture +def sensor_trigger(): + return HttpSensorTrigger( + http_conn_id=TEST_CONN_ID, + endpoint=TEST_ENDPOINT, + method=TEST_METHOD, + headers=TEST_HEADERS, + data=TEST_DATA, + extra_options=TEST_EXTRA_OPTIONS, + ) + + @pytest.fixture def client_response(): client_response = mock.AsyncMock(ClientResponse) @@ -153,3 +165,22 @@ async def test_trigger_on_post_with_data(self, mock_http_post, trigger): assert kwargs["data"] == TEST_DATA assert kwargs["json"] is None assert kwargs["params"] is None + + +class TestHttpSensorTrigger: + def test_serialization(self, sensor_trigger): + """ + Asserts that the HttpSensorTrigger correctly serializes its arguments + and classpath. + """ + classpath, kwargs = sensor_trigger.serialize() + assert classpath == "airflow.providers.http.triggers.http.HttpSensorTrigger" + assert kwargs == { + "http_conn_id": TEST_CONN_ID, + "endpoint": TEST_ENDPOINT, + "method": TEST_METHOD, + "headers": TEST_HEADERS, + "data": TEST_DATA, + "extra_options": TEST_EXTRA_OPTIONS, + "poke_interval": 5.0, + } From 6aca278d74b92c24cbc33359249d9bfb2b18dfcc Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Fri, 11 Oct 2024 15:31:10 +0800 Subject: [PATCH 065/199] AIP-84 Fix dag display name search (#42863) * Fix dag display name search * Fix CI --- airflow/models/dag.py | 12 ++++++++++++ tests/api_fastapi/views/public/test_dags.py | 8 +++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 5cc5cf4431407..8d85152677e0b 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -3046,6 +3046,18 @@ def set_is_paused(self, is_paused: bool, session=NEW_SESSION) -> None: def dag_display_name(self) -> str: return self._dag_display_property_value or self.dag_id + @dag_display_name.expression # type: ignore[no-redef] + def dag_display_name(self) -> str: + """ + Expression part of the ``dag_display`` name hybrid property. + + :meta private: + """ + return case( + (self._dag_display_property_value.isnot(None), self._dag_display_property_value), + else_=self.dag_id, + ) + @classmethod @internal_api_call @provide_session diff --git a/tests/api_fastapi/views/public/test_dags.py b/tests/api_fastapi/views/public/test_dags.py index 5512f7bb13849..13520b37b3ff2 100644 --- a/tests/api_fastapi/views/public/test_dags.py +++ b/tests/api_fastapi/views/public/test_dags.py @@ -35,7 +35,6 @@ DAG1_ID = "test_dag1" DAG1_DISPLAY_NAME = "display1" DAG2_ID = "test_dag2" -DAG2_DISPLAY_NAME = "display2" DAG2_START_DATE = datetime(2021, 6, 15, tzinfo=timezone.utc) DAG3_ID = "test_dag3" TASK_ID = "op1" @@ -99,7 +98,6 @@ def setup(dag_maker, session=None) -> None: with dag_maker( DAG2_ID, - dag_display_name=DAG2_DISPLAY_NAME, schedule=None, start_date=DAG2_START_DATE, doc_md="details", @@ -150,7 +148,7 @@ def setup(dag_maker, session=None) -> None: ), # Search ({"dag_id_pattern": "1"}, 1, [DAG1_ID]), - ({"dag_display_name_pattern": "display2"}, 1, [DAG2_ID]), + ({"dag_display_name_pattern": "test_dag2"}, 1, [DAG2_ID]), ], ) def test_get_dags(test_client, query_params, expected_total_entries, expected_ids): @@ -239,7 +237,7 @@ def test_patch_dags(test_client, query_params, body, expected_status_code, expec "query_params, dag_id, expected_status_code, dag_display_name, start_date", [ ({}, "fake_dag_id", 404, "fake_dag", datetime(2023, 12, 31, tzinfo=timezone.utc)), - ({}, DAG2_ID, 200, DAG2_DISPLAY_NAME, DAG2_START_DATE), + ({}, DAG2_ID, 200, DAG2_ID, DAG2_START_DATE), ], ) def test_dag_details(test_client, query_params, dag_id, expected_status_code, dag_display_name, start_date): @@ -309,7 +307,7 @@ def test_dag_details(test_client, query_params, dag_id, expected_status_code, da "query_params, dag_id, expected_status_code, dag_display_name", [ ({}, "fake_dag_id", 404, "fake_dag"), - ({}, DAG2_ID, 200, DAG2_DISPLAY_NAME), + ({}, DAG2_ID, 200, DAG2_ID), ], ) def test_get_dag(test_client, query_params, dag_id, expected_status_code, dag_display_name): From 0ef7419bf9f89682078c61ab7ada165d0d800040 Mon Sep 17 00:00:00 2001 From: GPK Date: Fri, 11 Oct 2024 09:19:37 +0100 Subject: [PATCH 066/199] Check _is_canary_run/pr condition in is_legacy_ui_api_labeled method (#42921) * check _is_canary_run condition in is_legacy_ui_api_labeled method * include pr check in is_legacy_ui_api_labeled --- dev/breeze/src/airflow_breeze/utils/selective_checks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index dfc24993f274a..18458af5f8a23 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1370,6 +1370,12 @@ def _is_canary_run(self): def is_legacy_ui_api_labeled(self) -> bool: # Selective check for legacy UI/API updates. # It is to ping the maintainer to add the label and make them aware of the changes. + if self._is_canary_run() or self._github_event not in ( + GithubEvents.PULL_REQUEST, + GithubEvents.PULL_REQUEST_TARGET, + ): + return False + if ( self._matching_files( FileGroupForCi.LEGACY_API_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES From 34fba8b57753026e64afd5738f1ac15723167701 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Fri, 11 Oct 2024 11:11:27 +0100 Subject: [PATCH 067/199] Allow python 3.12 for the breeze release commands (#42936) I'm not 100% sure why we added this check in the first place, but it doesn't seem to be needed anymore (I've tested things locally with this removed and it all seems to behave itself) --- dev/breeze/src/airflow_breeze/utils/python_versions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/python_versions.py b/dev/breeze/src/airflow_breeze/utils/python_versions.py index 3571bebb245dc..c7d514f5ff967 100644 --- a/dev/breeze/src/airflow_breeze/utils/python_versions.py +++ b/dev/breeze/src/airflow_breeze/utils/python_versions.py @@ -50,9 +50,6 @@ def check_python_version(): if not sys.version_info >= (3, 9): get_console().print("[error]At least Python 3.9 is required to prepare reproducible archives.\n") error = True - elif not sys.version_info < (3, 12): - get_console().print("[error]Python 3.12 is not supported.\n") - error = True if error: get_console().print( "[warning]Please reinstall Breeze using Python 3.9 - 3.11 environment.[/]\n\n" From 43a8693f1a29c79b3ca7d849222eceee481161b1 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Fri, 11 Oct 2024 13:53:26 +0200 Subject: [PATCH 068/199] Render errors when getting a list of dags (#42897) * Render errors when getting a list of dags * Restore axios, prettierignore pnpm-store * Add pnpm-store to prettier ignore --- airflow/ui/.prettierignore | 1 + airflow/ui/pnpm-lock.yaml | 8 +-- .../ui/src/components/DataTable/DataTable.tsx | 7 +- airflow/ui/src/components/ErrorAlert.tsx | 67 +++++++++++++++++++ airflow/ui/src/main.tsx | 6 +- airflow/ui/src/pages/DagsList/DagsList.tsx | 4 +- 6 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 airflow/ui/src/components/ErrorAlert.tsx diff --git a/airflow/ui/.prettierignore b/airflow/ui/.prettierignore index 7e860ea047193..49a8631b874a0 100644 --- a/airflow/ui/.prettierignore +++ b/airflow/ui/.prettierignore @@ -4,3 +4,4 @@ dist/ *.md *.yaml coverage/* +.pnpm-store diff --git a/airflow/ui/pnpm-lock.yaml b/airflow/ui/pnpm-lock.yaml index 6298d2ba3256f..c9f94d35f7358 100644 --- a/airflow/ui/pnpm-lock.yaml +++ b/airflow/ui/pnpm-lock.yaml @@ -2049,8 +2049,8 @@ packages: resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==} engines: {node: '>=14'} - form-data@4.0.0: - resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + form-data@4.0.1: + resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==} engines: {node: '>= 6'} framer-motion@11.3.29: @@ -5012,7 +5012,7 @@ snapshots: axios@1.7.7: dependencies: follow-redirects: 1.15.9 - form-data: 4.0.0 + form-data: 4.0.1 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug @@ -5697,7 +5697,7 @@ snapshots: cross-spawn: 7.0.3 signal-exit: 4.1.0 - form-data@4.0.0: + form-data@4.0.1: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 diff --git a/airflow/ui/src/components/DataTable/DataTable.tsx b/airflow/ui/src/components/DataTable/DataTable.tsx index 7f4c3cf083966..2ed1a4f16ea3a 100644 --- a/airflow/ui/src/components/DataTable/DataTable.tsx +++ b/airflow/ui/src/components/DataTable/DataTable.tsx @@ -41,6 +41,7 @@ type DataTableProps = { readonly columns: Array>; readonly data: Array; readonly displayMode?: "card" | "table"; + readonly errorMessage?: ReactNode | string; readonly getRowCanExpand?: (row: Row) => boolean; readonly initialState?: TableState; readonly isFetching?: boolean; @@ -62,6 +63,7 @@ export const DataTable = ({ columns, data, displayMode = "table", + errorMessage, getRowCanExpand = defaultGetRowCanExpand, initialState, isFetching, @@ -127,10 +129,9 @@ export const DataTable = ({ Boolean(isFetching) && !Boolean(isLoading) ? "visible" : "hidden" } /> + {errorMessage} {!Boolean(isLoading) && !rows.length && ( - - {noRowsMessage ?? `No ${modelName}s found.`} - + {noRowsMessage ?? `No ${modelName}s found.`} )} {display === "table" && } {display === "card" && cardDef !== undefined && ( diff --git a/airflow/ui/src/components/ErrorAlert.tsx b/airflow/ui/src/components/ErrorAlert.tsx new file mode 100644 index 0000000000000..3128a2cdec2c4 --- /dev/null +++ b/airflow/ui/src/components/ErrorAlert.tsx @@ -0,0 +1,67 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Alert, AlertIcon } from "@chakra-ui/react"; +import type { ApiError } from "openapi-gen/requests/core/ApiError"; +import type { + HTTPExceptionResponse, + HTTPValidationError, +} from "openapi-gen/requests/types.gen"; + +type ExpandedApiError = { + body: HTTPExceptionResponse | HTTPValidationError; +} & ApiError; + +type Props = { + readonly error?: unknown; +}; + +export const ErrorAlert = ({ error: err }: Props) => { + const error = err as ExpandedApiError; + + if (!Boolean(error)) { + return undefined; + } + + const details = error.body.detail; + let detailMessage; + + if (details !== undefined) { + if (typeof details === "string") { + detailMessage = details; + } else if (Array.isArray(details)) { + detailMessage = details.map( + (detail) => ` + ${detail.loc.join(".")} ${detail.msg}`, + ); + } else { + detailMessage = Object.keys(details).map( + (key) => `${key}: ${details[key] as string}`, + ); + } + } + + return ( + + + {error.message} +
+ {detailMessage} + + ); +}; diff --git a/airflow/ui/src/main.tsx b/airflow/ui/src/main.tsx index daf4bcd024cd6..12434ca7bade2 100644 --- a/airflow/ui/src/main.tsx +++ b/airflow/ui/src/main.tsx @@ -18,7 +18,7 @@ */ import { ChakraProvider } from "@chakra-ui/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import axios, { type AxiosError, type AxiosResponse } from "axios"; +import axios, { type AxiosError } from "axios"; import { createRoot } from "react-dom/client"; import { BrowserRouter } from "react-router-dom"; @@ -45,7 +45,7 @@ const queryClient = new QueryClient({ // redirect to login page if the API responds with unauthorized or forbidden errors axios.interceptors.response.use( - (response: AxiosResponse) => response, + (response) => response, (error: AxiosError) => { if (error.response?.status === 403 || error.response?.status === 401) { const params = new URLSearchParams(); @@ -53,6 +53,8 @@ axios.interceptors.response.use( params.set("next", globalThis.location.href); globalThis.location.replace(`/login?${params.toString()}`); } + + return Promise.reject(error); }, ); diff --git a/airflow/ui/src/pages/DagsList/DagsList.tsx b/airflow/ui/src/pages/DagsList/DagsList.tsx index 178663baf7478..60d6ef9c4f437 100644 --- a/airflow/ui/src/pages/DagsList/DagsList.tsx +++ b/airflow/ui/src/pages/DagsList/DagsList.tsx @@ -34,6 +34,7 @@ import { DataTable } from "src/components/DataTable"; import { ToggleTableDisplay } from "src/components/DataTable/ToggleTableDisplay"; import type { CardDef } from "src/components/DataTable/types"; import { useTableURLState } from "src/components/DataTable/useTableUrlState"; +import { ErrorAlert } from "src/components/ErrorAlert"; import { SearchBar } from "src/components/SearchBar"; import { TogglePause } from "src/components/TogglePause"; import { pluralize } from "src/utils/pluralize"; @@ -113,7 +114,7 @@ export const DagsList = () => { const [sort] = sorting; const orderBy = sort ? `${sort.desc ? "-" : ""}${sort.id}` : undefined; - const { data, isFetching, isLoading } = useDagServiceGetDags({ + const { data, error, isFetching, isLoading } = useDagServiceGetDags({ lastDagRunState, limit: pagination.pageSize, offset: pagination.pageIndex * pagination.pageSize, @@ -169,6 +170,7 @@ export const DagsList = () => { columns={columns} data={data?.dags ?? []} displayMode={display} + errorMessage={} initialState={tableURLState} isFetching={isFetching} isLoading={isLoading} From a9c8a87db14e20bac8c26c2e3dc86c96c7a51fdb Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Fri, 11 Oct 2024 13:35:00 +0100 Subject: [PATCH 069/199] Remove the referrer from Webserver to Scarf (#42901) This will make sure we don't receive any information about the Webserver URL sending the info like the number of plugins and such. --- airflow/www/templates/airflow/dags.html | 2 +- docs/apache-airflow/faq.rst | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html index c629936df7c00..6da7090a14781 100644 --- a/airflow/www/templates/airflow/dags.html +++ b/airflow/www/templates/airflow/dags.html @@ -489,6 +489,6 @@

{{ page_title }}

} {% if scarf_url %} - + {% endif %} {% endblock %} diff --git a/docs/apache-airflow/faq.rst b/docs/apache-airflow/faq.rst index 5d4aea6ddd562..0b2c76765e704 100644 --- a/docs/apache-airflow/faq.rst +++ b/docs/apache-airflow/faq.rst @@ -548,4 +548,3 @@ The telemetry data collected is limited to the following: - Number of DAGs - Number of Airflow plugins - Number of timetables, Flask blueprints, Flask AppBuilder views, and Flask Appbuilder menu items from Airflow plugins -- and information from tracking pixels like URL / IP-address as documented in the `Scarf Privacy Policy `__. From a9ac157b5d6993d451eaa1c4432faa074b847666 Mon Sep 17 00:00:00 2001 From: vlieven Date: Fri, 11 Oct 2024 17:12:35 +0200 Subject: [PATCH 070/199] Split providers out of the main "airflow/" tree into a UV workspace project (#42505) (#42624) This is only a partial split so far. It moves all the code and tests, but leaves the creation of `core/` to a separate PR as this is already large enough. In addition to the straight file rename the other changes I had to make here are: - Some mypy/typing fixes. Mypy can be fragile about what it picks up when, so maybe some of those changes were caused by that. But the typing changes aren't large. - Improve typing in common.sql type stub Again, likely a mypy file oddity, but the types should be safe - Removed the `check-providers-init-file-missing` check This isn't needed now that airflow/providers shouldn't exist at all in the main tree. - Create a "dev.tests_common" package that contains helper files and common pytest fixtures Since the provider tests are no longer under tests/ they don't automatically share the fixtures from the parent `tests/conftest.py` so they needed extracted. Ditto for `tests.test_utils` -- they can't be easily imported in provider tests anymore, so they are moved to a more explicit shared location. In future we should switch how the CI image is built to make better use of UV caching than our own approach as that would remvoe a lot of custom code. Co-authored-by: Ash Berlin-Taylor Co-authored-by: Ryan Hatter <25823361+RNHTTR@users.noreply.github.com> --- .../executors/kubernetes_executor.py | 1 + .../executors/kubernetes_executor_utils.py | 10 -------- .../executors/test_kubernetes_executor.py | 23 ++++++++++++------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py index 7c6e0d8852e5f..4301a54f02922 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +++ b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py @@ -482,6 +482,7 @@ def _change_state( self.running.remove(key) except KeyError: self.log.debug("TI key not in running, not adding to event_buffer: %s", key) + return # If we don't have a TI state, look it up from the db. event_buffer expects the TI state if state is None: diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py index 495c09ea23e7c..42d2a97dffed1 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +++ b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py @@ -273,16 +273,6 @@ def process_status( (pod_name, namespace, TaskInstanceState.FAILED, annotations, resource_version) ) elif status == "Succeeded": - # We get multiple events once the pod hits a terminal state, and we only want to - # send it along to the scheduler once. - # If our event type is DELETED, or the pod has a deletion timestamp, we've already - # seen the initial Succeeded event and sent it along to the scheduler. - if event["type"] == "DELETED" or pod.metadata.deletion_timestamp: - self.log.info( - "Skipping event for Succeeded pod %s - event for this pod already sent to executor", - pod_name, - ) - return self.log.info("Event: %s Succeeded, annotations: %s", pod_name, annotations_string) self.watcher_queue.put((pod_name, namespace, None, annotations, resource_version)) elif status == "Running": diff --git a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py index 5240bf0faecb2..07b42ee3fc22a 100644 --- a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -784,6 +784,21 @@ def test_change_state_adopted(self, mock_delete_pod, mock_get_kube_client, mock_ finally: executor.end() + @pytest.mark.db_test + @mock.patch("airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.providers.cncf.kubernetes.kube_client.get_kube_client") + def test_change_state_key_not_in_running(self, mock_get_kube_client, mock_kubernetes_job_watcher): + executor = self.kubernetes_executor + executor.start() + try: + key = ("dag_id", "task_id", "run_id", "try_number1") + executor.running = set() + executor._change_state(key, State.SUCCESS, "pod_name", "default") + assert executor.event_buffer.get(key) is None + assert executor.running == set() + finally: + executor.end() + @pytest.mark.db_test @pytest.mark.parametrize( "multi_namespace_mode_namespace_list, watchers_keys", @@ -1858,14 +1873,6 @@ def test_process_status_succeeded(self): # We don't know the TI state, so we send in None self.assert_watcher_queue_called_once_with_state(None) - def test_process_status_succeeded_dedup_timestamp(self): - self.pod.status.phase = "Succeeded" - self.pod.metadata.deletion_timestamp = timezone.utcnow() - self.events.append({"type": "MODIFIED", "object": self.pod}) - - self._run() - self.watcher.watcher_queue.put.assert_not_called() - @pytest.mark.parametrize( "ti_state", [ From f4835f362401c21efc0573bc43a207ac4fd2c887 Mon Sep 17 00:00:00 2001 From: GPK Date: Fri, 11 Oct 2024 20:03:42 +0100 Subject: [PATCH 071/199] increase backoff_factor and add try/catch in k8s tests (#42940) --- kubernetes_tests/test_base.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py index ac311daa52f8c..6929128173cbf 100644 --- a/kubernetes_tests/test_base.py +++ b/kubernetes_tests/test_base.py @@ -29,6 +29,7 @@ import requests import requests.exceptions from requests.adapters import HTTPAdapter +from urllib3.exceptions import MaxRetryError from urllib3.util.retry import Retry CLUSTER_FORWARDED_PORT = os.environ.get("CLUSTER_FORWARDED_PORT") or "8080" @@ -125,7 +126,7 @@ def _get_session_with_retries(self): session.auth = ("admin", "admin") retries = Retry( total=3, - backoff_factor=1, + backoff_factor=10, status_forcelist=[404], allowed_methods=Retry.DEFAULT_ALLOWED_METHODS | frozenset(["PATCH", "POST"]), ) @@ -225,10 +226,16 @@ def start_dag(self, dag_id, host): print(f"Calling [start_dag]#1 {patch_string}") max_attempts = 10 result = {} + # This loop retries until the DAG parser finishes with max_attempts and the DAG is available for execution. + # Keep the try/catch block, as the session object has a default retry configuration. + # If a MaxRetryError is raised, it can be safely ignored, indicating that the DAG is not yet parsed. while max_attempts: - result = self.session.patch(patch_string, json={"is_paused": False}) - if result.status_code == 200: - break + try: + result = self.session.patch(patch_string, json={"is_paused": False}) + if result.status_code == 200: + break + except MaxRetryError: + pass time.sleep(30) max_attempts -= 1 From e2552c8d0d884a81fcfa6763a003ee5c8f6b836e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 11 Oct 2024 21:22:05 +0200 Subject: [PATCH 072/199] Consistent python version checks and troubleshooting (#42944) Follow up after #42766 and #42936. * We do not have to check for minimum Python version for Python 3.9 any more (as we do not support 3.8 any more) * While Python 3.12 is not yet fully supported by Apache Beam, we should still not allow it for releasing providers, but all other commands should support Python 3.12 * When you had pre-commit installed with Python 3.8 before, various errors might appear when running pre-commit. Troubleshooting was added quoting the errors and explaining what to do. --- dev/breeze/doc/04_troubleshooting.rst | 32 +++++++++++++++++++ .../commands/release_management_commands.py | 2 +- .../airflow_breeze/utils/python_versions.py | 12 +++---- .../providers/apache/beam/provider.yaml | 5 ++- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/dev/breeze/doc/04_troubleshooting.rst b/dev/breeze/doc/04_troubleshooting.rst index fd0b1dfa401dc..fd5f92b03ea35 100644 --- a/dev/breeze/doc/04_troubleshooting.rst +++ b/dev/breeze/doc/04_troubleshooting.rst @@ -72,6 +72,38 @@ describe your problem. stated in `This comment `_ and allows to run Breeze with no problems. +Cannot import name 'cache' or Python >=3.9 required +--------------------------------------------------- + +When you see this error: + +.. code-block:: + + ImportError: cannot import name 'cache' from 'functools' (/Users/jarek/Library/Application Support/hatch/pythons/3.8/python/lib/python3.8/functools.py) + +or + +.. code-block:: + + ERROR: Package 'blacken-docs' requires a different Python: 3.8.18 not in '>=3.9' + + +It means that your pre-commit hook is installed with (already End-Of-Life) Python 3.8 and you should reinstall +it and clean pre-commit cache. + +This can be done (if you use ``pipx`` to install ``pre-commit``): + +.. code-block:: bash + + pipx uninstall pre-commit + pipx install pre-commit --python $(which python3.9) --force + pre-commit clean + pre-commit install + +If you installed ``pre-commit`` differently, you should remove and reinstall +it (and clean cache) in the way you installed it. + + Bad Interpreter Error --------------------- diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index b0a1ca2877332..95ecf30144ab5 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -797,7 +797,7 @@ def prepare_provider_packages( skip_tag_check: bool, version_suffix_for_pypi: str, ): - check_python_version() + check_python_version(release_provider_packages=True) perform_environment_checks() fix_ownership_using_docker() cleanup_python_generated_files() diff --git a/dev/breeze/src/airflow_breeze/utils/python_versions.py b/dev/breeze/src/airflow_breeze/utils/python_versions.py index c7d514f5ff967..d144139b06816 100644 --- a/dev/breeze/src/airflow_breeze/utils/python_versions.py +++ b/dev/breeze/src/airflow_breeze/utils/python_versions.py @@ -45,14 +45,12 @@ def get_python_version_list(python_versions: str) -> list[str]: return python_version_list -def check_python_version(): - error = False - if not sys.version_info >= (3, 9): - get_console().print("[error]At least Python 3.9 is required to prepare reproducible archives.\n") - error = True - if error: +def check_python_version(release_provider_packages: bool = False): + if not sys.version_info < (3, 12) and release_provider_packages: + get_console().print("[error]Python 3.12 is not supported.\n") get_console().print( - "[warning]Please reinstall Breeze using Python 3.9 - 3.11 environment.[/]\n\n" + "[warning]Please reinstall Breeze using Python 3.9 - 3.11 environment because not all " + "provider packages support Python 3.12 yet.[/]\n\n" "For example:\n\n" "pipx uninstall apache-airflow-breeze\n" "pipx install --python $(which python3.9) -e ./dev/breeze --force\n" diff --git a/providers/src/airflow/providers/apache/beam/provider.yaml b/providers/src/airflow/providers/apache/beam/provider.yaml index ae0c4d37005b8..684882f75b83a 100644 --- a/providers/src/airflow/providers/apache/beam/provider.yaml +++ b/providers/src/airflow/providers/apache/beam/provider.yaml @@ -72,7 +72,10 @@ additional-extras: - apache-beam[gcp] # Apache Beam currently does not support Python 3.12 -# There is an issue tracking it https://github.com/apache/beam/issues/29149 +# There is an issue tracking it https://github.com/apache/beam/issues/29149. +# While the original issue above is closed, Apache Beam still does not support Python 3.12 +# because the dill version used by them break our PythonVirtualenvOperator when dill is enabled +# See https://github.com/apache/beam/issues/32617 excluded-python-versions: ['3.12'] integrations: From 85a6175bb41a6a415d347df8541f6c4f23104743 Mon Sep 17 00:00:00 2001 From: Andre Miranda Date: Fri, 11 Oct 2024 15:39:29 -0400 Subject: [PATCH 073/199] Update README.rst (#42941) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grammar update for clarity 😄 👍 --- docs/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.rst b/docs/README.rst index 99196721958db..2e40799741166 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -20,7 +20,7 @@ Documentation This directory contains documentation for the `Apache Airflow project `__ and the providers packages that are closely related to it. You can contribute to the Airflow Docs in the same way and for the same reasons as contributing code; Docs contributions that improve existing content, fix bugs, and create new content are welcomed and encouraged. -This README gives an overview about how Airflow uses `Sphinx `__ to write and build docs. It also includes instructions for how to make Docs changes locally or with the GitHub UI. +This README gives an overview about how Airflow uses `Sphinx `__ to write and build docs. It also includes instructions on how to make Docs changes locally or with the GitHub UI. Development documentation preview ================================== From 52b41fd08cfcb70a8ebd02d4b9bb8cdedb8828cf Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:37:59 -0700 Subject: [PATCH 074/199] Add a "backfill create" command (#42922) I create a new command group "backfill" for management of backfills. The first action is "create" which creates a backfill. Some others may follow such as pause / cancel. --- airflow/cli/cli_config.py | 137 ++++------- airflow/cli/commands/backfill_command.py | 44 ++++ airflow/cli/commands/dag_command.py | 102 +------- airflow/executors/executor_loader.py | 4 +- airflow/models/dag.py | 3 - tests/cli/commands/test_backfill_command.py | 89 +++++++ tests/cli/commands/test_dag_command.py | 251 -------------------- tests/jobs/test_backfill_job.py | 20 -- 8 files changed, 176 insertions(+), 474 deletions(-) create mode 100644 airflow/cli/commands/backfill_command.py create mode 100644 tests/cli/commands/test_backfill_command.py diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 4b42f2b96fb2c..e68a464f61bb5 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -264,7 +264,7 @@ def string_lower_type(val): help="The number of next execution datetimes to show", ) -# backfill +# misc ARG_MARK_SUCCESS = Arg( ("-m", "--mark-success"), help="Mark jobs as succeeded without running them", action="store_true" ) @@ -300,78 +300,34 @@ def string_lower_type(val): ) ARG_VERBOSE = Arg(("-v", "--verbose"), help="Make logging output more verbose", action="store_true") ARG_LOCAL = Arg(("-l", "--local"), help="Run the task using the LocalExecutor", action="store_true") -ARG_DONOT_PICKLE = Arg( - ("-x", "--donot-pickle"), - help=( - "Do not attempt to pickle the DAG object to send over " - "to the workers, just tell the workers to run their version " - "of the code" - ), - action="store_true", -) -ARG_BF_IGNORE_DEPENDENCIES = Arg( - ("-i", "--ignore-dependencies"), - help=( - "Skip upstream tasks, run only the tasks " - "matching the regexp. Only works in conjunction " - "with task_regex" - ), - action="store_true", -) ARG_POOL = Arg(("--pool",), "Resource pool to use") -ARG_DELAY_ON_LIMIT = Arg( - ("--delay-on-limit",), - help=( - "Amount of time in seconds to wait when the limit " - "on maximum active dag runs (max_active_runs) has " - "been reached before trying to execute a dag run " - "again" - ), - type=float, - default=1.0, -) -ARG_RESET_DAG_RUN = Arg( - ("--reset-dagruns",), - help=( - "if set, the backfill will delete existing " - "backfill-related DAG runs and start " - "anew with fresh, running DAG runs" - ), - action="store_true", -) -ARG_RERUN_FAILED_TASKS = Arg( - ("--rerun-failed-tasks",), - help=( - "if set, the backfill will auto-rerun " - "all the failed tasks for the backfill date range " - "instead of throwing exceptions" - ), - action="store_true", -) -ARG_CONTINUE_ON_FAILURES = Arg( - ("--continue-on-failures",), - help=("if set, the backfill will keep going even if some of the tasks failed"), - action="store_true", + + +# backfill +ARG_BACKFILL_DAG = Arg(flags=("--dag",), help="The dag to backfill.", required=True) +ARG_BACKFILL_FROM_DATE = Arg( + ("--from-date",), help="Earliest logical date to backfill.", type=parsedate, required=True ) -ARG_DISABLE_RETRY = Arg( - ("--disable-retry",), - help=("if set, the backfill will set tasks as failed without retrying."), - action="store_true", +ARG_BACKFILL_TO_DATE = Arg( + ("--to-date",), help="Latest logical date to backfill", type=parsedate, required=True ) +ARG_DAG_RUN_CONF = Arg(flags=("--dag-run-conf",), help="JSON dag run configuration.") ARG_RUN_BACKWARDS = Arg( - ( - "-B", - "--run-backwards", - ), + flags=("--run-backwards",), help=( - "if set, the backfill will run tasks from the most " - "recent day first. if there are tasks that depend_on_past " - "this option will throw an exception" + "If set, the backfill will run tasks from the most recent logical date first. " + "Not supported if there are tasks that depend_on_past." ), action="store_true", ) +ARG_MAX_ACTIVE_RUNS = Arg( + ("--max-active-runs",), + type=positive_int(allow_zero=False), + help="Max active runs for this backfill.", +) +# misc ARG_TREAT_DAG_ID_AS_REGEX = Arg( ("--treat-dag-id-as-regex",), help=("if set, dag_id will be treated as regex instead of an exact string"), @@ -1056,6 +1012,22 @@ class GroupCommand(NamedTuple): CLICommand = Union[ActionCommand, GroupCommand] +BACKFILL_COMMANDS = ( + ActionCommand( + name="create", + help="Create a backfill for a dag.", + description="Run subsections of a DAG for a specified date range.", + func=lazy_load_command("airflow.cli.commands.backfill_command.create_backfill"), + args=( + ARG_BACKFILL_DAG, + ARG_BACKFILL_FROM_DATE, + ARG_BACKFILL_TO_DATE, + ARG_DAG_RUN_CONF, + ARG_RUN_BACKWARDS, + ARG_MAX_ACTIVE_RUNS, + ), + ), +) DAGS_COMMANDS = ( ActionCommand( name="details", @@ -1227,40 +1199,6 @@ class GroupCommand(NamedTuple): ARG_VERBOSE, ), ), - ActionCommand( - name="backfill", - help="Run subsections of a DAG for a specified date range", - description=( - "Run subsections of a DAG for a specified date range. If reset_dag_run option is used, " - "backfill will first prompt users whether airflow should clear all the previous dag_run and " - "task_instances within the backfill date range. If rerun_failed_tasks is used, backfill " - "will auto re-run the previous failed task instances within the backfill date range" - ), - func=lazy_load_command("airflow.cli.commands.dag_command.dag_backfill"), - args=( - ARG_DAG_ID, - ARG_TASK_REGEX, - ARG_START_DATE, - ARG_END_DATE, - ARG_MARK_SUCCESS, - ARG_LOCAL, - ARG_DONOT_PICKLE, - ARG_YES, - ARG_CONTINUE_ON_FAILURES, - ARG_DISABLE_RETRY, - ARG_BF_IGNORE_DEPENDENCIES, - ARG_SUBDIR, - ARG_POOL, - ARG_DELAY_ON_LIMIT, - ARG_DRY_RUN, - ARG_VERBOSE, - ARG_CONF, - ARG_RESET_DAG_RUN, - ARG_RERUN_FAILED_TASKS, - ARG_RUN_BACKWARDS, - ARG_TREAT_DAG_ID_AS_REGEX, - ), - ), ActionCommand( name="test", help="Execute one single DagRun", @@ -1913,6 +1851,11 @@ class GroupCommand(NamedTuple): help="Manage DAGs", subcommands=DAGS_COMMANDS, ), + GroupCommand( + name="backfill", + help="Manage backfills", + subcommands=BACKFILL_COMMANDS, + ), GroupCommand( name="tasks", help="Manage tasks", diff --git a/airflow/cli/commands/backfill_command.py b/airflow/cli/commands/backfill_command.py new file mode 100644 index 0000000000000..8714ed5585004 --- /dev/null +++ b/airflow/cli/commands/backfill_command.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import logging +import signal + +from airflow import settings +from airflow.models.backfill import _create_backfill +from airflow.utils import cli as cli_utils +from airflow.utils.cli import sigint_handler +from airflow.utils.providers_configuration_loader import providers_configuration_loaded + + +@cli_utils.action_cli +@providers_configuration_loaded +def create_backfill(args) -> None: + """Create backfill job or dry run for a DAG or list of DAGs using regex.""" + logging.basicConfig(level=settings.LOGGING_LEVEL, format=settings.SIMPLE_LOG_FORMAT) + signal.signal(signal.SIGTERM, sigint_handler) + + _create_backfill( + dag_id=args.dag, + from_date=args.from_date, + to_date=args.to_date, + max_active_runs=args.max_active_runs, + reverse=args.run_backwards, + dag_run_conf=args.dag_run_conf, + ) diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index 0697709754765..83d0430a717bd 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -23,7 +23,6 @@ import json import logging import operator -import signal import subprocess import sys from typing import TYPE_CHECKING @@ -31,128 +30,31 @@ import re2 from sqlalchemy import delete, select -from airflow import settings from airflow.api.client import get_current_api_client from airflow.api_connexion.schemas.dag_schema import dag_schema from airflow.cli.simple_table import AirflowConsole -from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.jobs.job import Job from airflow.models import DagBag, DagModel, DagRun, TaskInstance -from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel from airflow.utils import cli as cli_utils, timezone -from airflow.utils.cli import get_dag, get_dags, process_subdir, sigint_handler, suppress_logs_and_warning +from airflow.utils.cli import get_dag, process_subdir, suppress_logs_and_warning from airflow.utils.dag_parsing_context import _airflow_parsing_context_manager from airflow.utils.dot_renderer import render_dag, render_dag_dependencies from airflow.utils.helpers import ask_yesno from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.session import NEW_SESSION, create_session, provide_session from airflow.utils.state import DagRunState -from airflow.utils.types import DagRunTriggeredByType if TYPE_CHECKING: from graphviz.dot import Dot from sqlalchemy.orm import Session + from airflow.models.dag import DAG from airflow.timetables.base import DataInterval - log = logging.getLogger(__name__) -def _run_dag_backfill(dags: list[DAG], args) -> None: - # If only one date is passed, using same as start and end - args.end_date = args.end_date or args.start_date - args.start_date = args.start_date or args.end_date - - run_conf = None - if args.conf: - run_conf = json.loads(args.conf) - - for dag in dags: - if args.task_regex: - dag = dag.partial_subset( - task_ids_or_regex=args.task_regex, include_upstream=not args.ignore_dependencies - ) - if not dag.task_dict: - raise AirflowException( - f"There are no tasks that match '{args.task_regex}' regex. Nothing to run, exiting..." - ) - - if args.dry_run: - print(f"Dry run of DAG {dag.dag_id} on {args.start_date}") - dagrun_infos = dag.iter_dagrun_infos_between(earliest=args.start_date, latest=args.end_date) - for dagrun_info in dagrun_infos: - dr = DagRun( - dag.dag_id, - execution_date=dagrun_info.logical_date, - data_interval=dagrun_info.data_interval, - triggered_by=DagRunTriggeredByType.CLI, - ) - - for task in dag.tasks: - print(f"Task {task.task_id} located in DAG {dag.dag_id}") - ti = TaskInstance(task, run_id=None) - ti.dag_run = dr - ti.dry_run() - else: - if args.reset_dagruns: - DAG.clear_dags( - [dag], - start_date=args.start_date, - end_date=args.end_date, - confirm_prompt=not args.yes, - dag_run_state=DagRunState.QUEUED, - ) - - try: - dag.run( - start_date=args.start_date, - end_date=args.end_date, - mark_success=args.mark_success, - local=args.local, - donot_pickle=(args.donot_pickle or conf.getboolean("core", "donot_pickle")), - ignore_first_depends_on_past=args.ignore_first_depends_on_past, - ignore_task_deps=args.ignore_dependencies, - pool=args.pool, - delay_on_limit_secs=args.delay_on_limit, - verbose=args.verbose, - conf=run_conf, - rerun_failed_tasks=args.rerun_failed_tasks, - run_backwards=args.run_backwards, - continue_on_failures=args.continue_on_failures, - disable_retry=args.disable_retry, - ) - except ValueError as vr: - print(str(vr)) - sys.exit(1) - - -@cli_utils.action_cli -@providers_configuration_loaded -def dag_backfill(args, dag: list[DAG] | DAG | None = None) -> None: - """Create backfill job or dry run for a DAG or list of DAGs using regex.""" - logging.basicConfig(level=settings.LOGGING_LEVEL, format=settings.SIMPLE_LOG_FORMAT) - signal.signal(signal.SIGTERM, sigint_handler) - args.ignore_first_depends_on_past = True - - if not args.start_date and not args.end_date: - raise AirflowException("Provide a start_date and/or end_date") - - if not dag: - dags = get_dags(args.subdir, dag_id=args.dag_id, use_regex=args.treat_dag_id_as_regex) - elif isinstance(dag, list): - dags = dag - else: - dags = [dag] - del dag - - dags.sort(key=lambda d: d.dag_id) - _run_dag_backfill(dags, args) - if len(dags) > 1: - log.info("All of the backfills are done.") - - @cli_utils.action_cli @providers_configuration_loaded def dag_trigger(args) -> None: diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py index ec79860918b80..4a940793df27f 100644 --- a/airflow/executors/executor_loader.py +++ b/airflow/executors/executor_loader.py @@ -171,10 +171,8 @@ def set_default_executor(cls, executor: BaseExecutor) -> None: """ Externally set an executor to be the default. - This is used in rare cases such as dag.run which allows, as a user convenience, to provide + This is used in rare cases such as dag.test which allows, as a user convenience, to provide the executor by cli/argument instead of Airflow configuration - - todo: given comments above, is this needed anymore since DAG.run is removed? """ exec_class_name = executor.__class__.__qualname__ exec_name = ExecutorName(f"{executor.__module__}.{exec_class_name}") diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 8d85152677e0b..f5def92ea92a7 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2293,9 +2293,6 @@ def _remove_task(self, task_id: str) -> None: self.task_count = len(self.task_dict) - def run(self, *args, **kwargs): - """Leaving this here to be removed in other PR for simpler review.""" - def cli(self): """Exposes a CLI specific to this DAG.""" check_cycle(self) diff --git a/tests/cli/commands/test_backfill_command.py b/tests/cli/commands/test_backfill_command.py new file mode 100644 index 0000000000000..c01e1e4f9d07b --- /dev/null +++ b/tests/cli/commands/test_backfill_command.py @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import argparse +from datetime import datetime +from unittest import mock + +import pendulum +import pytest + +import airflow.cli.commands.backfill_command +from airflow.cli import cli_parser +from airflow.models import DagBag +from airflow.utils import timezone + +from dev.tests_common.test_utils.db import clear_db_backfills, clear_db_dags, clear_db_runs + +DEFAULT_DATE = timezone.make_aware(datetime(2015, 1, 1), timezone=timezone.utc) +if pendulum.__version__.startswith("3"): + DEFAULT_DATE_REPR = DEFAULT_DATE.isoformat(sep=" ") +else: + DEFAULT_DATE_REPR = DEFAULT_DATE.isoformat() + +# TODO: Check if tests needs side effects - locally there's missing DAG + +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] + + +class TestCliBackfill: + parser: argparse.ArgumentParser + + @classmethod + def setup_class(cls): + cls.dagbag = DagBag(include_examples=True) + cls.dagbag.sync_to_db() + cls.parser = cli_parser.get_parser() + + @classmethod + def teardown_class(cls) -> None: + clear_db_runs() + clear_db_dags() + clear_db_backfills() + + def setup_method(self): + clear_db_runs() # clean-up all dag run before start each test + clear_db_dags() + clear_db_backfills() + + @mock.patch("airflow.cli.commands.backfill_command._create_backfill") + def test_backfill(self, mock_create): + airflow.cli.commands.backfill_command.create_backfill( + self.parser.parse_args( + [ + "backfill", + "create", + "--dag", + "example_bash_operator", + "--from-date", + DEFAULT_DATE.isoformat(), + "--to-date", + DEFAULT_DATE.isoformat(), + ] + ) + ) + + mock_create.assert_called_once_with( + dag_id="example_bash_operator", + from_date=DEFAULT_DATE, + to_date=DEFAULT_DATE, + max_active_runs=None, + reverse=False, + dag_run_conf=None, + ) diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py index f0c9a18c1a567..e9d10a2e33c18 100644 --- a/tests/cli/commands/test_dag_command.py +++ b/tests/cli/commands/test_dag_command.py @@ -124,139 +124,6 @@ def test_reserialize_should_support_subdir_argument(self): serialized_dags_after_reserialize = session.query(SerializedDagModel).all() assert len(serialized_dags_after_reserialize) == 1 # Serialized DAG back - @mock.patch("airflow.cli.commands.dag_command.DAG.run") - def test_backfill(self, mock_run): - dag_command.dag_backfill( - self.parser.parse_args( - ["dags", "backfill", "example_bash_operator", "--start-date", DEFAULT_DATE.isoformat()] - ) - ) - - mock_run.assert_called_once_with( - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - conf=None, - delay_on_limit_secs=1.0, - donot_pickle=False, - ignore_first_depends_on_past=True, - ignore_task_deps=False, - local=False, - mark_success=False, - pool=None, - rerun_failed_tasks=False, - run_backwards=False, - verbose=False, - continue_on_failures=False, - disable_retry=False, - ) - mock_run.reset_mock() - dag = self.dagbag.get_dag("example_bash_operator") - - with contextlib.redirect_stdout(StringIO()) as stdout: - dag_command.dag_backfill( - self.parser.parse_args( - [ - "dags", - "backfill", - "example_bash_operator", - "--task-regex", - "runme_0", - "--dry-run", - "--start-date", - DEFAULT_DATE.isoformat(), - ] - ), - dag=dag, - ) - - output = stdout.getvalue() - assert f"Dry run of DAG example_bash_operator on {DEFAULT_DATE_REPR}\n" in output - assert "Task runme_0 located in DAG example_bash_operator\n" in output - - mock_run.assert_not_called() # Dry run shouldn't run the backfill - - dag_command.dag_backfill( - self.parser.parse_args( - [ - "dags", - "backfill", - "example_bash_operator", - "--dry-run", - "--start-date", - DEFAULT_DATE.isoformat(), - ] - ), - dag=dag, - ) - - mock_run.assert_not_called() # Dry run shouldn't run the backfill - - dag_command.dag_backfill( - self.parser.parse_args( - [ - "dags", - "backfill", - "example_bash_operator", - "--local", - "--start-date", - DEFAULT_DATE.isoformat(), - ] - ), - dag=dag, - ) - - mock_run.assert_called_once_with( - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - conf=None, - delay_on_limit_secs=1.0, - donot_pickle=False, - ignore_first_depends_on_past=True, - ignore_task_deps=False, - local=True, - mark_success=False, - pool=None, - rerun_failed_tasks=False, - run_backwards=False, - verbose=False, - continue_on_failures=False, - disable_retry=False, - ) - mock_run.reset_mock() - - with contextlib.redirect_stdout(StringIO()) as stdout: - dag_command.dag_backfill( - self.parser.parse_args( - [ - "dags", - "backfill", - "example_branch_(python_){0,1}operator(_decorator){0,1}", - "--task-regex", - "run_this_first", - "--dry-run", - "--treat-dag-id-as-regex", - "--start-date", - DEFAULT_DATE.isoformat(), - ] - ), - ) - - output = stdout.getvalue() - - assert f"Dry run of DAG example_branch_python_operator_decorator on {DEFAULT_DATE_REPR}\n" in output - assert "Task run_this_first located in DAG example_branch_python_operator_decorator\n" in output - assert f"Dry run of DAG example_branch_operator on {DEFAULT_DATE_REPR}\n" in output - assert "Task run_this_first located in DAG example_branch_operator\n" in output - - @mock.patch("airflow.cli.commands.dag_command.get_dag") - def test_backfill_fails_without_loading_dags(self, mock_get_dag): - cli_args = self.parser.parse_args(["dags", "backfill", "example_bash_operator"]) - - with pytest.raises(AirflowException): - dag_command.dag_backfill(cli_args) - - mock_get_dag.assert_not_called() - def test_show_dag_dependencies_print(self): with contextlib.redirect_stdout(StringIO()) as temp_stdout: dag_command.dag_dependencies_show(self.parser.parse_args(["dags", "show-dependencies"])) @@ -314,124 +181,6 @@ def test_show_dag_imgcat(self, mock_render_dag, mock_popen): assert "OUT" in out assert "ERR" in out - @mock.patch("airflow.cli.commands.dag_command.DAG.run") - def test_cli_backfill_ignore_first_depends_on_past(self, mock_run): - """ - Test that CLI respects -I argument - - We just check we call dag.run() right. The behaviour of that kwarg is - tested in test_jobs - """ - dag_id = "example_bash_operator" - run_date = DEFAULT_DATE + timedelta(days=1) - args = [ - "dags", - "backfill", - dag_id, - "--local", - "--start-date", - run_date.isoformat(), - ] - dag = self.dagbag.get_dag(dag_id) - - dag_command.dag_backfill(self.parser.parse_args(args), dag=dag) - - mock_run.assert_called_once_with( - start_date=run_date, - end_date=run_date, - conf=None, - delay_on_limit_secs=1.0, - donot_pickle=False, - ignore_first_depends_on_past=True, - ignore_task_deps=False, - local=True, - mark_success=False, - pool=None, - rerun_failed_tasks=False, - run_backwards=False, - verbose=False, - continue_on_failures=False, - disable_retry=False, - ) - - @pytest.mark.parametrize( - "cli_arg", - [ - pytest.param("-B", id="short"), - pytest.param("--run-backwards", id="full"), - ], - ) - @mock.patch("airflow.cli.commands.dag_command.DAG.run") - def test_cli_backfill_depends_on_past_run_backwards(self, mock_run, cli_arg: str): - """Test that CLI respects -B argument.""" - dag_id = "test_depends_on_past" - start_date = DEFAULT_DATE + timedelta(days=1) - end_date = start_date + timedelta(days=1) - args = [ - "dags", - "backfill", - dag_id, - "--local", - "--start-date", - start_date.isoformat(), - "--end-date", - end_date.isoformat(), - cli_arg, - ] - dag = self.dagbag.get_dag(dag_id) - - dag_command.dag_backfill(self.parser.parse_args(args), dag=dag) - mock_run.assert_called_once_with( - start_date=start_date, - end_date=end_date, - conf=None, - delay_on_limit_secs=1.0, - donot_pickle=False, - ignore_first_depends_on_past=True, - ignore_task_deps=False, - local=True, - mark_success=False, - pool=None, - rerun_failed_tasks=False, - run_backwards=True, - verbose=False, - continue_on_failures=False, - disable_retry=False, - ) - - @mock.patch("airflow.models.taskinstance.TaskInstance.dry_run") - @mock.patch("airflow.cli.commands.dag_command.DagRun") - def test_backfill_with_custom_timetable(self, mock_dagrun, mock_dry_run): - """ - when calling `dags backfill` on dag with custom timetable, the DagRun object should be created with - data_intervals. - """ - - start_date = DEFAULT_DATE + timedelta(days=1) - end_date = start_date + timedelta(days=1) - workdays = [ - start_date, - start_date + timedelta(days=1), - start_date + timedelta(days=2), - ] - cli_args = self.parser.parse_args( - [ - "dags", - "backfill", - "example_workday_timetable", - "--start-date", - start_date.isoformat(), - "--end-date", - end_date.isoformat(), - "--dry-run", - ] - ) - from airflow.example_dags.plugins.workday import AfterWorkdayTimetable - - with mock.patch.object(AfterWorkdayTimetable, "get_next_workday", side_effect=workdays): - dag_command.dag_backfill(cli_args) - assert "data_interval" in mock_dagrun.call_args.kwargs - def test_next_execution(self, tmp_path): dag_test_list = [ ("future_schedule_daily", "timedelta(days=5)", "'0 0 * * *'", "True"), diff --git a/tests/jobs/test_backfill_job.py b/tests/jobs/test_backfill_job.py index dead9be86230b..616f328ee4177 100644 --- a/tests/jobs/test_backfill_job.py +++ b/tests/jobs/test_backfill_job.py @@ -29,7 +29,6 @@ import pytest from airflow import settings -from airflow.cli import cli_parser from airflow.exceptions import ( AirflowException, BackfillUnfinished, @@ -186,7 +185,6 @@ def clean_db(): @pytest.fixture(autouse=True) def set_instance_attrs(self, dag_bag): self.clean_db() - self.parser = cli_parser.get_parser() self.dagbag = dag_bag # `airflow tasks run` relies on serialized_dag for dag in self.dagbag.dags.values(): @@ -1126,24 +1124,6 @@ def test_backfill_depends_on_past_backwards(self, mock_executor): with pytest.raises(AirflowException, match=expected_msg): run_job(job=job, execute_callable=job_runner._execute) - def test_cli_receives_delay_arg(self): - """ - Tests that the --delay argument is passed correctly to the BackfillJob - """ - dag_id = "example_bash_operator" - run_date = DEFAULT_DATE - args = [ - "dags", - "backfill", - dag_id, - "-s", - run_date.isoformat(), - "--delay-on-limit", - "0.5", - ] - parsed_args = self.parser.parse_args(args) - assert 0.5 == parsed_args.delay_on_limit - def _get_dag_test_max_active_limits( self, dag_maker_fixture, dag_id="test_dag", max_active_runs=1, **kwargs ): From 2845cfcc8febcb15f8299fd457c5c80f2e46254f Mon Sep 17 00:00:00 2001 From: Jianzhun Du <68252326+sfc-gh-jdu@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:41:32 -0700 Subject: [PATCH 075/199] Fix SNOWFLAKE_CONN_ID and DAG_ID in Snowpark system tests (#42952) --- .../tests/system/snowflake/example_snowpark_decorator.py | 4 ++-- providers/tests/system/snowflake/example_snowpark_operator.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/tests/system/snowflake/example_snowpark_decorator.py b/providers/tests/system/snowflake/example_snowpark_decorator.py index 3f74b97d0f23e..4c2319a91b21e 100644 --- a/providers/tests/system/snowflake/example_snowpark_decorator.py +++ b/providers/tests/system/snowflake/example_snowpark_decorator.py @@ -30,8 +30,8 @@ from airflow import DAG from airflow.decorators import task -SNOWFLAKE_CONN_ID = "snowflake_default" -DAG_ID = "example_snowpark" +SNOWFLAKE_CONN_ID = "my_snowflake_conn" +DAG_ID = "example_snowpark_decorator" with DAG( DAG_ID, diff --git a/providers/tests/system/snowflake/example_snowpark_operator.py b/providers/tests/system/snowflake/example_snowpark_operator.py index 0543596ec9b1f..1d7cce34f7aa1 100644 --- a/providers/tests/system/snowflake/example_snowpark_operator.py +++ b/providers/tests/system/snowflake/example_snowpark_operator.py @@ -30,8 +30,8 @@ from airflow import DAG from airflow.providers.snowflake.operators.snowpark import SnowparkOperator -SNOWFLAKE_CONN_ID = "snowflake_default" -DAG_ID = "example_snowpark" +SNOWFLAKE_CONN_ID = "my_snowflake_conn" +DAG_ID = "example_snowpark_operator" with DAG( DAG_ID, From fac537690464aeecb1dab94470d3c6f7a431e940 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sat, 12 Oct 2024 06:21:06 +0100 Subject: [PATCH 076/199] Fix broken links in Release Management docs (#42958) These links were pointing to the wrong location. This PR fixes it --- .../doc/09_release_management_tasks.rst | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 930f61159d16e..9feb9a5b195e6 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -26,7 +26,7 @@ do not need or have no access to run). Those are usually connected with releasin Those are all of the available release management commands: .. image:: ./images/output_release-management.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management.svg :width: 100% :alt: Breeze release management @@ -55,7 +55,7 @@ default is to build ``both`` type of packages ``sdist`` and ``wheel``. breeze release-management prepare-airflow-package --package-format=wheel .. image:: ./images/output_release-management_prepare-airflow-package.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_prepare-airflow-package.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-airflow-package.svg :width: 100% :alt: Breeze release-management prepare-airflow-package @@ -79,7 +79,7 @@ tarball for. breeze release-management prepare-airflow-tarball --version 2.8.0rc1 .. image:: ./images/output_release-management_prepare-airflow-tarball.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_prepare-airflow-tarball.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-airflow-tarball.svg :width: 100% :alt: Breeze release-management prepare-airflow-tarball @@ -94,7 +94,7 @@ automates it. breeze release-management create-minor-branch .. image:: ./images/output_release-management_create-minor-branch.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_create-minor-branch.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_create-minor-branch.svg :width: 100% :alt: Breeze release-management create-minor-branch @@ -109,7 +109,7 @@ When we prepare release candidate, we automate some of the steps we need to do. breeze release-management start-rc-process .. image:: ./images/output_release-management_start-rc-process.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_start-rc-process.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_start-rc-process.svg :width: 100% :alt: Breeze release-management start-rc-process @@ -123,7 +123,7 @@ When we prepare final release, we automate some of the steps we need to do. breeze release-management start-release .. image:: ./images/output_release-management_start-release.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_start-rc-process.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_start-rc-process.svg :width: 100% :alt: Breeze release-management start-rc-process @@ -154,7 +154,7 @@ You can also generate python client with custom security schemes. These are all of the available flags for the command: .. image:: ./images/output_release-management_prepare-python-client.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_prepare-python-client.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-python-client.svg :width: 100% :alt: Breeze release management prepare Python client @@ -185,7 +185,7 @@ step can be skipped if you pass the ``--skip-latest`` flag. These are all of the available flags for the ``release-prod-images`` command: .. image:: ./images/output_release-management_release-prod-images.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_release-prod-images.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_release-prod-images.svg :width: 100% :alt: Breeze release management release prod images @@ -208,7 +208,7 @@ However, If you want to disable this behaviour, set the envvar CLEAN_LOCAL_TAGS These are all of the available flags for the ``tag-providers`` command: .. image:: ./images/output_release-management_tag-providers.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_tag-providers.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_tag-providers.svg :width: 100% :alt: Breeze release management tag-providers @@ -234,7 +234,7 @@ which version of Helm Chart you are preparing the tarball for. breeze release-management prepare-helm-chart-tarball --version 1.12.0 --version-suffix rc1 .. image:: ./images/output_release-management_prepare-helm-chart-tarball.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_prepare-helm-chart-tarball.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-helm-chart-tarball.svg :width: 100% :alt: Breeze release-management prepare-helm-chart-tarball @@ -256,7 +256,7 @@ This prepares helm chart .tar.gz package in the dist folder. breeze release-management prepare-helm-chart-package --sign myemail@apache.org .. image:: ./images/output_release-management_prepare-helm-chart-package.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_prepare-helm-chart-package.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-helm-chart-package.svg :width: 100% :alt: Breeze release-management prepare-helm-chart-package @@ -292,7 +292,7 @@ The below example perform documentation preparation for provider packages. You can also add ``--answer yes`` to perform non-interactive build. .. image:: ./images/output_release-management_prepare-provider-documentation.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_prepare-provider-documentation.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg :width: 100% :alt: Breeze prepare-provider-documentation @@ -325,7 +325,7 @@ You can see all providers available by running this command: breeze release-management prepare-provider-packages --help .. image:: ./images/output_release-management_prepare-provider-packages.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_prepare-provider-packages.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg :width: 100% :alt: Breeze prepare-provider-packages @@ -349,7 +349,7 @@ You can also run the verification with an earlier airflow version to check for c All the command parameters are here: .. image:: ./images/output_release-management_install-provider-packages.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_install-provider-packages.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_install-provider-packages.svg :width: 100% :alt: Breeze install-provider-packages @@ -373,7 +373,7 @@ You can also run the verification with an earlier airflow version to check for c All the command parameters are here: .. image:: ./images/output_release-management_verify-provider-packages.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_verify-provider-packages.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg :width: 100% :alt: Breeze verify-provider-packages @@ -387,7 +387,7 @@ provider has been released) and date of the release of the provider version. These are all of the available flags for the ``generate-providers-metadata`` command: .. image:: ./images/output_release-management_generate-providers-metadata.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_generate-providers-metadata.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg :width: 100% :alt: Breeze release management generate providers metadata @@ -398,7 +398,7 @@ Generating Provider Issue You can use Breeze to generate a provider issue when you release new providers. .. image:: ./images/output_release-management_generate-issue-content-providers.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_generate-issue-content-providers.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg :width: 100% :alt: Breeze generate-issue-content-providers @@ -414,7 +414,7 @@ command. These are all available flags of ``clean-old-provider-artifacts`` command: .. image:: ./images/output_release-management_clean-old-provider-artifacts.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_clean-old-provider-artifacts.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_clean-old-provider-artifacts.svg :width: 100% :alt: Breeze Clean Old Provider Artifacts @@ -462,7 +462,7 @@ Constraints are generated separately for each python version and there are separ These are all available flags of ``generate-constraints`` command: .. image:: ./images/output_release-management_generate-constraints.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_generate-constraints.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_generate-constraints.svg :width: 100% :alt: Breeze generate-constraints @@ -485,7 +485,7 @@ tagged already in the past. This can be done using ``breeze release-management u These are all available flags of ``update-constraints`` command: .. image:: ./images/output_release-management_update-constraints.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_update-constraints.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_update-constraints.svg :width: 100% :alt: Breeze update-constraints @@ -552,7 +552,7 @@ publishing docs for multiple providers. These are all available flags of ``release-management publish-docs`` command: .. image:: ./images/output_release-management_publish-docs.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_publish-docs.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_publish-docs.svg :width: 100% :alt: Breeze Publish documentation @@ -596,7 +596,7 @@ providers - you can mix apache-airflow, helm-chart and provider packages this wa These are all available flags of ``release-management add-back-references`` command: .. image:: ./images/output_release-management_add-back-references.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_release-management_add-back-references.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_add-back-references.svg :width: 100% :alt: Breeze Add Back References @@ -606,7 +606,7 @@ SBOM generation tasks Maintainers also can use Breeze for SBOM generation: .. image:: ./images/output_sbom.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_sbom.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_sbom.svg :width: 100% :alt: Breeze sbom @@ -619,7 +619,7 @@ done by the ``generate-providers-requirements`` command. This command generates selected provider and python version, using the airflow version specified. .. image:: ./images/output_sbom_generate-providers-requirements.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_sbom_generate-providers-requirements.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg :width: 100% :alt: Breeze generate SBOM provider requirements @@ -634,7 +634,7 @@ information is written directly to ``docs-archive`` in airflow-site repository. These are all of the available flags for the ``update-sbom-information`` command: .. image:: ./images/output_sbom_update-sbom-information.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_sbomt_update-sbom-information.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_sbomt_update-sbom-information.svg :width: 100% :alt: Breeze update sbom information @@ -646,7 +646,7 @@ such images are built with the ``build-all-airflow-images`` command. This command will build one docker image per python version, with all the airflow versions >=2.0.0 compatible. .. image:: ./images/output_sbom_build-all-airflow-images.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_sbom_build-all-airflow-images.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_sbom_build-all-airflow-images.svg :width: 100% :alt: Breeze build all airflow images @@ -658,7 +658,7 @@ The SBOM information published on our website can be converted into a spreadshee properties of the dependencies. This is done by the ``export-dependency-information`` command. .. image:: ./images/output_sbom_export-dependency-information.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_sbom_export-dependency-information.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_sbom_export-dependency-information.svg :width: 100% :alt: Breeze sbom export dependency information From 46759d23ffd9d46de1002ae9c4441ea187c22716 Mon Sep 17 00:00:00 2001 From: GPK Date: Sat, 12 Oct 2024 08:47:57 +0100 Subject: [PATCH 077/199] Restrict looker-sdk version 24.18.0 and microsoft-kiota-http 1.3.4 (#42954) * restrict looker version 24.18.0 * update microsoft provider deps with microsoft-kiota-http * update gh issue ids to provider yaml files * ignore type in applfowhook * ignore type in applfowhook * ignore type in applfowhook --- generated/provider_dependencies.json | 3 ++- .../src/airflow/providers/amazon/aws/hooks/appflow.py | 8 ++++---- providers/src/airflow/providers/google/provider.yaml | 4 +++- .../src/airflow/providers/microsoft/azure/provider.yaml | 3 +++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index e6a0deca23cbf..4d921fc1fb911 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -681,7 +681,7 @@ "httpx>=0.25.0", "immutabledict>=4.2.0", "json-merge-patch>=0.2", - "looker-sdk>=22.4.0", + "looker-sdk>=22.4.0,!=24.18.0", "pandas-gbq>=0.7.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", @@ -827,6 +827,7 @@ "azure-storage-file-share>=12.7.0", "azure-synapse-artifacts>=0.17.0", "azure-synapse-spark>=0.2.0", + "microsoft-kiota-http>=1.3.0,!=1.3.4", "msgraph-core>=1.0.0" ], "devel-deps": [ diff --git a/providers/src/airflow/providers/amazon/aws/hooks/appflow.py b/providers/src/airflow/providers/amazon/aws/hooks/appflow.py index 5ef994917926b..e68637e50dc0b 100644 --- a/providers/src/airflow/providers/amazon/aws/hooks/appflow.py +++ b/providers/src/airflow/providers/amazon/aws/hooks/appflow.py @@ -117,9 +117,9 @@ def update_flow_filter(self, flow_name: str, filter_tasks, set_trigger_ondemand: self.conn.update_flow( flowName=response["flowName"], - destinationFlowConfigList=response["destinationFlowConfigList"], - sourceFlowConfig=response["sourceFlowConfig"], - triggerConfig=response["triggerConfig"], + destinationFlowConfigList=response["destinationFlowConfigList"], # type: ignore[arg-type] + sourceFlowConfig=response["sourceFlowConfig"], # type: ignore[arg-type] + triggerConfig=response["triggerConfig"], # type: ignore[arg-type] description=response.get("description", "Flow description."), - tasks=tasks, + tasks=tasks, # type: ignore[arg-type] ) diff --git a/providers/src/airflow/providers/google/provider.yaml b/providers/src/airflow/providers/google/provider.yaml index 9e469d278775b..196a035badc4d 100644 --- a/providers/src/airflow/providers/google/provider.yaml +++ b/providers/src/airflow/providers/google/provider.yaml @@ -157,7 +157,9 @@ dependencies: - grpcio-gcp>=0.2.2 - httpx>=0.25.0 - json-merge-patch>=0.2 - - looker-sdk>=22.4.0 + # looker-sdk 24.18.0 has issues in import looker_sdk.rtl, No module named looker_sdk.rtl + # See https://github.com/looker-open-source/sdk-codegen/issues/1518 + - looker-sdk>=22.4.0,!=24.18.0 - pandas-gbq>=0.7.0 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies diff --git a/providers/src/airflow/providers/microsoft/azure/provider.yaml b/providers/src/airflow/providers/microsoft/azure/provider.yaml index a47ec86de9f78..cf0b3f75ef9d8 100644 --- a/providers/src/airflow/providers/microsoft/azure/provider.yaml +++ b/providers/src/airflow/providers/microsoft/azure/provider.yaml @@ -108,6 +108,9 @@ dependencies: - azure-mgmt-containerregistry>=8.0.0 - azure-mgmt-containerinstance>=10.1.0 - msgraph-core>=1.0.0 + # msgraph-core has transient import failures with microsoft-kiota-http==1.3.4 + # See https://github.com/microsoftgraph/msgraph-sdk-python-core/issues/706 + - microsoft-kiota-http>=1.3.0,!=1.3.4 devel-dependencies: - pywinrm From 9a3d385e305bea2916e7ca5817922dc6dd4cc635 Mon Sep 17 00:00:00 2001 From: GPK Date: Sat, 12 Oct 2024 10:49:00 +0100 Subject: [PATCH 078/199] upgrade trove-classifiers (#42950) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 038393b139a02..844e1b6d3ffa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "pluggy==1.5.0", "smmap==5.0.1", "tomli==2.0.2; python_version < '3.11'", - "trove-classifiers==2024.9.12", + "trove-classifiers==2024.10.11", ] build-backend = "hatchling.build" From b1f33d8dd3f19177d016a712b784a1a37b12e6a9 Mon Sep 17 00:00:00 2001 From: Aakcht Date: Sat, 12 Oct 2024 19:23:54 +0500 Subject: [PATCH 079/199] Chart: fix VCT for scheduler in local and persistent mode (#42946) --- chart/templates/scheduler/scheduler-deployment.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 634c3a41f7de0..551ab94c8c48d 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -324,7 +324,9 @@ spec: emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }} {{- else }} volumeClaimTemplates: - - metadata: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: name: logs {{- if .Values.workers.persistence.annotations }} annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }} From 31118a70f09d82374c96e98b5bd9fed080ff2ed4 Mon Sep 17 00:00:00 2001 From: Jean-Eudes Peloye Date: Sat, 12 Oct 2024 16:28:30 +0200 Subject: [PATCH 080/199] fix: use instance base_container_name to fetch logs on trigger_reentry (#42960) --- .../providers/cncf/kubernetes/operators/pod.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py b/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py index 68081b5a67075..62af050a50cdd 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py @@ -31,7 +31,7 @@ from contextlib import AbstractContextManager from enum import Enum from functools import cached_property -from typing import TYPE_CHECKING, Any, Callable, Iterable, Sequence +from typing import TYPE_CHECKING, Any, Callable, Iterable, Literal, Sequence import kubernetes import tenacity @@ -91,7 +91,6 @@ if TYPE_CHECKING: import jinja2 from pendulum import DateTime - from typing_extensions import Literal from airflow.providers.cncf.kubernetes.secret import Secret from airflow.utils.context import Context @@ -285,7 +284,8 @@ def __init__( startup_timeout_seconds: int = 120, startup_check_interval_seconds: int = 5, get_logs: bool = True, - container_logs: Iterable[str] | str | Literal[True] = BASE_CONTAINER_NAME, + base_container_name: str | None = None, + container_logs: Iterable[str] | str | Literal[True] | None = None, image_pull_policy: str | None = None, annotations: dict | None = None, container_resources: k8s.V1ResourceRequirements | None = None, @@ -315,7 +315,6 @@ def __init__( termination_grace_period: int | None = None, configmaps: list[str] | None = None, skip_on_exit_code: int | Container[int] | None = None, - base_container_name: str | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: float = 2, log_pod_spec_on_failure: bool = True, @@ -357,9 +356,10 @@ def __init__( self.cluster_context = cluster_context self.reattach_on_restart = reattach_on_restart self.get_logs = get_logs - self.container_logs = container_logs - if self.container_logs == KubernetesPodOperator.BASE_CONTAINER_NAME: - self.container_logs = base_container_name or self.BASE_CONTAINER_NAME + # Fallback to the class variable BASE_CONTAINER_NAME here instead of via default argument value + # in the init method signature, to be compatible with subclasses overloading the class variable value. + self.base_container_name = base_container_name or self.BASE_CONTAINER_NAME + self.container_logs = container_logs or self.base_container_name self.image_pull_policy = image_pull_policy self.node_selector = node_selector or {} self.annotations = annotations or {} @@ -398,7 +398,6 @@ def __init__( if skip_on_exit_code is not None else [] ) - self.base_container_name = base_container_name or self.BASE_CONTAINER_NAME self.deferrable = deferrable self.poll_interval = poll_interval self.remote_pod: k8s.V1Pod | None = None @@ -785,7 +784,7 @@ def trigger_reentry(self, context: Context, event: dict[str, Any]) -> Any: pod_log_status = self.pod_manager.fetch_container_logs( pod=self.pod, - container_name=self.BASE_CONTAINER_NAME, + container_name=self.base_container_name, follow=follow, since_time=last_log_time, ) From f558d798e9046e21d167527e2098d25d89fa93d4 Mon Sep 17 00:00:00 2001 From: Geonwoo Kim <78521993+kgw7401@users.noreply.github.com> Date: Sat, 12 Oct 2024 23:37:24 +0900 Subject: [PATCH 081/199] fix: 03_contributors_quick_start_docs (#42927) * Change code-block position --- contributing-docs/03_contributors_quick_start.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index bddecc6f1e486..4ee65da0350b8 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -273,15 +273,14 @@ Setting up Breeze means that you are inside the Breeze container and ready to run most of the development tasks. You can leave the environment with ``exit`` and re-enter it with just ``breeze`` command. -.. code-block:: bash - - root@b76fcb399bb6:/opt/airflow# airflow db reset - - 6. Once you enter breeze environment, create airflow tables and users from the breeze CLI. ``airflow db reset`` is required to execute at least once for Airflow Breeze to get the database/tables created. If you run tests, however - the test database will be initialized automatically for you. +.. code-block:: bash + + root@b76fcb399bb6:/opt/airflow# airflow db reset + .. code-block:: bash root@b76fcb399bb6:/opt/airflow# airflow users create \ From 957c1868172cd385d3ddc5d80b24cd0f81b50b35 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sat, 12 Oct 2024 15:40:10 +0100 Subject: [PATCH 082/199] Add skeleton project for task-sdk (#42904) closes https://github.com/apache/airflow/issues/42856 This PR adds a skeleton project for [AIP-72 Task SDK](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-72+Task+Execution+Interface+aka+Task+SDK) It creates this new project, integrates them to breeze & runs it on CI. The code & tests are just placeholders at this point, we will start adding things needs for DAG Authors in separate PRs. --- .dockerignore | 1 + .github/boring-cyborg.yml | 3 + .github/workflows/ci.yml | 1 + .github/workflows/task-sdk-tests.yml | 91 ++++++++++++++ Dockerfile | 11 +- Dockerfile.ci | 12 +- .../doc/09_release_management_tasks.rst | 23 ++++ .../doc/images/output_release-management.svg | 18 ++- .../doc/images/output_release-management.txt | 2 +- ...se-management_prepare-task-sdk-package.svg | 116 ++++++++++++++++++ ...se-management_prepare-task-sdk-package.txt | 1 + ...utput_setup_check-all-params-in-groups.svg | 20 +-- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 20 +-- ...output_setup_regenerate-command-images.txt | 2 +- dev/breeze/doc/images/output_shell.svg | 4 +- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_testing_db-tests.svg | 6 +- .../doc/images/output_testing_db-tests.txt | 2 +- .../images/output_testing_non-db-tests.svg | 6 +- .../images/output_testing_non-db-tests.txt | 2 +- .../doc/images/output_testing_tests.svg | 8 +- .../doc/images/output_testing_tests.txt | 2 +- .../commands/release_management_commands.py | 107 +++++++++++++++- .../release_management_commands_config.py | 15 +++ .../src/airflow_breeze/global_constants.py | 18 +++ .../utils/docker_command_utils.py | 1 + .../src/airflow_breeze/utils/run_tests.py | 1 + .../airflow_breeze/utils/selective_checks.py | 21 +++- dev/breeze/tests/test_selective_checks.py | 10 +- pyproject.toml | 6 +- scripts/ci/docker-compose/local.yml | 3 + scripts/ci/kubernetes/k8s_requirements.txt | 1 + scripts/docker/install_airflow.sh | 11 +- task_sdk/README.md | 18 +++ task_sdk/pyproject.toml | 40 ++++++ task_sdk/src/airflow/sdk/__init__.py | 21 ++++ task_sdk/src/airflow/sdk/py.typed | 18 +++ task_sdk/tests/__init__.py | 16 +++ task_sdk/tests/conftest.py | 26 ++++ task_sdk/tests/test_hello.py | 23 ++++ 41 files changed, 650 insertions(+), 61 deletions(-) create mode 100644 .github/workflows/task-sdk-tests.yml create mode 100644 dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg create mode 100644 dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.txt create mode 100644 task_sdk/README.md create mode 100644 task_sdk/pyproject.toml create mode 100644 task_sdk/src/airflow/sdk/__init__.py create mode 100644 task_sdk/src/airflow/sdk/py.typed create mode 100644 task_sdk/tests/__init__.py create mode 100644 task_sdk/tests/conftest.py create mode 100644 task_sdk/tests/test_hello.py diff --git a/.dockerignore b/.dockerignore index bdbf5fb0883af..2d70030fd886b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -35,6 +35,7 @@ !docs !licenses !providers/ +!task_sdk/ # Add those folders to the context so that they are available in the CI container !scripts diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 019ea900f8e12..e790d65e2fe6f 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -679,6 +679,9 @@ labelPRBasedOnFilePath: area:system-tests: - tests/system/**/* + area:task-sdk: + - task_sdk/**/* + area:db-migrations: - airflow/migrations/versions/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 716323cb9acfd..2267154b03a7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,7 @@ jobs: run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} + run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml new file mode 100644 index 0000000000000..14fae903837c2 --- /dev/null +++ b/.github/workflows/task-sdk-tests.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Task SDK tests +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true + type: string + image-tag: + description: "Tag to set for the image" + required: true + type: string + canary-run: + description: "Whether this is a canary run" + required: true + type: string + default-python-version: + description: "Which version of python should be used by default" + required: true + type: string + python-versions: + description: "JSON-formatted array of Python versions to build images from" + required: true + type: string +jobs: + task-sdk-tests: + timeout-minutes: 80 + name: Task SDK:P${{ matrix.python-version }} tests + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + strategy: + fail-fast: false + matrix: + python-version: "${{fromJSON(inputs.python-versions)}}" + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + IMAGE_TAG: "${{ inputs.image-tag }}" + INCLUDE_NOT_READY_PROVIDERS: "true" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + VERBOSE: "true" + CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}" + if: inputs.run-task-sdk-tests == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare breeze & CI image: ${{ matrix.python-version }}:${{ inputs.image-tag }}" + uses: ./.github/actions/prepare_breeze_and_image + - name: "Cleanup dist files" + run: rm -fv ./dist/* + - name: "Prepare Task SDK packages: wheel" + run: > + breeze release-management prepare-task-sdk-package --package-format wheel + - name: "Verify wheel packages with twine" + run: | + pipx uninstall twine || true + pipx install twine && twine check dist/*.whl + - name: > + Run provider unit tests on + Airflow Task SDK:Python ${{ matrix.python-version }} + if: matrix.run-tests == 'true' + run: > + breeze testing tests --run-in-parallel + --parallel-test-types TaskSDK + --use-packages-from-dist + --package-format wheel diff --git a/Dockerfile b/Dockerfile index ce229c75facef..7b5fa8d0a2b44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -877,13 +877,18 @@ function install_airflow() { # Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then - # We need _a _ file in there otherwise the editable install doesn't include anything in the .pth file + # We need _a_ file in there otherwise the editable install doesn't include anything in the .pth file mkdir -p ./providers/src/airflow/providers/ touch ./providers/src/airflow/providers/__init__.py - trap 'rm -f ./providers/src/airflow/providers/__init__.py 2>/dev/null' EXIT + + # Similarly we need _a_ file for task_sdk too + mkdir -p ./task_sdk/src/airflow/sdk/ + touch ./task_sdk/src/airflow/__init__.py + + trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT # When installing from sources - we always use `--editable` mode - installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers" + installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers --editable ./task_sdk" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then diff --git a/Dockerfile.ci b/Dockerfile.ci index a3e982614de69..9339e9af6d6f8 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -676,13 +676,18 @@ function install_airflow() { # Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then - # We need _a _ file in there otherwise the editable install doesn't include anything in the .pth file + # We need _a_ file in there otherwise the editable install doesn't include anything in the .pth file mkdir -p ./providers/src/airflow/providers/ touch ./providers/src/airflow/providers/__init__.py - trap 'rm -f ./providers/src/airflow/providers/__init__.py 2>/dev/null' EXIT + + # Similarly we need _a_ file for task_sdk too + mkdir -p ./task_sdk/src/airflow/sdk/ + touch ./task_sdk/src/airflow/__init__.py + + trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT # When installing from sources - we always use `--editable` mode - installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers" + installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers --editable ./task_sdk" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then @@ -1356,6 +1361,7 @@ RUN bash /scripts/docker/install_pipx_tools.sh # So in case pyproject.toml changes we can install latest dependencies required. COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml +COPY task_sdk/pyproject.toml ${AIRFLOW_SOURCES}/task_sdk/pyproject.toml COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/ COPY generated/* ${AIRFLOW_SOURCES}/generated/ COPY constraints/* ${AIRFLOW_SOURCES}/constraints/ diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 9feb9a5b195e6..9cef5d6ccd15e 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -666,3 +666,26 @@ properties of the dependencies. This is done by the ``export-dependency-informat Next step: Follow the `Advanced Breeze topics <10_advanced_breeze_topics.rst>`_ to learn more about Breeze internals. + +Preparing airflow Task SDK packages +""""""""""""""""""""""""""""""""""" + +You can prepare airflow packages using Breeze: + +.. code-block:: bash + + breeze release-management prepare-task-sdk-package + +This prepares airflow Task SDK .whl package in the dist folder. + +Again, you can specify optional ``--package-format`` flag to build selected formats of the Task SDK packages, +default is to build ``both`` type of packages ``sdist`` and ``wheel``. + +.. code-block:: bash + + breeze release-management prepare-task-sdk-package --package-format=wheel + +.. image:: ./images/output_release-management_prepare-task-sdk-package.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-airflow-package.svg + :width: 100% + :alt: Breeze release-management prepare-task-sdk-package diff --git a/dev/breeze/doc/images/output_release-management.svg b/dev/breeze/doc/images/output_release-management.svg index 3392f33e92dfe..2798700d8e26e 100644 --- a/dev/breeze/doc/images/output_release-management.svg +++ b/dev/breeze/doc/images/output_release-management.svg @@ -1,4 +1,4 @@ - + │generate-constraints    Generates pinned constraint files with all extras from pyproject.toml in parallel.        │ │update-constraints      Update released constraints with manual changes.                                          │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│prepare-task-sdk-package                Prepare sdist/whl package of Airflow Task SDK.                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management.txt b/dev/breeze/doc/images/output_release-management.txt index 4655263b2bcfe..a186de953f690 100644 --- a/dev/breeze/doc/images/output_release-management.txt +++ b/dev/breeze/doc/images/output_release-management.txt @@ -1 +1 @@ -46d2a272002ee4a7dfefba98127d1680 +dbd26a6fc8d708bcbbe450896906bef6 diff --git a/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg b/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg new file mode 100644 index 0000000000000..63bb39a5189e8 --- /dev/null +++ b/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management prepare-task-sdk-package + + + + + + + + + + +Usage:breeze release-management prepare-task-sdk-package[OPTIONS] + +Prepare sdist/whl package of Airflow Task SDK. + +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--use-local-hatchUse local hatch instead of docker to build the package. You need to have hatch installed.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.txt b/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.txt new file mode 100644 index 0000000000000..f20d3a31463fa --- /dev/null +++ b/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.txt @@ -0,0 +1 @@ +adc65fd78e7040d42660aea2289d0f96 diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 71feee5875aa1..a5f797e9cf211 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -190,16 +190,16 @@ │release-management:prepare-airflow-tarball | release-management:prepare-helm-chart-package |           │ │release-management:prepare-helm-chart-tarball | release-management:prepare-provider-documentation |    │ │release-management:prepare-provider-packages | release-management:prepare-python-client |              │ -│release-management:publish-docs | release-management:release-prod-images |                             │ -│release-management:start-rc-process | release-management:start-release |                               │ -│release-management:tag-providers | release-management:update-constraints |                             │ -│release-management:verify-provider-packages | sbom | sbom:build-all-airflow-images |                   │ -│sbom:export-dependency-information | sbom:generate-providers-requirements |                            │ -│sbom:update-sbom-information | setup | setup:autocomplete | setup:check-all-params-in-groups |         │ -│setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts | │ -│setup:version | shell | start-airflow | static-checks | testing | testing:db-tests |                   │ -│testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:non-db-tests | │ -│testing:tests)                                                                                         │ +│release-management:prepare-task-sdk-package | release-management:publish-docs |                        │ +│release-management:release-prod-images | release-management:start-rc-process |                         │ +│release-management:start-release | release-management:tag-providers |                                  │ +│release-management:update-constraints | release-management:verify-provider-packages | sbom |           │ +│sbom:build-all-airflow-images | sbom:export-dependency-information |                                   │ +│sbom:generate-providers-requirements | sbom:update-sbom-information | setup | setup:autocomplete |     │ +│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images | setup:self-upgrade │ +│| setup:synchronize-local-mounts | setup:version | shell | start-airflow | static-checks | testing |   │ +│testing:db-tests | testing:docker-compose-tests | testing:helm-tests | testing:integration-tests |     │ +│testing:non-db-tests | testing:tests)                                                                  │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--verbose-vPrint verbose information about performed steps.│ diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt index 098640fbe5bb8..d2cab78ff8c5b 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt @@ -1 +1 @@ -a6a4664c4cf6fdf2bc51ffaa9c0e8177 +852bdb14696f768b8a22551ba88bf061 diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg index 83018c3d8d39f..c107a1843b4e5 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg @@ -204,16 +204,16 @@ │release-management:prepare-airflow-tarball | release-management:prepare-helm-chart-package |        │ │release-management:prepare-helm-chart-tarball | release-management:prepare-provider-documentation | │ │release-management:prepare-provider-packages | release-management:prepare-python-client |           │ -│release-management:publish-docs | release-management:release-prod-images |                          │ -│release-management:start-rc-process | release-management:start-release |                            │ -│release-management:tag-providers | release-management:update-constraints |                          │ -│release-management:verify-provider-packages | sbom | sbom:build-all-airflow-images |                │ -│sbom:export-dependency-information | sbom:generate-providers-requirements |                         │ -│sbom:update-sbom-information | setup | setup:autocomplete | setup:check-all-params-in-groups |      │ -│setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts│ -│| setup:version | shell | start-airflow | static-checks | testing | testing:db-tests |              │ -│testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:non-db-tests│ -│| testing:tests)                                                                                    │ +│release-management:prepare-task-sdk-package | release-management:publish-docs |                     │ +│release-management:release-prod-images | release-management:start-rc-process |                      │ +│release-management:start-release | release-management:tag-providers |                               │ +│release-management:update-constraints | release-management:verify-provider-packages | sbom |        │ +│sbom:build-all-airflow-images | sbom:export-dependency-information |                                │ +│sbom:generate-providers-requirements | sbom:update-sbom-information | setup | setup:autocomplete |  │ +│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images |                 │ +│setup:self-upgrade | setup:synchronize-local-mounts | setup:version | shell | start-airflow |       │ +│static-checks | testing | testing:db-tests | testing:docker-compose-tests | testing:helm-tests |    │ +│testing:integration-tests | testing:non-db-tests | testing:tests)                                   │ │--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ │together with --command flag or --force.                                                            │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt index b0b5b55ddd286..751b0329d71fc 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt @@ -1 +1 @@ -4a736bf190984799f73b3bf85dc62c4c +326695396d27b77b860202ebe9267746 diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 1e23fe528b6d3..19b93c0ca15ea 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -654,8 +654,8 @@ │(All | Default | API | Always | BranchExternalPython |  │ │BranchPythonVenv | CLI | Core | ExternalPython |        │ │Operators | Other | PlainAsserts | Providers |          │ -│PythonVenv | Serialization | WWW | All-Postgres |       │ -│All-MySQL | All-Quarantined)                            │ +│PythonVenv | Serialization | TaskSDK | WWW |            │ +│All-Postgres | All-MySQL | All-Quarantined)             │ │[default: Default]                                      │ │--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It  │ │can also be version (to install from PyPI), `none`,     │ diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index 51aa99ed73ca6..e09e77274a041 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -94eabd0345df65038958fc4842a0ff4b +10bd351167267ce16d35b96898a64b74 diff --git a/dev/breeze/doc/images/output_testing_db-tests.svg b/dev/breeze/doc/images/output_testing_db-tests.svg index 708665af52de3..d9d92ed40f050 100644 --- a/dev/breeze/doc/images/output_testing_db-tests.svg +++ b/dev/breeze/doc/images/output_testing_db-tests.svg @@ -410,15 +410,15 @@ │--parallel-test-typesSpace separated list of test types used for testing in parallel                   │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | WWW)                                                              │ +│Serialization | TaskSDK | WWW)                                                    │ │[default: API Always BranchExternalPython BranchPythonVenv CLI Core ExternalPython│ │Operators Other PlainAsserts Providers[-amazon,google] Providers[amazon]          │ -│Providers[google] PythonVenv Serialization WWW]                                   │ +│Providers[google] PythonVenv Serialization TaskSDK WWW]                           │ │--database-isolationRun airflow in database isolation mode.│ │--excluded-parallel-test-typesSpace separated list of test types that will be excluded from parallel tes runs.  │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | WWW)                                                              │ +│Serialization | TaskSDK | WWW)                                                    │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Test options ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--test-timeoutTest timeout in seconds. Set the pytest setup, execution and teardown timeouts to this     │ diff --git a/dev/breeze/doc/images/output_testing_db-tests.txt b/dev/breeze/doc/images/output_testing_db-tests.txt index 6027a2d1e666a..6245a387214f2 100644 --- a/dev/breeze/doc/images/output_testing_db-tests.txt +++ b/dev/breeze/doc/images/output_testing_db-tests.txt @@ -1 +1 @@ -7b406b63cd4a75aba6ac38b8c0b7431c +690396dbea7c9b6e018704e1ee7f727d diff --git a/dev/breeze/doc/images/output_testing_non-db-tests.svg b/dev/breeze/doc/images/output_testing_non-db-tests.svg index da43621dd6931..daf8061b13e40 100644 --- a/dev/breeze/doc/images/output_testing_non-db-tests.svg +++ b/dev/breeze/doc/images/output_testing_non-db-tests.svg @@ -376,14 +376,14 @@ │--parallel-test-typesSpace separated list of test types used for testing in parallel                   │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | WWW)                                                              │ +│Serialization | TaskSDK | WWW)                                                    │ │[default: API Always BranchExternalPython BranchPythonVenv CLI Core ExternalPython│ │Operators Other PlainAsserts Providers[-amazon,google] Providers[amazon]          │ -│Providers[google] PythonVenv Serialization WWW]                                   │ +│Providers[google] PythonVenv Serialization TaskSDK WWW]                           │ │--excluded-parallel-test-typesSpace separated list of test types that will be excluded from parallel tes runs.  │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | WWW)                                                              │ +│Serialization | TaskSDK | WWW)                                                    │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Test options ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--test-timeoutTest timeout in seconds. Set the pytest setup, execution and teardown timeouts to this     │ diff --git a/dev/breeze/doc/images/output_testing_non-db-tests.txt b/dev/breeze/doc/images/output_testing_non-db-tests.txt index 1080b5a377717..122c3b1acd145 100644 --- a/dev/breeze/doc/images/output_testing_non-db-tests.txt +++ b/dev/breeze/doc/images/output_testing_non-db-tests.txt @@ -1 +1 @@ -2cc222da8b9f31b93b527220b76b48a2 +dbecb30a3e03c7dffd09d46c16687e62 diff --git a/dev/breeze/doc/images/output_testing_tests.svg b/dev/breeze/doc/images/output_testing_tests.svg index c20e2ef16b243..8c8c881915a31 100644 --- a/dev/breeze/doc/images/output_testing_tests.svg +++ b/dev/breeze/doc/images/output_testing_tests.svg @@ -466,19 +466,19 @@ │`Providers[-amazon,google]`                                                       │ │(All | Default | API | Always | BranchExternalPython | BranchPythonVenv | CLI |   │ │Core | ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv │ -│| Serialization | WWW | All-Postgres | All-MySQL | All-Quarantined)               │ +│| Serialization | TaskSDK | WWW | All-Postgres | All-MySQL | All-Quarantined)     │ │[default: Default]                                                                │ │--parallel-test-typesSpace separated list of test types used for testing in parallel                   │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | WWW)                                                              │ +│Serialization | TaskSDK | WWW)                                                    │ │[default: API Always BranchExternalPython BranchPythonVenv CLI Core ExternalPython│ │Operators Other PlainAsserts Providers[-amazon,google] Providers[amazon]          │ -│Providers[google] PythonVenv Serialization WWW]                                   │ +│Providers[google] PythonVenv Serialization TaskSDK WWW]                           │ │--excluded-parallel-test-typesSpace separated list of test types that will be excluded from parallel tes runs.  │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | WWW)                                                              │ +│Serialization | TaskSDK | WWW)                                                    │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Test options ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--test-timeoutTest timeout in seconds. Set the pytest setup, execution and teardown timeouts to this   │ diff --git a/dev/breeze/doc/images/output_testing_tests.txt b/dev/breeze/doc/images/output_testing_tests.txt index d19f6cf5abdae..ce894444f7f3b 100644 --- a/dev/breeze/doc/images/output_testing_tests.txt +++ b/dev/breeze/doc/images/output_testing_tests.txt @@ -1 +1 @@ -15002aa129ce25039921f800fb1cf744 +c131a2a8ab980041a1a0f5e91fe58ea8 diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 95ecf30144ab5..a56947f026f04 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -338,10 +338,16 @@ def from_wheel(cls, filepath: Path) -> DistributionPackageInfo: @classmethod def dist_packages( - cls, *, package_format: str, dist_directory: Path, build_type: Literal["airflow", "providers"] + cls, + *, + package_format: str, + dist_directory: Path, + build_type: Literal["airflow", "providers", "task-sdk"], ) -> tuple[DistributionPackageInfo, ...]: if build_type == "airflow": default_glob_pattern = "apache[_-]airflow-[0-9]" + elif build_type == "task-sdk": + default_glob_pattern = "apache[_-]airflow[_-]task[_-]sdk" else: default_glob_pattern = "apache[_-]airflow[_-]providers" dists_info = [] @@ -549,6 +555,105 @@ def prepare_airflow_packages( get_console().print("[success]Successfully prepared Airflow packages") +TASK_SDK_DIR_PATH = AIRFLOW_SOURCES_ROOT / "task_sdk" +TASK_SDK_DIST_DIR_PATH = TASK_SDK_DIR_PATH / "dist" + + +@release_management.command( + name="prepare-task-sdk-package", + help="Prepare sdist/whl package of Airflow Task SDK.", +) +@option_package_format +@option_use_local_hatch +@option_verbose +@option_dry_run +def prepare_airflow_task_sdk_packages( + package_format: str, + use_local_hatch: bool, +): + check_python_version() + perform_environment_checks() + fix_ownership_using_docker() + cleanup_python_generated_files() + + def _build_package_with_hatch(package_format: str): + command = [ + "hatch", + "build", + "-c", + ] + if package_format == "sdist" or package_format == "both": + command += ["-t", "sdist"] + if package_format == "wheel" or package_format == "both": + command += ["-t", "wheel"] + env_copy = os.environ.copy() + run_command( + cmd=command, + cwd=TASK_SDK_DIR_PATH, + env=env_copy, + check=True, + ) + shutil.copytree(TASK_SDK_DIST_DIR_PATH, DIST_DIR, dirs_exist_ok=True) + + def _build_package_with_docker(package_format: str): + _build_local_build_image() + command = "hatch build -c " + if package_format == "sdist" or package_format == "both": + command += "-t sdist " + if package_format == "wheel" or package_format == "both": + command += "-t wheel " + container_id = f"airflow-task-sdk-build-{random.getrandbits(64):08x}" + result = run_command( + cmd=[ + "docker", + "run", + "--name", + container_id, + "-t", + "-e", + "HOME=/opt/airflow/files/home", + "-e", + "GITHUB_ACTIONS", + "-w", + "/opt/airflow/task_sdk", + AIRFLOW_BUILD_IMAGE_TAG, + "bash", + "-c", + command, + ], + check=False, + ) + if result.returncode != 0: + get_console().print("[error]Error preparing Airflow Task SDK[/]") + fix_ownership_using_docker() + sys.exit(result.returncode) + DIST_DIR.mkdir(parents=True, exist_ok=True) + get_console().print() + # Copy all files in the dist directory in container to the host dist directory (note '/.' in SRC) + run_command(["docker", "cp", f"{container_id}:/opt/airflow/task_sdk/dist/.", "./dist"], check=True) + run_command(["docker", "rm", "--force", container_id], check=False, stdout=DEVNULL, stderr=DEVNULL) + + if use_local_hatch: + _build_package_with_hatch( + package_format=package_format, + ) + get_console().print("[info]Checking if sdist packages can be built into wheels[/]") + packages = DistributionPackageInfo.dist_packages( + package_format=package_format, dist_directory=DIST_DIR, build_type="task-sdk" + ) + get_console().print() + _check_sdist_to_wheel_dists(packages) + get_console().print("\n[info]Packages available in dist:[/]\n") + for dist_info in packages: + get_console().print(str(dist_info)) + get_console().print() + else: + _build_package_with_docker( + package_format=package_format, + ) + get_console().print("[success]Successfully prepared Airflow Task SDK packages") + + def provider_action_summary(description: str, message_type: MessageType, packages: list[str]): if packages: get_console().print(f"{description}: {len(packages)}\n") diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index 6030848444b34..b232666aba7e5 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -52,6 +52,12 @@ ], } +RELEASE_AIRFLOW_TASK_SDK_COMMANDS: dict[str, str | list[str]] = { + "name": "Airflow Task SDK release commands", + "commands": [ + "prepare-task-sdk-package", + ], +} RELEASE_OTHER_COMMANDS: dict[str, str | list[str]] = { "name": "Other release commands", @@ -83,6 +89,15 @@ ], } ], + "breeze release-management prepare-task-sdk-package": [ + { + "name": "Package flags", + "options": [ + "--package-format", + "--use-local-hatch", + ], + } + ], "breeze release-management prepare-helm-chart-tarball": [ { "name": "Package flags", diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 64bf8112112ab..3879c121433f5 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -185,6 +185,7 @@ class SelectiveUnitTestTypes(Enum): PLAIN_ASSERTS = "PlainAsserts" PROVIDERS = "Providers" PYTHON_VENV = "PythonVenv" + TASK_SDK = "TaskSDK" WWW = "WWW" @@ -218,6 +219,23 @@ def all_helm_test_packages() -> list[str]: *all_helm_test_packages(), ] + +@cache +def all_task_sdk_test_packages() -> list[str]: + return sorted( + [ + candidate.name + for candidate in (AIRFLOW_SOURCES_ROOT / "task_sdk" / "tests").iterdir() + if candidate.is_dir() and candidate.name != "__pycache__" + ] + ) + + +ALLOWED_TASK_SDK_TEST_PACKAGES = [ + "all", + *all_task_sdk_test_packages(), +] + ALLOWED_PACKAGE_FORMATS = ["wheel", "sdist", "both"] ALLOWED_INSTALLATION_PACKAGE_FORMATS = ["wheel", "sdist"] ALLOWED_INSTALLATION_METHODS = [".", "apache-airflow"] diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 618d064e0e41f..164a1583e1fea 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -93,6 +93,7 @@ ("hooks", "/opt/airflow/hooks"), ("logs", "/root/airflow/logs"), ("providers", "/opt/airflow/providers"), + ("task_sdk", "/opt/airflow/task_sdk"), ("pyproject.toml", "/opt/airflow/pyproject.toml"), ("scripts", "/opt/airflow/scripts"), ("scripts/docker/entrypoint_ci.sh", "/entrypoint"), diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 23fe5b6e1965e..acde9d0cc4d42 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -169,6 +169,7 @@ def get_excluded_provider_args(python_version: str) -> list[str]: "tests/serialization", ], "System": ["tests/system"], + "TaskSDK": ["task_sdk/tests"], "WWW": [ "tests/www", ], diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 18458af5f8a23..0cb69a194713d 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -87,12 +87,12 @@ "API Always BranchExternalPython BranchPythonVenv " "CLI Core ExternalPython Operators Other PlainAsserts " "Providers[-amazon,google] Providers[amazon] Providers[google] " - "PythonVenv Serialization WWW" + "PythonVenv Serialization TaskSDK WWW" ) ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS = ( "API Always BranchExternalPython BranchPythonVenv CLI Core " - "ExternalPython Operators Other PlainAsserts PythonVenv Serialization WWW" + "ExternalPython Operators Other PlainAsserts PythonVenv Serialization TaskSDK WWW" ) ALL_PROVIDERS_SELECTIVE_TEST_TYPES = "Providers[-amazon,google] Providers[amazon] Providers[google]" @@ -112,6 +112,7 @@ class FileGroupForCi(Enum): LEGACY_WWW_FILES = "legacy_www_files" SYSTEM_TEST_FILES = "system_tests" KUBERNETES_FILES = "kubernetes_files" + TASK_SDK_FILES = "task_sdk_files" ALL_PYTHON_FILES = "all_python_files" ALL_SOURCE_FILES = "all_sources_for_tests" ALL_AIRFLOW_PYTHON_FILES = "all_airflow_python_files" @@ -227,6 +228,8 @@ def __hash__(self): r"^chart", r"^providers/src/", r"^providers/tests/", + r"^task_sdk/src/", + r"^task_sdk/tests/", r"^tests", r"^kubernetes_tests", ], @@ -243,6 +246,10 @@ def __hash__(self): r"^tests/utils/", r"^dev/tests_common/.*\.py$", ], + FileGroupForCi.TASK_SDK_FILES: [ + r"^task_sdk/src/airflow/sdk/.*\.py$", + r"^task_sdk/tests/.*\.py$", + ], } ) @@ -258,6 +265,8 @@ def __hash__(self): r"^providers/tests/.*", r"^providers/tests/system/.*", r"^tests/dags/test_imports.py", + r"^task_sdk/src/airflow/sdk/.*\.py$", + r"^task_sdk/tests/.*\.py$", ] } ) @@ -296,6 +305,10 @@ def __hash__(self): r"^airflow/serialization/", r"^tests/serialization/", ], + SelectiveUnitTestTypes.TASK_SDK: [ + r"^task_sdk/src/airflow/sdk/", + r"^task_sdk/tests/", + ], SelectiveUnitTestTypes.PYTHON_VENV: PYTHON_OPERATOR_FILES, SelectiveUnitTestTypes.BRANCH_PYTHON_VENV: PYTHON_OPERATOR_FILES, SelectiveUnitTestTypes.EXTERNAL_PYTHON: PYTHON_OPERATOR_FILES, @@ -700,6 +713,10 @@ def run_amazon_tests(self) -> bool: or "Providers" in self.parallel_test_types_list_as_string.split(" ") ) + @cached_property + def run_task_sdk_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.TASK_SDK_FILES) + @cached_property def run_kubernetes_tests(self) -> bool: return self._should_be_run(FileGroupForCi.KUBERNETES_FILES) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index be3922d461963..6d031f70b875a 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1140,7 +1140,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "separate-test-types-list-as-string": "API Always BranchExternalPython BranchPythonVenv " "CLI Core ExternalPython Operators Other PlainAsserts " + LIST_OF_ALL_PROVIDER_TESTS - + " PythonVenv Serialization WWW", + + " PythonVenv Serialization TaskSDK WWW", "needs-mypy": "true", "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", }, @@ -1171,10 +1171,10 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "API Always BranchExternalPython " "BranchPythonVenv CLI Core ExternalPython Operators Other PlainAsserts " - "PythonVenv Serialization WWW", + "PythonVenv Serialization TaskSDK WWW", "separate-test-types-list-as-string": "API Always BranchExternalPython " "BranchPythonVenv CLI Core ExternalPython Operators Other PlainAsserts " - "PythonVenv Serialization WWW", + "PythonVenv Serialization TaskSDK WWW", "needs-mypy": "true", "mypy-folders": "['airflow', 'docs', 'dev']", }, @@ -1300,7 +1300,7 @@ def test_expected_output_full_tests_needed( "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", "parallel-test-types-list-as-string": "API Always BranchExternalPython BranchPythonVenv " - "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization WWW", + "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization TaskSDK WWW", "needs-mypy": "true", "mypy-folders": "['airflow']", }, @@ -1536,7 +1536,7 @@ def test_expected_output_pull_request_target( "docs-list-as-string": "apache-airflow docker-stack", "upgrade-to-newer-dependencies": "true", "parallel-test-types-list-as-string": "API Always BranchExternalPython BranchPythonVenv " - "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization WWW", + "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization TaskSDK WWW", "needs-mypy": "true", "mypy-folders": "['airflow', 'docs', 'dev']", }, diff --git a/pyproject.toml b/pyproject.toml index 844e1b6d3ffa6..6fafa7e99b06c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -342,7 +342,7 @@ section-order = [ # Make sure we put the "dev" imports at the end, not as a third-party module [tool.ruff.lint.isort.sections] -testing = ["dev", "providers.tests"] +testing = ["dev", "providers.tests", "task_sdk.tests"] [tool.ruff.lint.extend-per-file-ignores] "airflow/__init__.py" = ["F401", "TCH004"] @@ -626,12 +626,14 @@ ignore_errors = true [tool.uv] dev-dependencies = [ "local-providers", + "apache-airflow-task-sdk" ] [tool.uv.sources] # These names must match the names as defined in the pyproject.toml of the workspace items, # *not* the workspace folder paths local-providers = { workspace = true } +apache-airflow-task-sdk = { workspace = true } [tool.uv.workspace] -members = ["providers"] +members = ["providers", "task_sdk"] diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 66a4736c5fd8d..f0bce46894164 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -88,6 +88,9 @@ services: - type: bind source: ../../../providers target: /opt/airflow/providers + - type: bind + source: ../../../task_sdk + target: /opt/airflow/task_sdk - type: bind source: ../../../pyproject.toml target: /opt/airflow/pyproject.toml diff --git a/scripts/ci/kubernetes/k8s_requirements.txt b/scripts/ci/kubernetes/k8s_requirements.txt index 1e587051f5845..e04ef56412794 100644 --- a/scripts/ci/kubernetes/k8s_requirements.txt +++ b/scripts/ci/kubernetes/k8s_requirements.txt @@ -1,3 +1,4 @@ --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.9.txt -e .[devel-devscripts,devel-tests,cncf.kubernetes] -e ./providers +-e ./task_sdk diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 324bca7ccfe09..769279c2cb0f4 100644 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -48,13 +48,18 @@ function install_airflow() { # Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then - # We need _a _ file in there otherwise the editable install doesn't include anything in the .pth file + # We need _a_ file in there otherwise the editable install doesn't include anything in the .pth file mkdir -p ./providers/src/airflow/providers/ touch ./providers/src/airflow/providers/__init__.py - trap 'rm -f ./providers/src/airflow/providers/__init__.py 2>/dev/null' EXIT + + # Similarly we need _a_ file for task_sdk too + mkdir -p ./task_sdk/src/airflow/sdk/ + touch ./task_sdk/src/airflow/__init__.py + + trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT # When installing from sources - we always use `--editable` mode - installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers" + installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers --editable ./task_sdk" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then diff --git a/task_sdk/README.md b/task_sdk/README.md new file mode 100644 index 0000000000000..ef14affc68c62 --- /dev/null +++ b/task_sdk/README.md @@ -0,0 +1,18 @@ + diff --git a/task_sdk/pyproject.toml b/task_sdk/pyproject.toml new file mode 100644 index 0000000000000..149a9731ce994 --- /dev/null +++ b/task_sdk/pyproject.toml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[project] +name = "apache-airflow-task-sdk" +version = "0.1.0.dev0" +description = "Python Task SDK for Apache Airflow DAG Authors" +#readme = "README.md" +requires-python = ">=3.9, <3.13" +dependencies = [] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/airflow"] + +[tool.ruff] +extend = "../pyproject.toml" +src = ["src"] + +[tool.ruff.lint.per-file-ignores] + +# Ignore Doc rules et al for anything outside of tests +"!src/*" = ["D", "TID253", "S101", "TRY002"] diff --git a/task_sdk/src/airflow/sdk/__init__.py b/task_sdk/src/airflow/sdk/__init__.py new file mode 100644 index 0000000000000..2a3e01b64bc4b --- /dev/null +++ b/task_sdk/src/airflow/sdk/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +def hello() -> str: + return "Hello from task-sdk!" diff --git a/task_sdk/src/airflow/sdk/py.typed b/task_sdk/src/airflow/sdk/py.typed new file mode 100644 index 0000000000000..445548d86368f --- /dev/null +++ b/task_sdk/src/airflow/sdk/py.typed @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Marker file for PEP 561. This package uses inline types. diff --git a/task_sdk/tests/__init__.py b/task_sdk/tests/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/task_sdk/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/task_sdk/tests/conftest.py b/task_sdk/tests/conftest.py new file mode 100644 index 0000000000000..209fad46d7d62 --- /dev/null +++ b/task_sdk/tests/conftest.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +pytest_plugins = "dev.tests_common.pytest_plugin" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + config.inicfg["airflow_deprecations_ignore"] = () diff --git a/task_sdk/tests/test_hello.py b/task_sdk/tests/test_hello.py new file mode 100644 index 0000000000000..62cfdc069ca05 --- /dev/null +++ b/task_sdk/tests/test_hello.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.sdk import hello + + +def test_hello(): + assert hello() == "Hello from task-sdk!" From 349cbece85a20ccfe48d3ff6eeb66b8cc60e2ef0 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sat, 12 Oct 2024 17:33:58 +0100 Subject: [PATCH 083/199] Upgrade Trove classifier to `2024.10.12` (#42961) Test was failing: https://github.com/apache/airflow/actions/runs/11306055404/job/31446236535 ``` pre-commit hook(s) made changes. If you are seeing this message in CI, reproduce locally with: `pre-commit run --all-files`. To run `pre-commit` as part of git workflow, use `pre-commit install`. All changes made by hooks: diff --git a/pyproject.toml b/pyproject.toml index 844e1b6..5e9f587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "pluggy==1.5.0", "smmap==5.0.1", "tomli==2.0.2; python_version < '3.11'", - "trove-classifiers==2024.10.11", + "trove-classifiers==2024.10.12", ] build-backend = "hatchling.build" ``` --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6fafa7e99b06c..6edf4e5c71175 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "pluggy==1.5.0", "smmap==5.0.1", "tomli==2.0.2; python_version < '3.11'", - "trove-classifiers==2024.10.11", + "trove-classifiers==2024.10.12", ] build-backend = "hatchling.build" From e8dd5cdf6038719c41961dbce1b8b106e42b241a Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sat, 12 Oct 2024 22:55:26 +0100 Subject: [PATCH 084/199] Add missed brackets for our dev script for Spell checks (#42965) A nit PR while I was looking at our doc builds --- dev/breeze/src/airflow_breeze/utils/spelling_checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/spelling_checks.py b/dev/breeze/src/airflow_breeze/utils/spelling_checks.py index 1dd9d7ec8cb2b..ee4c37e338f44 100644 --- a/dev/breeze/src/airflow_breeze/utils/spelling_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/spelling_checks.py @@ -167,9 +167,9 @@ def display_spelling_error_summary(spelling_errors: dict[str, list[SpellingError """ console.print(msg) console.print() - console.print + console.print() console.print("[red]" + "#" * 30 + " End docs build errors summary " + "#" * 30 + "[/]") - console.print + console.print() def _display_error(error: SpellingError): From 11ed8aaa931050b080ce35504f4f2fcf93728f66 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 13 Oct 2024 00:17:37 +0200 Subject: [PATCH 085/199] FIX: Don't raise a warning in ExecutorSafeguard when execute is called from an extended operator (#42849) * refactor: Don't raise a warning when execute is called from an extended operator, as this should always be allowed. * refactored: Fixed import of test_utils in test_dag_run --------- Co-authored-by: David Blain --- airflow/models/baseoperator.py | 11 ++++++++++- tests/models/test_baseoperatormeta.py | 24 +++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 7b5c8e7c4f8c4..0c3d119be1983 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -34,6 +34,7 @@ import warnings from datetime import datetime, timedelta from functools import total_ordering, wraps +from threading import local from types import FunctionType from typing import ( TYPE_CHECKING, @@ -392,6 +393,8 @@ class ExecutorSafeguard: """ test_mode = conf.getboolean("core", "unit_test_mode") + _sentinel = local() + _sentinel.callers = {} @classmethod def decorator(cls, func): @@ -399,7 +402,13 @@ def decorator(cls, func): def wrapper(self, *args, **kwargs): from airflow.decorators.base import DecoratedOperator - sentinel = kwargs.pop(f"{self.__class__.__name__}__sentinel", None) + sentinel_key = f"{self.__class__.__name__}__sentinel" + sentinel = kwargs.pop(sentinel_key, None) + + if sentinel: + cls._sentinel.callers[sentinel_key] = sentinel + else: + sentinel = cls._sentinel.callers.pop(f"{func.__qualname__.split('.')[0]}__sentinel", None) if not cls.test_mode and not sentinel == _sentinel and not isinstance(self, DecoratedOperator): message = f"{self.__class__.__name__}.{func.__name__} cannot be called outside TaskInstance!" diff --git a/tests/models/test_baseoperatormeta.py b/tests/models/test_baseoperatormeta.py index 6c6567b23899e..5244e86b2c386 100644 --- a/tests/models/test_baseoperatormeta.py +++ b/tests/models/test_baseoperatormeta.py @@ -40,6 +40,11 @@ def execute(self, context: Context) -> Any: return f"Hello {self.owner}!" +class ExtendedHelloWorldOperator(HelloWorldOperator): + def execute(self, context: Context) -> Any: + return super().execute(context) + + class TestExecutorSafeguard: def setup_method(self): ExecutorSafeguard.test_mode = False @@ -49,12 +54,29 @@ def teardown_method(self, method): @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test - def test_executor_when_classic_operator_called_from_dag(self, dag_maker): + @patch.object(HelloWorldOperator, "log") + def test_executor_when_classic_operator_called_from_dag(self, mock_log, dag_maker): with dag_maker() as dag: HelloWorldOperator(task_id="hello_operator") dag_run = dag.test() assert dag_run.state == DagRunState.SUCCESS + mock_log.warning.assert_not_called() + + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode + @pytest.mark.db_test + @patch.object(HelloWorldOperator, "log") + def test_executor_when_extended_classic_operator_called_from_dag( + self, + mock_log, + dag_maker, + ): + with dag_maker() as dag: + ExtendedHelloWorldOperator(task_id="hello_operator") + + dag_run = dag.test() + assert dag_run.state == DagRunState.SUCCESS + mock_log.warning.assert_not_called() @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( From 0b1b1505c7f7e76332af9045fb1484b66bfc9387 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 13 Oct 2024 02:31:51 +0200 Subject: [PATCH 086/199] Upgrade Helm Chart dependencies to latest released (#42816) From 23cbeef666991e7c98d705fc7bf9f644b37003cc Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU <68415893+jason810496@users.noreply.github.com> Date: Sun, 13 Oct 2024 09:55:16 +0800 Subject: [PATCH 087/199] Fix PythonOperator DAG error when DAG has hyphen in name (#42902) --- airflow/utils/file.py | 2 +- tests/utils/test_file.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/airflow/utils/file.py b/airflow/utils/file.py index 2e39eb7dd7b52..86b7a7891ca8f 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -355,6 +355,6 @@ def get_unique_dag_module_name(file_path: str) -> str: """Return a unique module name in the format unusual_prefix_{sha1 of module's file path}_{original module name}.""" if isinstance(file_path, str): path_hash = hashlib.sha1(file_path.encode("utf-8")).hexdigest() - org_mod_name = Path(file_path).stem + org_mod_name = re2.sub(r"[.-]", "_", Path(file_path).stem) return MODIFIED_DAG_MODULE_NAME.format(path_hash=path_hash, module_name=org_mod_name) raise ValueError("file_path should be a string to generate unique module name") diff --git a/tests/utils/test_file.py b/tests/utils/test_file.py index 2ffb89ec34196..9424b90a92cf4 100644 --- a/tests/utils/test_file.py +++ b/tests/utils/test_file.py @@ -212,3 +212,21 @@ def test_get_modules_from_invalid_file(self): modules = list(file_utils.iter_airflow_imports(file_path)) assert len(modules) == 0 + + +@pytest.mark.parametrize( + "edge_filename, expected_modification", + [ + ("test_dag.py", "unusual_prefix_mocked_path_hash_sha1_test_dag"), + ("test-dag.py", "unusual_prefix_mocked_path_hash_sha1_test_dag"), + ("test-dag-1.py", "unusual_prefix_mocked_path_hash_sha1_test_dag_1"), + ("test-dag_1.py", "unusual_prefix_mocked_path_hash_sha1_test_dag_1"), + ("test-dag.dev.py", "unusual_prefix_mocked_path_hash_sha1_test_dag_dev"), + ("test_dag.prod.py", "unusual_prefix_mocked_path_hash_sha1_test_dag_prod"), + ], +) +def test_get_unique_dag_module_name(edge_filename, expected_modification): + with mock.patch("hashlib.sha1") as mocked_sha1: + mocked_sha1.return_value.hexdigest.return_value = "mocked_path_hash_sha1" + modify_module_name = file_utils.get_unique_dag_module_name(edge_filename) + assert modify_module_name == expected_modification From 0feb1f2c11c10c10eebe93434da4a14a3012323d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 13 Oct 2024 04:46:16 +0200 Subject: [PATCH 088/199] fIX handling removal of dependencies (#42967) There was a problem with CI image builds when cache has been disabled - they still used "pre-cached" main version (unlike PROD builds). This PR synchronizes the behaviour between CI and PROD builds. For removing dependencies, you need to set both: * `disable docker cache` label * incerase DEPENDENCIES_EPOCH_NUMBER in `Dockerfile.ci` Comments and documentation in both places has been updated to reflect it. Since documentation for labels has been updated, Part of this PR is to improve the description of possible labels that could be used during the build. The description grew from a small number of labels to a "wall of text" that was difficult to read. This PR reformats it in the form of table that makes it far easier to see actions that the maintainer can do and what labels should be set for each of them. --- .../workflows/additional-ci-image-checks.yml | 7 ++ .../workflows/additional-prod-image-tests.yml | 6 + .github/workflows/build-images.yml | 4 + .github/workflows/ci-image-build.yml | 5 + .github/workflows/ci.yml | 6 + .github/workflows/finalize-tests.yml | 5 + .github/workflows/prod-image-build.yml | 6 + .github/workflows/prod-image-extra-checks.yml | 6 + .github/workflows/push-image-cache.yml | 6 + Dockerfile.ci | 5 +- dev/breeze/doc/ci/07_debugging.md | 67 ++++------- .../doc/images/output_ci-image_build.svg | 112 +++++++++--------- .../doc/images/output_ci-image_build.txt | 2 +- .../doc/images/output_prod-image_build.txt | 2 +- .../commands/ci_image_commands.py | 4 + .../commands/ci_image_commands_config.py | 1 + .../commands/common_image_options.py | 7 ++ .../commands/production_image_commands.py | 7 +- .../airflow_breeze/params/build_ci_params.py | 1 - .../params/build_prod_params.py | 5 - .../params/common_build_params.py | 5 +- .../airflow_breeze/utils/selective_checks.py | 4 + dev/breeze/tests/test_selective_checks.py | 20 ++++ 23 files changed, 179 insertions(+), 114 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index ae9efdb6b0340..878800324b784 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -64,6 +64,10 @@ on: # yamllint disable-line rule:truthy description: "Docker cache specification to build the image (registry, local, disabled)." required: true type: string + disable-airflow-repo-cache: + description: "Disable airflow repo cache read from main." + required: true + type: string canary-run: description: "Whether this is a canary run (true/false)" required: true @@ -112,6 +116,7 @@ jobs: use-uv: "true" include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} + disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} if: inputs.branch == 'main' # Check that after earlier cache push, breeze command will build quickly @@ -168,3 +173,5 @@ jobs: # use-uv: "true" # upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} # docker-cache: ${{ inputs.docker-cache }} +# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} +# diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 4c9606e1343e6..5ffd2001e0e26 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -48,6 +48,10 @@ on: # yamllint disable-line rule:truthy description: "Docker cache specification to build the image (registry, local, disabled)." required: true type: string + disable-airflow-repo-cache: + description: "Disable airflow repo cache read from main." + required: true + type: string canary-run: description: "Whether to run the canary run (true/false)" required: true @@ -72,6 +76,7 @@ jobs: chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} + disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} if: inputs.default-branch == 'main' && inputs.canary-run == 'true' prod-image-extra-checks-release-branch: @@ -89,6 +94,7 @@ jobs: chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} + disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} if: inputs.default-branch != 'main' && inputs.canary-run == 'true' test-examples-of-prod-image-building: diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 6c6d55d75045e..943b01f8f8916 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -71,6 +71,7 @@ jobs: prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} default-branch: ${{ steps.selective-checks.outputs.default-branch }} + disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }} runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }} @@ -210,6 +211,8 @@ jobs: constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + generate-constraints: name: Generate constraints @@ -256,3 +259,4 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 1c4b31b55a604..b8e2feac1755f 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -95,6 +95,10 @@ on: # yamllint disable-line rule:truthy description: "Docker cache specification to build the image (registry, local, disabled)." required: true type: string + disable-airflow-repo-cache: + description: "Disable airflow repo cache read from main." + required: true + type: string jobs: build-ci-images: strategy: @@ -171,6 +175,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" --python "${{ matrix.python-version }}" --platform "${{ inputs.platform }}" env: DOCKER_CACHE: ${{ inputs.docker-cache }} + DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2267154b03a7a..8a9d716cd8421 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: outputs: image-tag: ${{ github.event.pull_request.head.sha || github.sha }} docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} affected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} @@ -208,6 +209,7 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} wait-for-ci-images: timeout-minutes: 120 @@ -264,6 +266,7 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} @@ -559,6 +562,7 @@ jobs: chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} wait-for-prod-images: timeout-minutes: 80 @@ -615,6 +619,7 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} canary-run: ${{ needs.build-info.outputs.canary-run }} if: needs.build-info.outputs.prod-image-build == 'true' @@ -670,6 +675,7 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} notify-slack-failure: diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 8b392ba204664..6fae105e0a646 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -64,6 +64,10 @@ on: # yamllint disable-line rule:truthy description: "Docker cache specification to build the image (registry, local, disabled)." required: true type: string + disable-airflow-repo-cache: + description: "Disable airflow repo cache read from main." + required: true + type: string include-success-outputs: description: "Whether to include success outputs (true/false)" required: true @@ -148,6 +152,7 @@ jobs: use-uv: "true" include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} + disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} if: inputs.canary-run == 'true' # push-buildx-cache-to-github-registry-arm: diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 75d9d0054ec78..db80a6ec247ec 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -114,6 +114,10 @@ on: # yamllint disable-line rule:truthy description: "Docker cache specification to build the image (registry, local, disabled)." required: true type: string + disable-airflow-repo-cache: + description: "Disable airflow repo cache read from main." + required: true + type: string jobs: build-prod-packages: @@ -276,6 +280,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" env: PUSH: ${{ inputs.push-image }} DOCKER_CACHE: ${{ inputs.docker-cache }} + DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }} DEBIAN_VERSION: ${{ inputs.debian-version }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} @@ -291,6 +296,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" env: PUSH: ${{ inputs.push-image }} DOCKER_CACHE: ${{ inputs.docker-cache }} + DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }} DEBIAN_VERSION: ${{ inputs.debian-version }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 82d327ba2f16d..bb63faef7b243 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -63,6 +63,10 @@ on: # yamllint disable-line rule:truthy description: "Docker cache specification to build the image (registry, local, disabled)." required: true type: string + disable-airflow-repo-cache: + description: "Disable airflow repo cache read from main." + required: true + type: string jobs: myssql-client-image: uses: ./.github/workflows/prod-image-build.yml @@ -84,6 +88,7 @@ jobs: chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} + disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} pip-image: uses: ./.github/workflows/prod-image-build.yml @@ -107,3 +112,4 @@ jobs: chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} + disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 0dc83a3fd66ea..10a33275ad3f3 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -76,6 +76,10 @@ on: # yamllint disable-line rule:truthy description: "Docker cache specification to build the image (registry, local, disabled)." required: true type: string + disable-airflow-repo-cache: + description: "Disable airflow repo cache read from main." + required: true + type: string jobs: push-ci-image-cache: name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " @@ -100,6 +104,7 @@ jobs: DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} DOCKER_CACHE: ${{ inputs.docker-cache }} + DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} @@ -162,6 +167,7 @@ jobs: DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} DOCKER_CACHE: ${{ inputs.docker-cache }} + DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} diff --git a/Dockerfile.ci b/Dockerfile.ci index 9339e9af6d6f8..464e33f147fb1 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1180,7 +1180,10 @@ SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "n ARG PYTHON_BASE_IMAGE ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" -# By increasing this number we can do force build of all dependencies +# By increasing this number we can do force build of all dependencies. +# NOTE! When you want to make sure dependencies are installed from scratch in your PR after removing +# some dependencies, you also need to set "disable image cache" in your PR to make sure the image is +# not built using the "main" version of those dependencies. ARG DEPENDENCIES_EPOCH_NUMBER="11" # Make sure noninteractive debian install is used and language variables set diff --git a/dev/breeze/doc/ci/07_debugging.md b/dev/breeze/doc/ci/07_debugging.md index 6e6d46584edfa..9e7173ae84721 100644 --- a/dev/breeze/doc/ci/07_debugging.md +++ b/dev/breeze/doc/ci/07_debugging.md @@ -21,11 +21,11 @@ **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* -- [Debugging CI Jobs in Github Actions](#debugging-ci-jobs-in-github-actions) +- [Debugging CI Jobs in Github Actions and changing their behaviour](#debugging-ci-jobs-in-github-actions-and-changing-their-behaviour) -# Debugging CI Jobs in Github Actions +# Debugging CI Jobs in Github Actions and changing their behaviour The CI jobs are notoriously difficult to test, because you can only really see results of it when you run them in CI environment, and the @@ -39,49 +39,28 @@ difficulty is that `Build Images` workflow is `pull-request-target` type, which means that it will always run using the `main` version - no matter what is in your Pull Request. -There are several ways how you can debug the CI jobs when you are -maintainer. +There are several ways how you can debug the CI jobs and modify their +behaviour when you are maintainer. + +When you create the PR you can set one of the labels below, also +in some cases, you need to run the PR as coming from the "apache" +repository rather than from your fork. + +You can also apply the label later and rebase the PR or close/reopen +the PR to apply the label to the PR. + +| Action to perform | Label to set | PR from "apache" repo | +|------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|:---------------------:| +| Run the build with all combinations of all
python, backends, kubernetes etc on PR,
and run all types of tests for all test
groups. | full tests needed | | +| Force to use public runners for the build | use public runners | | +| Debug resources used during the build for
parallel jobs | debug ci resources | | +| Force running PR on latest versions of
python, backends, kubernetes etc. when you
want to save resources and test only latest
versions | latest versions only | | +| Force running PR on minimal (default)
versions of python, backends, kubernetes etc.
in order to save resources and run tests only
for minimum versions | default versions only | | +| Make sure to clean dependency cache
usually when removing dependencies
You also need to increase
`DEPENDENCIES_EPOCH_NUMBER` in `Dockerfile.ci` | disable image cache | | +| Change build images workflows, breeze code or
scripts that are used during image build
so that the scripts can be modified by PR
| | Yes | +| Treat your build as "canary" build - including
updating constraints and pushing "main"
documentation. | | Yes | +| Remove any behaviour specific for the committers
such as using different runners by default. | non committer build | | -- When you want to tests the build with all combinations of all python, - backends etc on regular PR, add `full tests needed` label to the PR. -- When you want to test maintainer PR using public runners, add - `public runners` label to the PR -- When you want to see resources used by the run, add - `debug ci resources` label to the PR -- When you want to test changes to breeze that include changes to how - images are build you should push your PR to `apache` repository not to - your fork. This will run the images as part of the `CI` workflow - rather than using `Build images` workflow and use the same breeze - version for building image and testing -- When you want to test changes to workflows and CI scripts you can set - `all versions` label to the PR or `latest versions only`. - This will make the PR run using "all" versions of - Python, Kubernetes and the DBS. By default - unless you also change - dependencies in `pyproject.toml` or `generated/provider_dependencies.json` - such PRs will only use "default" versions of Python, Kubernetes and - DBs. This is useful when you want to test changes to the CI scripts - are not affected by the versions of Python, Kubernetes and DBs. -- Even if you change dependencies in `pyproject.toml`, or - `generated/provider_dependencies.json`, when you want to test changes to workflows - and CI scripts you can set `default versions only` label to the - This will make the PR run using the default (or latest) versions of - Python and Kubernetes and DBs. This is useful when you want to test - changes to the CI scripts and workflows and you want to use far - less resources than the full tests. -- When you want to test changes to `build-images.yml` workflow you - should push your branch as `main` branch in your local fork. This will - run changed `build-images.yml` workflow as it will be in `main` branch - of your fork -- When you are a committer and you change build images workflow, together - with build scripts, your build might fail because your scripts are used - in `build-images.yml` workflow, but the workflow is run using the `main` - version. Setting `non committer build` label will make your PR run using - the main version of the scripts and the workflow -- When you are a committer want to test how changes in your workflow affect - `canary` run, as maintainer, you should push your PR to `apache` repository - not to your fork and set `canary` label to the PR -- When you are a committer and want to test if the tests are passing if the - image is freshly built without cache, you can set `disable image cache` label. ----- diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index 6dd856c3dc8db..62339f7053924 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -1,4 +1,4 @@ - + │--commit-shaCommit SHA that is used to build the images.(TEXT)│ │--debian-versionDebian version used in Airflow image as base for building images.(bookworm)│ │[default: bookworm]                                              │ -│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ -│--python-imageIf specified this is the base python image used to build the image. Should be     │ -│something like: python:VERSION-slim-bookworm.                                     │ -│(TEXT)                                                                            │ -│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ -│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).│ -│(INTEGER RANGE)                                                     │ -│[default: 300; x>=1]                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting constraint location (for power users) ────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ -│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Choosing dependencies and extras (for power users) ─────────────────────────────────────────────────────────────────╮ -│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Backtracking options ───────────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-timeout-minutesOptional timeout for the build in minutes. Useful to detect `pip`        │ -│backtracking problems.                                                   │ -│(INTEGER)                                                                │ -│--eager-upgrade-additional-requirementsOptional additional requirements to upgrade eagerly to avoid backtracking│ -│(see `breeze ci find-backtracking-candidates`).                          │ -│(TEXT)                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ +│--python-imageIf specified this is the base python image used to build the image. Should be     │ +│something like: python:VERSION-slim-bookworm.                                     │ +│(TEXT)                                                                            │ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ +│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).│ +│(INTEGER RANGE)                                                     │ +│[default: 300; x>=1]                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting constraint location (for power users) ────────────────────────────────────────────────────────────────────╮ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ +│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Choosing dependencies and extras (for power users) ─────────────────────────────────────────────────────────────────╮ +│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Backtracking options ───────────────────────────────────────────────────────────────────────────────────────────────╮ +│--build-timeout-minutesOptional timeout for the build in minutes. Useful to detect `pip`        │ +│backtracking problems.                                                   │ +│(INTEGER)                                                                │ +│--eager-upgrade-additional-requirementsOptional additional requirements to upgrade eagerly to avoid backtracking│ +│(see `breeze ci find-backtracking-candidates`).                          │ +│(TEXT)                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci-image_build.txt b/dev/breeze/doc/images/output_ci-image_build.txt index 905847bedc783..a53f9ce997325 100644 --- a/dev/breeze/doc/images/output_ci-image_build.txt +++ b/dev/breeze/doc/images/output_ci-image_build.txt @@ -1 +1 @@ -686950c27e41fa50cf22abb8c25e496a +b03ca6dcb44ef05f9c27f72318ff9500 diff --git a/dev/breeze/doc/images/output_prod-image_build.txt b/dev/breeze/doc/images/output_prod-image_build.txt index b8131522f128e..7799e6f009ebf 100644 --- a/dev/breeze/doc/images/output_prod-image_build.txt +++ b/dev/breeze/doc/images/output_prod-image_build.txt @@ -1 +1 @@ -3d6fee0b510d006927e45a56aa45d505 +d0214e8e95fcb56c91e0e416690eb24f diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index cc6274775f4dd..3c49e530fcec2 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -41,6 +41,7 @@ option_debian_version, option_dev_apt_command, option_dev_apt_deps, + option_disable_airflow_repo_cache, option_docker_cache, option_image_tag_for_building, option_image_tag_for_pulling, @@ -296,6 +297,7 @@ def get_exitcode(status: int) -> int: @option_debug_resources @option_dev_apt_command @option_dev_apt_deps +@option_disable_airflow_repo_cache @option_docker_cache @option_docker_host @option_dry_run @@ -339,6 +341,7 @@ def build( debug_resources: bool, dev_apt_command: str | None, dev_apt_deps: str | None, + disable_airflow_repo_cache: bool, docker_cache: str, docker_host: str | None, eager_upgrade_additional_requirements: str | None, @@ -413,6 +416,7 @@ def run_build(ci_image_params: BuildCiParams) -> None: debian_version=debian_version, dev_apt_command=dev_apt_command, dev_apt_deps=dev_apt_deps, + disable_airflow_repo_cache=disable_airflow_repo_cache, docker_cache=docker_cache, docker_host=docker_host, eager_upgrade_additional_requirements=eager_upgrade_additional_requirements, diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py index 2c5d4a4068ed6..0fe458c9d6ad2 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -57,6 +57,7 @@ "--additional-pip-install-flags", "--commit-sha", "--debian-version", + "--disable-airflow-repo-cache", "--install-mysql-client-type", "--python-image", "--use-uv", diff --git a/dev/breeze/src/airflow_breeze/commands/common_image_options.py b/dev/breeze/src/airflow_breeze/commands/common_image_options.py index 1b8f9460949e3..e80b2f284d266 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_image_options.py +++ b/dev/breeze/src/airflow_breeze/commands/common_image_options.py @@ -108,6 +108,12 @@ help="Apt dev dependencies to use when building the images.", envvar="DEV_APT_DEPS", ) +option_disable_airflow_repo_cache = click.option( + "--disable-airflow-repo-cache", + help="Disable cache from Airflow repository during building.", + is_flag=True, + envvar="DISABLE_AIRFLOW_REPO_CACHE", +) option_docker_cache = click.option( "-c", "--docker-cache", @@ -115,6 +121,7 @@ default=ALLOWED_BUILD_CACHE[0], show_default=True, type=BetterChoice(ALLOWED_BUILD_CACHE), + envvar="DOCKER_CACHE", ) option_image_tag_for_pulling = click.option( "-t", diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 6e21aabffdff5..d266dd027bdc0 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -36,6 +36,7 @@ option_debian_version, option_dev_apt_command, option_dev_apt_deps, + option_disable_airflow_repo_cache, option_docker_cache, option_image_tag_for_building, option_image_tag_for_pulling, @@ -192,11 +193,6 @@ def prod_image(): @click.option("--disable-mysql-client-installation", help="Do not install MySQL client.", is_flag=True) @click.option("--disable-mssql-client-installation", help="Do not install MsSQl client.", is_flag=True) @click.option("--disable-postgres-client-installation", help="Do not install Postgres client.", is_flag=True) -@click.option( - "--disable-airflow-repo-cache", - help="Disable cache from Airflow repository during building.", - is_flag=True, -) @click.option( "--install-airflow-reference", help="Install Airflow using GitHub tag or branch.", @@ -222,6 +218,7 @@ def prod_image(): @option_debug_resources @option_dev_apt_command @option_dev_apt_deps +@option_disable_airflow_repo_cache @option_docker_cache @option_docker_host @option_dry_run diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index 05179df07b8c4..c38776a3f6144 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -34,7 +34,6 @@ class BuildCiParams(CommonBuildParams): airflow_constraints_mode: str = "constraints-source-providers" airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_extras: str = "devel-ci" - airflow_pre_cached_pip_packages: bool = True force_build: bool = False upgrade_to_newer_dependencies: bool = False upgrade_on_failure: bool = False diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 2533c30d6f3ae..6fa5828b40fa5 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -44,7 +44,6 @@ class BuildProdParams(CommonBuildParams): airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH cleanup_context: bool = False airflow_extras: str = field(default_factory=get_airflow_extras) - disable_airflow_repo_cache: bool = False disable_mssql_client_installation: bool = False disable_mysql_client_installation: bool = False disable_postgres_client_installation: bool = False @@ -186,10 +185,6 @@ def _extra_prod_docker_build_flags(self) -> list[str]: ) return extra_build_flags - @property - def airflow_pre_cached_pip_packages(self) -> str: - return "false" if self.disable_airflow_repo_cache else "true" - @property def install_mssql_client(self) -> str: return "false" if self.disable_mssql_client_installation else "true" diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index ce0f2c0063f50..535ba96480536 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -56,6 +56,7 @@ class CommonBuildParams: commit_sha: str | None = None dev_apt_command: str | None = None dev_apt_deps: str | None = None + disable_airflow_repo_cache: bool = False docker_cache: str = "registry" docker_host: str | None = os.environ.get("DOCKER_HOST") github_actions: str = os.environ.get("GITHUB_ACTIONS", "false") @@ -89,8 +90,8 @@ def image_type(self) -> str: raise NotImplementedError() @property - def airflow_pre_cached_pip_packages(self): - raise NotImplementedError() + def airflow_pre_cached_pip_packages(self) -> str: + return "false" if self.disable_airflow_repo_cache else "true" @property def airflow_base_image_name(self): diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 0cb69a194713d..20bf514379f0b 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1132,6 +1132,10 @@ def docker_cache(self) -> str: def debug_resources(self) -> bool: return DEBUG_CI_RESOURCES_LABEL in self._pr_labels + @cached_property + def disable_airflow_repo_cache(self) -> bool: + return self.docker_cache == "disabled" + @cached_property def helm_test_packages(self) -> str: return json.dumps(all_helm_test_packages()) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 6d031f70b875a..619dc8eec2f99 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2308,6 +2308,26 @@ def test_mypy_matches( ("non committer build",), id="Committer regular PR - forcing non-committer build", ), + pytest.param( + ("README.md",), + { + "docker-cache": "disabled", + "disable-airflow-repo-cache": "true", + }, + "potiuk", + ("disable image cache",), + id="Disabled cache", + ), + pytest.param( + ("README.md",), + { + "docker-cache": "registry", + "disable-airflow-repo-cache": "false", + }, + "potiuk", + (), + id="Standard cache", + ), ], ) def test_pr_labels( From c7db2aa6aa33e24e6518437d1a84140e2efe02d6 Mon Sep 17 00:00:00 2001 From: GPK Date: Sun, 13 Oct 2024 05:36:27 +0100 Subject: [PATCH 089/199] mark test_task_workflow_trigger_success as flaky (#42972) --- tests/triggers/test_external_task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/triggers/test_external_task.py b/tests/triggers/test_external_task.py index ced867c4bd6aa..202eafb0b9e3d 100644 --- a/tests/triggers/test_external_task.py +++ b/tests/triggers/test_external_task.py @@ -36,6 +36,7 @@ class TestWorkflowTrigger: RUN_ID = "external_task_run_id" STATES = ["success", "fail"] + @pytest.mark.flaky(reruns=5) @mock.patch("airflow.triggers.external_task._get_count") @pytest.mark.asyncio async def test_task_workflow_trigger_success(self, mock_get_count): From cdb4852566c77df5fb3c9358d69f182670b2c4ec Mon Sep 17 00:00:00 2001 From: Aryan Khurana <101019909+AryanK1511@users.noreply.github.com> Date: Sun, 13 Oct 2024 06:34:02 -0400 Subject: [PATCH 090/199] Create a User Settings button with light/dark mode toggle as a menu item (#42964) --- airflow/ui/src/layouts/Nav/Nav.tsx | 16 +---- .../ui/src/layouts/Nav/UserSettingsButton.tsx | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 airflow/ui/src/layouts/Nav/UserSettingsButton.tsx diff --git a/airflow/ui/src/layouts/Nav/Nav.tsx b/airflow/ui/src/layouts/Nav/Nav.tsx index 9886b5eb75760..3b47595e8c173 100644 --- a/airflow/ui/src/layouts/Nav/Nav.tsx +++ b/airflow/ui/src/layouts/Nav/Nav.tsx @@ -21,7 +21,6 @@ import { Flex, Icon, Link, - useColorMode, useColorModeValue, VStack, } from "@chakra-ui/react"; @@ -32,9 +31,7 @@ import { FiDatabase, FiGlobe, FiHome, - FiMoon, FiSettings, - FiSun, } from "react-icons/fi"; import { AirflowPin } from "src/assets/AirflowPin"; @@ -42,9 +39,9 @@ import { DagIcon } from "src/assets/DagIcon"; import { DocsButton } from "./DocsButton"; import { NavButton } from "./NavButton"; +import { UserSettingsButton } from "./UserSettingsButton"; export const Nav = () => { - const { colorMode, toggleColorMode } = useColorMode(); const navBg = useColorModeValue("blue.100", "blue.900"); return ( @@ -106,16 +103,7 @@ export const Nav = () => { title="Return to legacy UI" /> - - ) : ( - - ) - } - onClick={toggleColorMode} - /> + ); diff --git a/airflow/ui/src/layouts/Nav/UserSettingsButton.tsx b/airflow/ui/src/layouts/Nav/UserSettingsButton.tsx new file mode 100644 index 0000000000000..c43c17b6d039b --- /dev/null +++ b/airflow/ui/src/layouts/Nav/UserSettingsButton.tsx @@ -0,0 +1,58 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { + IconButton, + Menu, + MenuButton, + useColorMode, + MenuItem, + MenuList, +} from "@chakra-ui/react"; +import { FiMoon, FiSun, FiUser } from "react-icons/fi"; + +import { navButtonProps } from "./navButtonProps"; + +export const UserSettingsButton = () => { + const { colorMode, toggleColorMode } = useColorMode(); + + return ( + + } + {...navButtonProps} + /> + + + {colorMode === "light" ? ( + <> + + Switch to Dark Mode + + ) : ( + <> + + Switch to Light Mode + + )} + + + + ); +}; From f4af0f420615ed86a335d50b21ea56ffe6c380f0 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Sun, 13 Oct 2024 16:05:10 +0530 Subject: [PATCH 091/199] AIP-84 Migrate delete Dag Run endpoint to FastAPI (#42910) * delete dag_run init * delete dag_run init * add delete dag run endpoint * add tests for delete dag run --- .../endpoints/dag_run_endpoint.py | 1 + airflow/api_fastapi/openapi/v1-generated.yaml | 52 +++++++++++++++++++ airflow/api_fastapi/views/public/dag_run.py | 15 ++++++ airflow/ui/openapi-gen/queries/common.ts | 3 ++ airflow/ui/openapi-gen/queries/queries.ts | 43 +++++++++++++++ .../ui/openapi-gen/requests/services.gen.ts | 31 +++++++++++ airflow/ui/openapi-gen/requests/types.gen.ts | 36 +++++++++++++ .../api_fastapi/views/public/test_dag_run.py | 12 +++++ 8 files changed, 193 insertions(+) diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py b/airflow/api_connexion/endpoints/dag_run_endpoint.py index a862b7c969503..74eae13ddd4d0 100644 --- a/airflow/api_connexion/endpoints/dag_run_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py @@ -78,6 +78,7 @@ from airflow.api_connexion.types import APIResponse +@mark_fastapi_migration_done @security.requires_access_dag("DELETE", DagAccessEntity.RUN) @provide_session @action_logging diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index 7debfbb1008af..6c35ca212a2a6 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -679,6 +679,58 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - DagRun + summary: Delete Dag Run + description: Delete a DAG Run entry. + operationId: delete_dag_run + parameters: + - name: dag_id + in: path + required: true + schema: + type: string + title: Dag Id + - name: dag_run_id + in: path + required: true + schema: + type: string + title: Dag Run Id + responses: + '204': + description: Successful Response + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Bad Request + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' components: schemas: ConnectionResponse: diff --git a/airflow/api_fastapi/views/public/dag_run.py b/airflow/api_fastapi/views/public/dag_run.py index d39fb6f2f331c..2f44f03ea6103 100644 --- a/airflow/api_fastapi/views/public/dag_run.py +++ b/airflow/api_fastapi/views/public/dag_run.py @@ -42,3 +42,18 @@ async def get_dag_run( ) return DAGRunResponse.model_validate(dag_run, from_attributes=True) + + +@dag_run_router.delete( + "/{dag_run_id}", status_code=204, responses=create_openapi_http_exception_doc([400, 401, 403, 404]) +) +async def delete_dag_run(dag_id: str, dag_run_id: str, session: Annotated[Session, Depends(get_session)]): + """Delete a DAG Run entry.""" + dag_run = session.scalar(select(DagRun).filter_by(dag_id=dag_id, run_id=dag_run_id)) + + if dag_run is None: + raise HTTPException( + 404, f"The DagRun with dag_id: `{dag_id}` and run_id: `{dag_run_id}` was not found" + ) + + session.delete(dag_run) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index aaff196c0791d..393fad520a679 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -197,3 +197,6 @@ export type ConnectionServiceDeleteConnectionMutationResult = Awaited< export type VariableServiceDeleteVariableMutationResult = Awaited< ReturnType >; +export type DagRunServiceDeleteDagRunMutationResult = Awaited< + ReturnType +>; diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 19bb17b342a84..01e919cce53bf 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -506,3 +506,46 @@ export const useVariableServiceDeleteVariable = < }) as unknown as Promise, ...options, }); +/** + * Delete Dag Run + * Delete a DAG Run entry. + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns void Successful Response + * @throws ApiError + */ +export const useDagRunServiceDeleteDagRun = < + TData = Common.DagRunServiceDeleteDagRunMutationResult, + TError = unknown, + TContext = unknown, +>( + options?: Omit< + UseMutationOptions< + TData, + TError, + { + dagId: string; + dagRunId: string; + }, + TContext + >, + "mutationFn" + >, +) => + useMutation< + TData, + TError, + { + dagId: string; + dagRunId: string; + }, + TContext + >({ + mutationFn: ({ dagId, dagRunId }) => + DagRunService.deleteDagRun({ + dagId, + dagRunId, + }) as unknown as Promise, + ...options, + }); diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 9a126aef25fbc..f0cbc099370f8 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -27,6 +27,8 @@ import type { GetVariableResponse, GetDagRunData, GetDagRunResponse, + DeleteDagRunData, + DeleteDagRunResponse, } from "./types.gen"; export class AssetService { @@ -391,4 +393,33 @@ export class DagRunService { }, }); } + + /** + * Delete Dag Run + * Delete a DAG Run entry. + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns void Successful Response + * @throws ApiError + */ + public static deleteDagRun( + data: DeleteDagRunData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "DELETE", + url: "/public/dags/{dag_id}/dagRuns/{dag_run_id}", + path: { + dag_id: data.dagId, + dag_run_id: data.dagRunId, + }, + errors: { + 400: "Bad Request", + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Validation Error", + }, + }); + } } diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 45bfa51aec9c4..5fe3615c7d4e6 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -355,6 +355,13 @@ export type GetDagRunData = { export type GetDagRunResponse = DAGRunResponse; +export type DeleteDagRunData = { + dagId: string; + dagRunId: string; +}; + +export type DeleteDagRunResponse = void; + export type $OpenApiTs = { "/ui/next_run_assets/{dag_id}": { get: { @@ -657,5 +664,34 @@ export type $OpenApiTs = { 422: HTTPValidationError; }; }; + delete: { + req: DeleteDagRunData; + res: { + /** + * Successful Response + */ + 204: void; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; }; }; diff --git a/tests/api_fastapi/views/public/test_dag_run.py b/tests/api_fastapi/views/public/test_dag_run.py index 176ae07d3fadb..f8d6780e50a3a 100644 --- a/tests/api_fastapi/views/public/test_dag_run.py +++ b/tests/api_fastapi/views/public/test_dag_run.py @@ -136,3 +136,15 @@ def test_get_dag_run_not_found(test_client): assert response.status_code == 404 body = response.json() assert body["detail"] == "The DagRun with dag_id: `test_dag1` and run_id: `invalid` was not found" + + +class TestDeleteDagRun: + def test_delete_dag_run(self, test_client): + response = test_client.delete(f"/public/dags/{DAG1_ID}/dagRuns/{DAG1_RUN1_ID}") + assert response.status_code == 204 + + def test_delete_dag_run_not_found(self, test_client): + response = test_client.delete(f"/public/dags/{DAG1_ID}/dagRuns/invalid") + assert response.status_code == 404 + body = response.json() + assert body["detail"] == "The DagRun with dag_id: `test_dag1` and run_id: `invalid` was not found" From b19a945fb5d92989056a032b2dfd009dfa101622 Mon Sep 17 00:00:00 2001 From: michaeljs-c <72604759+michaeljs-c@users.noreply.github.com> Date: Sun, 13 Oct 2024 13:57:51 +0100 Subject: [PATCH 092/199] Make datStats endpoint dag_ids parameter optional (#42955) * Make datStats dag_id parameter optional with pagination * move pagination out of sql query * tidy --------- Co-authored-by: Michael Smith-Chandler --- .../endpoints/dag_stats_endpoint.py | 40 ++-- airflow/api_connexion/openapi/v1.yaml | 4 +- airflow/www/static/js/types/api-generated.ts | 6 +- .../endpoints/test_dag_stats_endpoint.py | 194 +++++++++++++++++- 4 files changed, 227 insertions(+), 17 deletions(-) diff --git a/airflow/api_connexion/endpoints/dag_stats_endpoint.py b/airflow/api_connexion/endpoints/dag_stats_endpoint.py index 705af10d41d99..3b6c6ab8e0df3 100644 --- a/airflow/api_connexion/endpoints/dag_stats_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_stats_endpoint.py @@ -39,24 +39,40 @@ @security.requires_access_dag("GET", DagAccessEntity.RUN) @provide_session -def get_dag_stats(*, dag_ids: str, session: Session = NEW_SESSION) -> APIResponse: +def get_dag_stats( + *, + dag_ids: str | None = None, + limit: int | None = None, + offset: int | None = None, + session: Session = NEW_SESSION, +) -> APIResponse: """Get Dag statistics.""" allowed_dag_ids = get_auth_manager().get_permitted_dag_ids(methods=["GET"], user=g.user) - dags_list = set(dag_ids.split(",")) - filter_dag_ids = dags_list.intersection(allowed_dag_ids) + if dag_ids: + dags_list = set(dag_ids.split(",")) + filter_dag_ids = dags_list.intersection(allowed_dag_ids) + else: + filter_dag_ids = allowed_dag_ids + query_dag_ids = sorted(list(filter_dag_ids)) + if offset is not None: + query_dag_ids = query_dag_ids[offset:] + if limit is not None: + query_dag_ids = query_dag_ids[:limit] query = ( select(DagRun.dag_id, DagRun.state, func.count(DagRun.state)) .group_by(DagRun.dag_id, DagRun.state) - .where(DagRun.dag_id.in_(filter_dag_ids)) + .where(DagRun.dag_id.in_(query_dag_ids)) ) dag_state_stats = session.execute(query) - dag_state_data = {(dag_id, state): count for dag_id, state, count in dag_state_stats} - dag_stats = { - dag_id: [{"state": state, "count": dag_state_data.get((dag_id, state), 0)} for state in DagRunState] - for dag_id in filter_dag_ids - } - - dags = [{"dag_id": stat, "stats": dag_stats[stat]} for stat in dag_stats] - return dag_stats_collection_schema.dump({"dags": dags, "total_entries": len(dag_stats)}) + dags = [ + { + "dag_id": dag_id, + "stats": [ + {"state": state, "count": dag_state_data.get((dag_id, state), 0)} for state in DagRunState + ], + } + for dag_id in query_dag_ids + ] + return dag_stats_collection_schema.dump({"dags": dags, "total_entries": len(dags)}) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index b39d1cd955dd9..e99f91639c49e 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2384,11 +2384,13 @@ paths: operationId: get_dag_stats tags: [DagStats] parameters: + - $ref: "#/components/parameters/PageLimit" + - $ref: "#/components/parameters/PageOffset" - name: dag_ids in: query schema: type: string - required: true + required: false description: | One or more DAG IDs separated by commas to filter relevant Dags. responses: diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 15391c2942432..ef45dbd3b57b6 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -4962,8 +4962,12 @@ export interface operations { get_dag_stats: { parameters: { query: { + /** The numbers of items to return. */ + limit?: components["parameters"]["PageLimit"]; + /** The number of items to skip before starting to collect the result set. */ + offset?: components["parameters"]["PageOffset"]; /** One or more DAG IDs separated by commas to filter relevant Dags. */ - dag_ids: string; + dag_ids?: string; }; }; responses: { diff --git a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py index a447e2a6a4b23..fe563b9444030 100644 --- a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py @@ -76,9 +76,10 @@ def _create_dag(self, dag_id): self.app.dag_bag.bag_dag(dag) return dag_instance - def test_should_respond_200(self, session): + def _create_dag_runs(self, session): self._create_dag("dag_stats_dag") self._create_dag("dag_stats_dag_2") + self._create_dag("dag_stats_dag_3") dag_1_run_1 = DagRun( dag_id="dag_stats_dag", run_id="test_dag_run_id_1", @@ -106,8 +107,20 @@ def test_should_respond_200(self, session): external_trigger=True, state="queued", ) - session.add_all((dag_1_run_1, dag_1_run_2, dag_2_run_1)) + dag_3_run_1 = DagRun( + dag_id="dag_stats_dag_3", + run_id="test_dag_3_run_id_1", + run_type=DagRunType.MANUAL, + execution_date=timezone.parse(self.default_time), + start_date=timezone.parse(self.default_time), + external_trigger=True, + state="success", + ) + session.add_all((dag_1_run_1, dag_1_run_2, dag_2_run_1, dag_3_run_1)) session.commit() + + def test_should_respond_200(self, session): + self._create_dag_runs(session) exp_payload = { "dags": [ { @@ -165,7 +178,182 @@ def test_should_respond_200(self, session): assert sorted(response.json["dags"], key=lambda d: d["dag_id"]) == sorted( exp_payload["dags"], key=lambda d: d["dag_id"] ) - response.json["total_entries"] == 2 + assert response.json["total_entries"] == 2 + + @pytest.mark.parametrize( + "url, exp_payload", + [ + ( + "api/v1/dagStats", + { + "dags": [ + { + "dag_id": "dag_stats_dag", + "stats": [ + { + "state": DagRunState.QUEUED, + "count": 0, + }, + { + "state": DagRunState.RUNNING, + "count": 1, + }, + { + "state": DagRunState.SUCCESS, + "count": 0, + }, + { + "state": DagRunState.FAILED, + "count": 1, + }, + ], + }, + { + "dag_id": "dag_stats_dag_2", + "stats": [ + { + "state": DagRunState.QUEUED, + "count": 1, + }, + { + "state": DagRunState.RUNNING, + "count": 0, + }, + { + "state": DagRunState.SUCCESS, + "count": 0, + }, + { + "state": DagRunState.FAILED, + "count": 0, + }, + ], + }, + { + "dag_id": "dag_stats_dag_3", + "stats": [ + { + "state": DagRunState.QUEUED, + "count": 0, + }, + { + "state": DagRunState.RUNNING, + "count": 0, + }, + { + "state": DagRunState.SUCCESS, + "count": 1, + }, + { + "state": DagRunState.FAILED, + "count": 0, + }, + ], + }, + ], + "total_entries": 3, + }, + ), + ( + "api/v1/dagStats?limit=1", + { + "dags": [ + { + "dag_id": "dag_stats_dag", + "stats": [ + { + "state": DagRunState.QUEUED, + "count": 0, + }, + { + "state": DagRunState.RUNNING, + "count": 1, + }, + { + "state": DagRunState.SUCCESS, + "count": 0, + }, + { + "state": DagRunState.FAILED, + "count": 1, + }, + ], + } + ], + "total_entries": 1, + }, + ), + ( + "api/v1/dagStats?offset=2", + { + "dags": [ + { + "dag_id": "dag_stats_dag_3", + "stats": [ + { + "state": DagRunState.QUEUED, + "count": 0, + }, + { + "state": DagRunState.RUNNING, + "count": 0, + }, + { + "state": DagRunState.SUCCESS, + "count": 1, + }, + { + "state": DagRunState.FAILED, + "count": 0, + }, + ], + }, + ], + "total_entries": 1, + }, + ), + ( + "api/v1/dagStats?offset=1&limit=1", + { + "dags": [ + { + "dag_id": "dag_stats_dag_2", + "stats": [ + { + "state": DagRunState.QUEUED, + "count": 1, + }, + { + "state": DagRunState.RUNNING, + "count": 0, + }, + { + "state": DagRunState.SUCCESS, + "count": 0, + }, + { + "state": DagRunState.FAILED, + "count": 0, + }, + ], + }, + ], + "total_entries": 1, + }, + ), + ("api/v1/dagStats?offset=10&limit=1", {"dags": [], "total_entries": 0}), + ], + ) + def test_optional_dag_ids_with_limit_offset(self, url, exp_payload, session): + self._create_dag_runs(session) + + response = self.client.get(url, environ_overrides={"REMOTE_USER": "test"}) + num_dags = len(exp_payload["dags"]) + assert response.status_code == 200 + assert sorted(response.json["dags"], key=lambda d: d["dag_id"]) == sorted( + exp_payload["dags"], key=lambda d: d["dag_id"] + ) + assert response.json["total_entries"] == num_dags def test_should_raises_401_unauthenticated(self): dag_ids = "dag_stats_dag,dag_stats_dag_2" From 4a152b0a7b2356437dae55d322cde63c798bdf11 Mon Sep 17 00:00:00 2001 From: GPK Date: Sun, 13 Oct 2024 16:12:57 +0100 Subject: [PATCH 093/199] trove classifier upgrade (#42979) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6edf4e5c71175..160e8e2ce0b0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "pluggy==1.5.0", "smmap==5.0.1", "tomli==2.0.2; python_version < '3.11'", - "trove-classifiers==2024.10.12", + "trove-classifiers==2024.10.13", ] build-backend = "hatchling.build" From 5930331236979557794dc40cf4033b1c45ff215c Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Sun, 13 Oct 2024 18:06:59 -0400 Subject: [PATCH 094/199] Change directory used by simple auth manager to store generated passwords (#42860) * Change directory used by simple auth manager to store generated passwords * Update airflow/auth/managers/simple/simple_auth_manager.py Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> --------- Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> --- Dockerfile.ci | 3 +++ .../managers/simple/simple_auth_manager.py | 20 ++++++++++--------- scripts/docker/entrypoint_ci.sh | 3 +++ .../simple/test_simple_auth_manager.py | 4 ++-- tests/auth/managers/simple/views/test_auth.py | 2 +- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 464e33f147fb1..7e0ee74556f9d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -914,6 +914,9 @@ function environment_initialization() { # Added to have run-tests on path export PATH=${PATH}:${AIRFLOW_SOURCES} + # Directory where simple auth manager store generated passwords + export AIRFLOW_AUTH_MANAGER_CREDENTIAL_DIRECTORY="/files" + mkdir -pv "${AIRFLOW_HOME}/logs/" # Change the default worker_concurrency for tests diff --git a/airflow/auth/managers/simple/simple_auth_manager.py b/airflow/auth/managers/simple/simple_auth_manager.py index 4a9639a998c46..78dccf7c2a980 100644 --- a/airflow/auth/managers/simple/simple_auth_manager.py +++ b/airflow/auth/managers/simple/simple_auth_manager.py @@ -30,7 +30,7 @@ from airflow.auth.managers.base_auth_manager import BaseAuthManager, ResourceMethod from airflow.auth.managers.simple.user import SimpleAuthManagerUser from airflow.auth.managers.simple.views.auth import SimpleAuthManagerAuthenticationViews -from hatch_build import AIRFLOW_ROOT_PATH +from airflow.configuration import AIRFLOW_HOME if TYPE_CHECKING: from airflow.auth.managers.models.base_user import BaseUser @@ -78,20 +78,22 @@ class SimpleAuthManager(BaseAuthManager): :param appbuilder: the flask app builder """ - # File that contains the generated passwords - GENERATED_PASSWORDS_FILE = ( - AIRFLOW_ROOT_PATH / "generated" / "simple_auth_manager_passwords.json.generated" - ) - # Cache containing the password associated to a username passwords: dict[str, str] = {} + @staticmethod + def get_generated_password_file() -> str: + return os.path.join( + os.getenv("AIRFLOW_AUTH_MANAGER_CREDENTIAL_DIRECTORY", AIRFLOW_HOME), + "simple_auth_manager_passwords.json.generated", + ) + def init(self) -> None: user_passwords_from_file = {} # Read passwords from file - if os.path.isfile(self.GENERATED_PASSWORDS_FILE): - with open(self.GENERATED_PASSWORDS_FILE) as file: + if os.path.isfile(self.get_generated_password_file()): + with open(self.get_generated_password_file()) as file: passwords_str = file.read().strip() user_passwords_from_file = json.loads(passwords_str) @@ -109,7 +111,7 @@ def init(self) -> None: self._print_output(f"Password for user '{user['username']}': {self.passwords[user['username']]}") - with open(self.GENERATED_PASSWORDS_FILE, "w") as file: + with open(self.get_generated_password_file(), "w") as file: file.write(json.dumps(self.passwords)) def is_logged_in(self) -> bool: diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index d0946b39e0e28..a96e58b9c21a5 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -139,6 +139,9 @@ function environment_initialization() { # Added to have run-tests on path export PATH=${PATH}:${AIRFLOW_SOURCES} + # Directory where simple auth manager store generated passwords + export AIRFLOW_AUTH_MANAGER_CREDENTIAL_DIRECTORY="/files" + mkdir -pv "${AIRFLOW_HOME}/logs/" # Change the default worker_concurrency for tests diff --git a/tests/auth/managers/simple/test_simple_auth_manager.py b/tests/auth/managers/simple/test_simple_auth_manager.py index d4bd4e4fbfed2..434c0d60fcc76 100644 --- a/tests/auth/managers/simple/test_simple_auth_manager.py +++ b/tests/auth/managers/simple/test_simple_auth_manager.py @@ -50,7 +50,7 @@ class TestSimpleAuthManager: @pytest.mark.db_test def test_init_with_no_user(self, auth_manager_with_appbuilder): auth_manager_with_appbuilder.init() - with open(SimpleAuthManager.GENERATED_PASSWORDS_FILE) as file: + with open(auth_manager_with_appbuilder.get_generated_password_file()) as file: passwords_str = file.read().strip() user_passwords_from_file = json.loads(passwords_str) @@ -65,7 +65,7 @@ def test_init_with_users(self, auth_manager_with_appbuilder): } ] auth_manager_with_appbuilder.init() - with open(SimpleAuthManager.GENERATED_PASSWORDS_FILE) as file: + with open(auth_manager_with_appbuilder.get_generated_password_file()) as file: passwords_str = file.read().strip() user_passwords_from_file = json.loads(passwords_str) diff --git a/tests/auth/managers/simple/views/test_auth.py b/tests/auth/managers/simple/views/test_auth.py index a4e2a12fdcf1c..f61a9278f79a5 100644 --- a/tests/auth/managers/simple/views/test_auth.py +++ b/tests/auth/managers/simple/views/test_auth.py @@ -37,7 +37,7 @@ def simple_app(): ): "airflow.auth.managers.simple.simple_auth_manager.SimpleAuthManager", } ): - with open(SimpleAuthManager.GENERATED_PASSWORDS_FILE, "w") as file: + with open(SimpleAuthManager.get_generated_password_file(), "w") as file: user = {"test": "test"} file.write(json.dumps(user)) From a8c708063da3e2f1f7422a6babb40c01f4a70fd0 Mon Sep 17 00:00:00 2001 From: GPK Date: Sun, 13 Oct 2024 23:25:55 +0100 Subject: [PATCH 095/199] Disable flaky mssql based integration tests (#42811) * quarantined flaky mssql integration tests * disable mssql from ci integration checks --- dev/breeze/src/airflow_breeze/global_constants.py | 3 +++ dev/breeze/src/airflow_breeze/utils/selective_checks.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 3879c121433f5..cd8a59aa05cef 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -68,6 +68,9 @@ "trino", "ydb", ] +DISABLE_TESTABLE_INTEGRATIONS_FROM_CI = [ + "mssql", +] OTHER_INTEGRATIONS = ["statsd", "otel", "openlineage"] ALLOWED_DEBIAN_VERSIONS = ["bookworm"] ALL_INTEGRATIONS = sorted( diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 20bf514379f0b..cf4e124ff94e3 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -42,6 +42,7 @@ DEFAULT_MYSQL_VERSION, DEFAULT_POSTGRES_VERSION, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, KIND_VERSION, RUNS_ON_PUBLIC_RUNNER, @@ -1318,7 +1319,11 @@ def excluded_providers_as_string(self) -> str: @cached_property def testable_integrations(self) -> list[str]: - return TESTABLE_INTEGRATIONS + return [ + integration + for integration in TESTABLE_INTEGRATIONS + if integration not in DISABLE_TESTABLE_INTEGRATIONS_FROM_CI + ] @cached_property def is_committer_build(self): From 1568623c315c7078e5e65f68bd1e8fe002d521ba Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Sun, 13 Oct 2024 18:15:11 -0700 Subject: [PATCH 096/199] Remove BackfillJobRunner class (#42943) --- airflow/cli/cli_config.py | 2 +- airflow/exceptions.py | 25 - airflow/jobs/backfill_job_runner.py | 1106 --------- airflow/jobs/job.py | 4 +- airflow/models/dag.py | 3 +- airflow/models/dagpickle.py | 2 +- airflow/models/dagrun.py | 2 +- airflow/models/taskinstance.py | 2 +- airflow/ti_deps/dependencies_deps.py | 8 - airflow/ti_deps/dependencies_states.py | 9 - .../listeners.rst | 2 +- docs/apache-airflow/howto/listener-plugin.rst | 4 +- tests/core/test_impersonation_tests.py | 24 +- tests/jobs/test_backfill_job.py | 2094 ----------------- tests/jobs/test_scheduler_job.py | 29 +- 15 files changed, 29 insertions(+), 3287 deletions(-) delete mode 100644 airflow/jobs/backfill_job_runner.py delete mode 100644 tests/jobs/test_backfill_job.py diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index e68a464f61bb5..8fe239c27ba40 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -924,7 +924,7 @@ def string_lower_type(val): # jobs check ARG_JOB_TYPE_FILTER = Arg( ("--job-type",), - choices=("BackfillJob", "LocalTaskJob", "SchedulerJob", "TriggererJob", "DagProcessorJob"), + choices=("LocalTaskJob", "SchedulerJob", "TriggererJob", "DagProcessorJob"), action="store", help="The type of job(s) that will be checked.", ) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 55dd02fdae313..ccf62ca5e8178 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -327,31 +327,6 @@ class PoolNotFound(AirflowNotFoundException): """Raise when a Pool is not available in the system.""" -class NoAvailablePoolSlot(AirflowException): - """Raise when there is not enough slots in pool.""" - - -class DagConcurrencyLimitReached(AirflowException): - """Raise when DAG max_active_tasks limit is reached.""" - - -class TaskConcurrencyLimitReached(AirflowException): - """Raise when task max_active_tasks limit is reached.""" - - -class BackfillUnfinished(AirflowException): - """ - Raises when not all tasks succeed in backfill. - - :param message: The human-readable description of the exception - :param ti_status: The information about all task statuses - """ - - def __init__(self, message, ti_status): - super().__init__(message) - self.ti_status = ti_status - - class FileSyntaxError(NamedTuple): """Information about a single error in a file.""" diff --git a/airflow/jobs/backfill_job_runner.py b/airflow/jobs/backfill_job_runner.py deleted file mode 100644 index 19dda4d698221..0000000000000 --- a/airflow/jobs/backfill_job_runner.py +++ /dev/null @@ -1,1106 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import time -from typing import TYPE_CHECKING, Any, Iterable, Iterator, Mapping, Sequence - -import attr -import pendulum -from sqlalchemy import case, or_, select, tuple_, update -from sqlalchemy.exc import OperationalError -from sqlalchemy.orm.session import make_transient -from tabulate import tabulate - -from airflow import models -from airflow.exceptions import ( - AirflowException, - BackfillUnfinished, - DagConcurrencyLimitReached, - NoAvailablePoolSlot, - PoolNotFound, - TaskConcurrencyLimitReached, - UnknownExecutorException, -) -from airflow.executors.executor_loader import ExecutorLoader -from airflow.jobs.base_job_runner import BaseJobRunner -from airflow.jobs.job import Job, perform_heartbeat -from airflow.models import DAG, DagPickle -from airflow.models.dagrun import DagRun -from airflow.models.taskinstance import TaskInstance -from airflow.ti_deps.dep_context import DepContext -from airflow.ti_deps.dependencies_deps import BACKFILL_QUEUED_DEPS -from airflow.timetables.base import DagRunInfo -from airflow.utils import helpers, timezone -from airflow.utils.configuration import tmp_configuration_copy -from airflow.utils.log.logging_mixin import LoggingMixin -from airflow.utils.session import NEW_SESSION, provide_session -from airflow.utils.state import DagRunState, State, TaskInstanceState -from airflow.utils.types import DagRunTriggeredByType, DagRunType - -if TYPE_CHECKING: - import datetime - - from sqlalchemy.orm.session import Session - - from airflow.executors.base_executor import BaseExecutor - from airflow.models.abstractoperator import AbstractOperator - from airflow.models.taskinstance import TaskInstanceKey - - -class BackfillJobRunner(BaseJobRunner, LoggingMixin): - """ - A backfill job runner consists of a dag for a specific time range. - - It triggers a set of task instance runs, in the right order and lasts for - as long as it takes for the set of task instance to be completed. - """ - - job_type = "BackfillJob" - - STATES_COUNT_AS_RUNNING = (TaskInstanceState.RUNNING, TaskInstanceState.QUEUED) - - @attr.define - class _DagRunTaskStatus: - """ - Internal status of the backfill job. - - This class is intended to be instantiated only within a BackfillJobRunner - instance and will track the execution of tasks, e.g. running, skipped, - succeeded, failed, etc. Information about the dag runs related to the - backfill job are also being tracked in this structure, e.g. finished runs, etc. - Any other status related information related to the execution of dag runs / tasks - can be included in this structure since it makes it easier to pass it around. - - :param to_run: Tasks to run in the backfill - :param running: Maps running task instance key to task instance object - :param skipped: Tasks that have been skipped - :param succeeded: Tasks that have succeeded so far - :param failed: Tasks that have failed - :param not_ready: Tasks not ready for execution - :param deadlocked: Deadlocked tasks - :param active_runs: Active dag runs at a certain point in time - :param executed_dag_run_dates: Datetime objects for the executed dag runs - :param finished_runs: Number of finished runs so far - :param total_runs: Number of total dag runs able to run - """ - - to_run: dict[TaskInstanceKey, TaskInstance] = attr.ib(factory=dict) - running: dict[TaskInstanceKey, TaskInstance] = attr.ib(factory=dict) - skipped: set[TaskInstanceKey] = attr.ib(factory=set) - succeeded: set[TaskInstanceKey] = attr.ib(factory=set) - failed: set[TaskInstanceKey] = attr.ib(factory=set) - not_ready: set[TaskInstanceKey] = attr.ib(factory=set) - deadlocked: set[TaskInstance] = attr.ib(factory=set) - active_runs: set[DagRun] = attr.ib(factory=set) - executed_dag_run_dates: set[pendulum.DateTime] = attr.ib(factory=set) - finished_runs: int = 0 - total_runs: int = 0 - - def __init__( - self, - job: Job, - dag: DAG, - start_date=None, - end_date=None, - mark_success=False, - donot_pickle=False, - ignore_first_depends_on_past=False, - ignore_task_deps=False, - pool=None, - delay_on_limit_secs=1.0, - verbose=False, - conf=None, - rerun_failed_tasks=False, - run_backwards=False, - run_at_least_once=False, - continue_on_failures=False, - disable_retry=False, - ) -> None: - """ - Create a BackfillJobRunner. - - :param dag: DAG object. - :param start_date: start date for the backfill date range. - :param end_date: end date for the backfill date range. - :param mark_success: flag whether to mark the task auto success. - :param donot_pickle: whether pickle - :param ignore_first_depends_on_past: whether to ignore depend on past - :param ignore_task_deps: whether to ignore the task dependency - :param pool: pool to backfill - :param delay_on_limit_secs: - :param verbose: - :param conf: a dictionary which user could pass k-v pairs for backfill - :param rerun_failed_tasks: flag to whether to - auto rerun the failed task in backfill - :param run_backwards: Whether to process the dates from most to least recent - :param run_at_least_once: If true, always run the DAG at least once even - if no logical run exists within the time range. - :param args: - :param kwargs: - """ - super().__init__(job) - self.dag = dag - self.dag_id = dag.dag_id - self.bf_start_date = start_date - self.bf_end_date = end_date - self.mark_success = mark_success - self.donot_pickle = donot_pickle - self.ignore_first_depends_on_past = ignore_first_depends_on_past - self.ignore_task_deps = ignore_task_deps - self.pool = pool - self.delay_on_limit_secs = delay_on_limit_secs - self.verbose = verbose - self.conf = conf - self.rerun_failed_tasks = rerun_failed_tasks - self.run_backwards = run_backwards - self.run_at_least_once = run_at_least_once - self.continue_on_failures = continue_on_failures - self.disable_retry = disable_retry - - def _update_counters(self, ti_status: _DagRunTaskStatus, session: Session) -> None: - """ - Update the counters per state of the tasks that were running. - - Can re-add to tasks to run when required. - - :param ti_status: the internal status of the backfill job tasks - """ - tis_to_be_scheduled = [] - refreshed_tis = [] - TI = TaskInstance - - ti_primary_key_to_ti_key = {ti_key.primary: ti_key for ti_key in ti_status.running.keys()} - - filter_for_tis = TI.filter_for_tis(list(ti_status.running.values())) - if filter_for_tis is not None: - refreshed_tis = session.scalars(select(TI).where(filter_for_tis)).all() - - for ti in refreshed_tis: - # Use primary key to match in memory information - ti_key = ti_primary_key_to_ti_key[ti.key.primary] - if ti.state == TaskInstanceState.SUCCESS: - ti_status.succeeded.add(ti_key) - self.log.debug("Task instance %s succeeded. Don't rerun.", ti) - ti_status.running.pop(ti_key) - continue - if ti.state == TaskInstanceState.SKIPPED: - ti_status.skipped.add(ti_key) - self.log.debug("Task instance %s skipped. Don't rerun.", ti) - ti_status.running.pop(ti_key) - continue - if ti.state == TaskInstanceState.FAILED: - self.log.error("Task instance %s failed", ti) - ti_status.failed.add(ti_key) - ti_status.running.pop(ti_key) - continue - # special case: if the task needs to run again put it back - if ti.state == TaskInstanceState.UP_FOR_RETRY: - self.log.warning("Task instance %s is up for retry", ti) - ti_status.running.pop(ti_key) - ti_status.to_run[ti.key] = ti - # special case: if the task needs to be rescheduled put it back - elif ti.state == TaskInstanceState.UP_FOR_RESCHEDULE: - self.log.warning("Task instance %s is up for reschedule", ti) - ti_status.running.pop(ti_key) - ti_status.to_run[ti.key] = ti - # special case: The state of the task can be set to NONE by the task itself - # when it reaches concurrency limits. It could also happen when the state - # is changed externally, e.g. by clearing tasks from the ui. We need to cover - # for that as otherwise those tasks would fall outside the scope of - # the backfill suddenly. - elif ti.state is None: - self.log.warning( - "FIXME: task instance %s state was set to none externally or " - "reaching concurrency limits. Re-adding task to queue.", - ti, - ) - tis_to_be_scheduled.append(ti) - ti_status.running.pop(ti_key) - ti_status.to_run[ti.key] = ti - # special case: Deferrable task can go from DEFERRED to SCHEDULED; - # when that happens, we need to put it back as in UP_FOR_RESCHEDULE - elif ti.state == TaskInstanceState.SCHEDULED: - self.log.debug("Task instance %s is resumed from deferred state", ti) - ti_status.running.pop(ti_key) - ti_status.to_run[ti.key] = ti - - # Batch schedule of task instances - if tis_to_be_scheduled: - filter_for_tis = TI.filter_for_tis(tis_to_be_scheduled) - session.execute( - update(TI) - .where(filter_for_tis) - .values( - state=TaskInstanceState.SCHEDULED, - try_number=case( - ( - or_(TI.state.is_(None), TI.state != TaskInstanceState.UP_FOR_RESCHEDULE), - TI.try_number + 1, - ), - else_=TI.try_number, - ), - ) - .execution_options(synchronize_session=False) - ) - session.flush() - - def _manage_executor_state( - self, - running: Mapping[TaskInstanceKey, TaskInstance], - executor: BaseExecutor, - session: Session, - ) -> Iterator[tuple[AbstractOperator, str, Sequence[TaskInstance], int]]: - """ - Compare task instances' states with that of the executor. - - Expands downstream mapped tasks when necessary. - - :param running: dict of key, task to verify - :return: An iterable of expanded TaskInstance per MappedTask - """ - # list of tuples (dag_id, task_id, execution_date, map_index) of running tasks in executor - buffered_events = list(executor.get_event_buffer().items()) - running_tis_ids = [ - (key.dag_id, key.task_id, key.run_id, key.map_index) - for key, _ in buffered_events - if key in running - ] - # list of TaskInstance of running tasks in executor (refreshed from db in batch) - refreshed_running_tis = session.scalars( - select(TaskInstance).where( - tuple_( - TaskInstance.dag_id, - TaskInstance.task_id, - TaskInstance.run_id, - TaskInstance.map_index, - ).in_(running_tis_ids) - ) - ).all() - # dict of refreshed TaskInstance by key to easily find them - running_dict = {(ti.dag_id, ti.task_id, ti.run_id, ti.map_index): ti for ti in refreshed_running_tis} - need_refresh = False - - for key, value in buffered_events: - state, info = value - ti_key = (key.dag_id, key.task_id, key.run_id, key.map_index) - if ti_key not in running_dict: - self.log.warning("%s state %s not in running=%s", key, state, running.values()) - continue - - ti = running_dict[ti_key] - if need_refresh: - ti.refresh_from_db(session=session) - - self.log.debug("Executor state: %s task %s", state, ti) - - if ( - state in (TaskInstanceState.FAILED, TaskInstanceState.SUCCESS) - and ti.state in self.STATES_COUNT_AS_RUNNING - ): - msg = ( - f"The executor reported that the task instance {ti} finished with state {state}, " - f"but the task instance's state attribute is {ti.state}. " - "Learn more: https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#task-state-changed-externally" - ) - if info is not None: - msg += f" Extra info: {info}" - self.log.error(msg) - ti.handle_failure(error=msg) - continue - - def _iter_task_needing_expansion() -> Iterator[AbstractOperator]: - from airflow.models.mappedoperator import AbstractOperator - - for node in self.dag.get_task(ti.task_id).iter_mapped_dependants(): - if isinstance(node, AbstractOperator): - yield node - else: # A (mapped) task group. All its children need expansion. - yield from node.iter_tasks() - - if ti.state not in self.STATES_COUNT_AS_RUNNING: - # Don't use ti.task; if this task is mapped, that attribute - # would hold the unmapped task. We need to original task here. - for node in _iter_task_needing_expansion(): - new_tis, num_mapped_tis = node.expand_mapped_task(ti.run_id, session=session) - yield node, ti.run_id, new_tis, num_mapped_tis - - @provide_session - def _get_dag_run( - self, - dagrun_info: DagRunInfo, - dag: DAG, - session: Session = NEW_SESSION, - ) -> DagRun | None: - """ - Return an existing dag run for the given run date or create one. - - If the max_active_runs limit is reached, this function will return None. - - :param dagrun_info: Schedule information for the dag run - :param dag: DAG - :param session: the database session object - :return: a DagRun in state RUNNING or None - """ - run_date = dagrun_info.logical_date - - respect_dag_max_active_limit = bool(dag.timetable.can_be_scheduled) - - current_active_dag_count = dag.get_num_active_runs(external_trigger=False) - - # check if we are scheduling on top of an already existing DAG run - # we could find a "scheduled" run instead of a "backfill" - runs = DagRun.find(dag_id=dag.dag_id, execution_date=run_date, session=session) - run: DagRun | None - if runs: - run = runs[0] - if run.state == DagRunState.RUNNING: - respect_dag_max_active_limit = False - # Fixes --conf overwrite for backfills with already existing DagRuns - run.conf = self.conf or {} - # start_date is cleared for existing DagRuns - run.start_date = timezone.utcnow() - else: - run = None - - # enforce max_active_runs limit for dag, special cases already - # handled by respect_dag_max_active_limit - if respect_dag_max_active_limit and current_active_dag_count >= dag.max_active_runs: - return None - - run = run or dag.create_dagrun( - execution_date=run_date, - data_interval=dagrun_info.data_interval, - start_date=timezone.utcnow(), - state=DagRunState.RUNNING, - external_trigger=False, - session=session, - conf=self.conf, - run_type=DagRunType.BACKFILL_JOB, - creating_job_id=self.job.id, - triggered_by=DagRunTriggeredByType.TIMETABLE, - ) - - # set required transient field - run.dag = dag - - # explicitly mark as backfill and running - run.state = DagRunState.RUNNING - run.run_type = DagRunType.BACKFILL_JOB - run.verify_integrity(session=session) - - run.notify_dagrun_state_changed(msg="started") - return run - - @provide_session - def _task_instances_for_dag_run( - self, - dag: DAG, - dag_run: DagRun, - session: Session = NEW_SESSION, - ) -> dict[TaskInstanceKey, TaskInstance]: - """ - Return a map of task instance keys to task instance objects for the given dag run. - - :param dag_run: the dag run to get the tasks from - :param session: the database session object - """ - tasks_to_run = {} - - if dag_run is None: - return tasks_to_run - - # check if we have orphaned tasks - self.reset_state_for_orphaned_tasks(filter_by_dag_run=dag_run, session=session) - - # for some reason if we don't refresh the reference to run is lost - dag_run.refresh_from_db(session=session) - make_transient(dag_run) - - dag_run.dag = dag - info = dag_run.task_instance_scheduling_decisions(session=session) - schedulable_tis = info.schedulable_tis - try: - for ti in dag_run.get_task_instances(session=session): - if ti in schedulable_tis: - if ti.state != TaskInstanceState.UP_FOR_RESCHEDULE: - ti.try_number += 1 - ti.set_state(TaskInstanceState.SCHEDULED) - if ti.state != TaskInstanceState.REMOVED: - tasks_to_run[ti.key] = ti - session.commit() - except Exception: - session.rollback() - raise - return tasks_to_run - - def _log_progress(self, ti_status: _DagRunTaskStatus) -> None: - self.log.info( - "[backfill progress] | finished run %s of %s | tasks waiting: %s | succeeded: %s | " - "running: %s | failed: %s | skipped: %s | deadlocked: %s | not ready: %s", - ti_status.finished_runs, - ti_status.total_runs, - len(ti_status.to_run), - len(ti_status.succeeded), - len(ti_status.running), - len(ti_status.failed), - len(ti_status.skipped), - len(ti_status.deadlocked), - len(ti_status.not_ready), - ) - - self.log.debug("Finished dag run loop iteration. Remaining tasks %s", ti_status.to_run.values()) - - def _process_backfill_task_instances( - self, - ti_status: _DagRunTaskStatus, - pickle_id: int | None, - start_date: datetime.datetime | None = None, - *, - session: Session, - ) -> list: - """ - Process a set of task instances from a set of DAG runs. - - Special handling is done to account for different task instance states - that could be present when running them in a backfill process. - - :param ti_status: the internal status of the job - :param executor: the executor to run the task instances - :param pickle_id: the pickle_id if dag is pickled, None otherwise - :param start_date: the start date of the backfill job - :param session: the current session object - :return: the list of execution_dates for the finished dag runs - """ - executed_run_dates = [] - - while (ti_status.to_run or ti_status.running) and not ti_status.deadlocked: - self.log.debug("Clearing out not_ready list") - ti_status.not_ready.clear() - - # we need to execute the tasks bottom to top - # or leaf to root, as otherwise tasks might be - # determined deadlocked while they are actually - # waiting for their upstream to finish - def _per_task_process(key, ti: TaskInstance, session): - ti.refresh_from_db(lock_for_update=True, session=session) - - task = self.dag.get_task(ti.task_id) - ti.task = task - - self.log.debug("Task instance to run %s state %s", ti, ti.state) - - # The task was already marked successful or skipped by a - # different Job. Don't rerun it. - if ti.state == TaskInstanceState.SUCCESS: - ti_status.succeeded.add(key) - self.log.debug("Task instance %s succeeded. Don't rerun.", ti) - ti_status.to_run.pop(key) - if key in ti_status.running: - ti_status.running.pop(key) - return - elif ti.state == TaskInstanceState.SKIPPED: - ti_status.skipped.add(key) - self.log.debug("Task instance %s skipped. Don't rerun.", ti) - ti_status.to_run.pop(key) - if key in ti_status.running: - ti_status.running.pop(key) - return - - if self.rerun_failed_tasks: - # Rerun failed tasks or upstreamed failed tasks - if ti.state in (TaskInstanceState.FAILED, TaskInstanceState.UPSTREAM_FAILED): - self.log.error("Task instance %s with state %s", ti, ti.state) - if key in ti_status.running: - ti_status.running.pop(key) - # Reset the failed task in backfill to scheduled state - ti.try_number += 1 - ti.set_state(TaskInstanceState.SCHEDULED, session=session) - if ti.dag_run not in ti_status.active_runs: - ti_status.active_runs.add(ti.dag_run) - else: - # Default behaviour which works for subdag. - if ti.state in (TaskInstanceState.FAILED, TaskInstanceState.UPSTREAM_FAILED): - self.log.error("Task instance %s with state %s", ti, ti.state) - ti_status.failed.add(key) - ti_status.to_run.pop(key) - if key in ti_status.running: - ti_status.running.pop(key) - return - - if self.ignore_first_depends_on_past: - dagrun = ti.get_dagrun(session=session) - ignore_depends_on_past = dagrun.execution_date == (start_date or ti.start_date) - else: - ignore_depends_on_past = False - - backfill_context = DepContext( - deps=BACKFILL_QUEUED_DEPS, - ignore_depends_on_past=ignore_depends_on_past, - ignore_task_deps=self.ignore_task_deps, - wait_for_past_depends_before_skipping=False, - flag_upstream_failed=True, - ) - - executor = ExecutorLoader.load_executor(str(ti.executor) if ti.executor else None) - # Is the task runnable? -- then run it - # the dependency checker can change states of tis - if ti.are_dependencies_met( - dep_context=backfill_context, session=session, verbose=self.verbose - ): - if executor.has_task(ti): - self.log.debug("Task Instance %s already in executor waiting for queue to clear", ti) - else: - self.log.debug("Sending %s to executor", ti) - # Skip scheduled state, we are executing immediately - if ti.state in (TaskInstanceState.UP_FOR_RETRY, None): - # i am not sure why this is necessary. - # seemingly a quirk of backfill runner. - # it should be handled elsewhere i think. - # seems the leaf tasks are set SCHEDULED but others not. - # but i am not going to look too closely since we need - # to nuke the current backfill approach anyway. - ti.try_number += 1 - ti.state = TaskInstanceState.QUEUED - ti.queued_by_job_id = self.job.id - ti.queued_dttm = timezone.utcnow() - session.merge(ti) - try: - session.commit() - except OperationalError: - self.log.exception("Failed to commit task state change due to operational error") - session.rollback() - # early exit so the outer loop can retry - return - - cfg_path = None - - if executor.is_local: - cfg_path = tmp_configuration_copy() - - executor.queue_task_instance( - ti, - mark_success=self.mark_success, - pickle_id=pickle_id, - ignore_task_deps=self.ignore_task_deps, - ignore_depends_on_past=ignore_depends_on_past, - wait_for_past_depends_before_skipping=False, - pool=self.pool, - cfg_path=cfg_path, - ) - ti_status.running[key] = ti - ti_status.to_run.pop(key) - return - - if ti.state == TaskInstanceState.UPSTREAM_FAILED: - self.log.error("Task instance %s upstream failed", ti) - ti_status.failed.add(key) - ti_status.to_run.pop(key) - if key in ti_status.running: - ti_status.running.pop(key) - return - - # special case - if ti.state == TaskInstanceState.UP_FOR_RETRY: - self.log.debug("Task instance %s retry period not expired yet", ti) - if key in ti_status.running: - ti_status.running.pop(key) - ti_status.to_run[key] = ti - return - - # special case - if ti.state == TaskInstanceState.UP_FOR_RESCHEDULE: - self.log.debug("Task instance %s reschedule period not expired yet", ti) - if key in ti_status.running: - ti_status.running.pop(key) - ti_status.to_run[key] = ti - return - - # all remaining tasks - self.log.debug("Adding %s to not_ready", ti) - ti_status.not_ready.add(key) - - try: - for task in self.dag.topological_sort(): - for key, ti in list(ti_status.to_run.items()): - # Attempt to workaround deadlock on backfill by attempting to commit the transaction - # state update few times before giving up - max_attempts = 5 - for i in range(max_attempts): - if task.task_id != ti.task_id: - continue - - pool = session.scalar( - select(models.Pool).where(models.Pool.pool == task.pool).limit(1) - ) - if not pool: - raise PoolNotFound(f"Unknown pool: {task.pool}") - - open_slots = pool.open_slots(session=session) - if open_slots <= 0: - raise NoAvailablePoolSlot( - f"Not scheduling since there are {open_slots} " - f"open slots in pool {task.pool}" - ) - - num_running_task_instances_in_dag = DAG.get_num_task_instances( - self.dag_id, - states=self.STATES_COUNT_AS_RUNNING, - session=session, - ) - - if num_running_task_instances_in_dag >= self.dag.max_active_tasks: - raise DagConcurrencyLimitReached( - "Not scheduling since DAG max_active_tasks limit is reached." - ) - - if task.max_active_tis_per_dag is not None: - num_running_task_instances_in_task = DAG.get_num_task_instances( - dag_id=self.dag_id, - task_ids=[task.task_id], - states=self.STATES_COUNT_AS_RUNNING, - session=session, - ) - - if num_running_task_instances_in_task >= task.max_active_tis_per_dag: - raise TaskConcurrencyLimitReached( - "Not scheduling since Task concurrency limit is reached." - ) - - if task.max_active_tis_per_dagrun is not None: - num_running_task_instances_in_task_dagrun = DAG.get_num_task_instances( - dag_id=self.dag_id, - run_id=ti.run_id, - task_ids=[task.task_id], - states=self.STATES_COUNT_AS_RUNNING, - session=session, - ) - - if ( - num_running_task_instances_in_task_dagrun - >= task.max_active_tis_per_dagrun - ): - raise TaskConcurrencyLimitReached( - "Not scheduling since Task concurrency per DAG run limit is reached." - ) - - _per_task_process(key, ti, session) - try: - session.commit() - except OperationalError: - self.log.exception( - "Failed to commit task state due to operational error. " - "The job will retry this operation so if your backfill succeeds, " - "you can safely ignore this message.", - ) - session.rollback() - if i == max_attempts - 1: - raise - # retry the loop - else: - # break the retry loop - break - except (NoAvailablePoolSlot, DagConcurrencyLimitReached, TaskConcurrencyLimitReached) as e: - self.log.debug(e) - - perform_heartbeat( - job=self.job, - heartbeat_callback=self.heartbeat_callback, - only_if_necessary=True, - ) - # execute the tasks in the queue - for executor in self.job.executors: - executor.heartbeat() - - # If the set of tasks that aren't ready ever equals the set of - # tasks to run and there are no running tasks then the backfill - # is deadlocked - if ti_status.not_ready and ti_status.not_ready == set(ti_status.to_run) and not ti_status.running: - self.log.warning("Deadlock discovered for ti_status.to_run=%s", ti_status.to_run.values()) - ti_status.deadlocked.update(ti_status.to_run.values()) - ti_status.to_run.clear() - - for executor in self.job.executors: - # check executor state -- and expand any mapped TIs - for node, run_id, new_mapped_tis, max_map_index in self._manage_executor_state( - ti_status.running, executor, session - ): - - def to_keep(key: TaskInstanceKey) -> bool: - if key.dag_id != node.dag_id or key.task_id != node.task_id or key.run_id != run_id: - # For another Dag/Task/Run -- don't remove - return True - return 0 <= key.map_index <= max_map_index - - # remove the old unmapped TIs for node -- they have been replaced with the mapped TIs - ti_status.to_run = {key: ti for (key, ti) in ti_status.to_run.items() if to_keep(key)} - - ti_status.to_run.update({ti.key: ti for ti in new_mapped_tis}) - - for new_ti in new_mapped_tis: - new_ti.try_number += 1 - new_ti.set_state(TaskInstanceState.SCHEDULED, session=session) - - # Set state to failed for running TIs that are set up for retry if disable-retry flag is set - for ti in ti_status.running.values(): - if self.disable_retry and ti.state == TaskInstanceState.UP_FOR_RETRY: - ti.set_state(TaskInstanceState.FAILED, session=session) - - # update the task counters - self._update_counters(ti_status=ti_status, session=session) - session.commit() - - # update dag run state - _dag_runs = ti_status.active_runs.copy() - for run in _dag_runs: - run.update_state(session=session) - if run.state in State.finished_dr_states: - ti_status.finished_runs += 1 - ti_status.active_runs.remove(run) - executed_run_dates.append(run.execution_date) - - self._log_progress(ti_status) - session.commit() - time.sleep(1) - - # return updated status - return executed_run_dates - - @provide_session - def _collect_errors(self, ti_status: _DagRunTaskStatus, session: Session = NEW_SESSION) -> Iterator[str]: - def tabulate_ti_keys_set(ti_keys: Iterable[TaskInstanceKey]) -> str: - # Sorting by execution date first - sorted_ti_keys: Any = sorted( - ti_keys, - key=lambda ti_key: ( - ti_key.run_id, - ti_key.dag_id, - ti_key.task_id, - ti_key.map_index, - ti_key.try_number, - ), - ) - - if all(key.map_index == -1 for key in ti_keys): - headers = ["DAG ID", "Task ID", "Run ID", "Try number"] - sorted_ti_keys = (k[0:4] for k in sorted_ti_keys) - else: - headers = ["DAG ID", "Task ID", "Run ID", "Map Index", "Try number"] - - return tabulate(sorted_ti_keys, headers=headers) - - if ti_status.failed: - yield "Some task instances failed:\n" - yield tabulate_ti_keys_set(ti_status.failed) - if ti_status.deadlocked: - yield "BackfillJob is deadlocked." - deadlocked_depends_on_past = any( - t.are_dependencies_met( - dep_context=DepContext(ignore_depends_on_past=False), - session=session, - verbose=self.verbose, - ) - != t.are_dependencies_met( - dep_context=DepContext(ignore_depends_on_past=True), session=session, verbose=self.verbose - ) - for t in ti_status.deadlocked - ) - if deadlocked_depends_on_past: - yield ( - "Some of the deadlocked tasks were unable to run because " - 'of "depends_on_past" relationships. Try running the ' - "backfill with the option " - '"ignore_first_depends_on_past=True" or passing "-I" at ' - "the command line." - ) - yield "\nThese tasks have succeeded:\n" - yield tabulate_ti_keys_set(ti_status.succeeded) - yield "\n\nThese tasks are running:\n" - yield tabulate_ti_keys_set(ti_status.running) - yield "\n\nThese tasks have failed:\n" - yield tabulate_ti_keys_set(ti_status.failed) - yield "\n\nThese tasks are skipped:\n" - yield tabulate_ti_keys_set(ti_status.skipped) - yield "\n\nThese tasks are deadlocked:\n" - yield tabulate_ti_keys_set([ti.key for ti in ti_status.deadlocked]) - - @provide_session - def _execute_dagruns( - self, - dagrun_infos: Iterable[DagRunInfo], - ti_status: _DagRunTaskStatus, - pickle_id: int | None, - start_date: datetime.datetime | None, - session: Session = NEW_SESSION, - ) -> None: - """ - Compute and execute dag runs and their respective task instances for the given dates. - - Returns a list of execution dates of the dag runs that were executed. - - :param dagrun_infos: Schedule information for dag runs - :param ti_status: internal BackfillJobRunner status structure to tis track progress - :param pickle_id: numeric id of the pickled dag, None if not pickled - :param start_date: backfill start date - :param session: the current session object - """ - for dagrun_info in dagrun_infos: - dag_run = self._get_dag_run(dagrun_info, self.dag, session=session) - if dag_run is not None: - tis_map = self._task_instances_for_dag_run(self.dag, dag_run, session=session) - ti_status.active_runs.add(dag_run) - ti_status.to_run.update(tis_map or {}) - - tis_missing_executor = [] - for ti in ti_status.to_run.values(): - if ti.executor: - try: - ExecutorLoader.lookup_executor_name_by_str(ti.executor) - except UnknownExecutorException: - tis_missing_executor.append(ti) - - if tis_missing_executor: - raise UnknownExecutorException( - "The following task instances are configured to use an executor that is not present. " - "Review the core.executors Airflow configuration to add it or clear the task instance to " - "clear the executor configuration for this task.\n" - + "\n".join( - [f" {ti.task_id}: {ti.run_id} (executor: {ti.executor})" for ti in tis_missing_executor] - ) - ) - processed_dag_run_dates = self._process_backfill_task_instances( - ti_status=ti_status, - pickle_id=pickle_id, - start_date=start_date, - session=session, - ) - - ti_status.executed_dag_run_dates.update(processed_dag_run_dates) - - @provide_session - def _set_unfinished_dag_runs_to_failed( - self, - dag_runs: Iterable[DagRun], - session: Session = NEW_SESSION, - ) -> None: - """ - Update the state of each dagrun based on the task_instance state and set unfinished runs to failed. - - :param dag_runs: DAG runs - :param session: session - :return: None - """ - for dag_run in dag_runs: - dag_run.update_state() - if dag_run.state not in State.finished_dr_states: - dag_run.set_state(DagRunState.FAILED) - session.merge(dag_run) - - @provide_session - def _execute(self, session: Session = NEW_SESSION) -> None: - """ - Initialize all required components of a dag for a specified date range and execute the tasks. - - :meta private: - """ - ti_status = BackfillJobRunner._DagRunTaskStatus() - - start_date = self.bf_start_date - - # Get DagRun schedule between the start/end dates, which will turn into dag runs. - dagrun_start_date = timezone.coerce_datetime(start_date) - if self.bf_end_date is None: - dagrun_end_date = pendulum.now(timezone.utc) - else: - dagrun_end_date = pendulum.instance(self.bf_end_date) - dagrun_infos = list(self.dag.iter_dagrun_infos_between(dagrun_start_date, dagrun_end_date)) - if self.run_backwards: - tasks_that_depend_on_past = [t.task_id for t in self.dag.task_dict.values() if t.depends_on_past] - if tasks_that_depend_on_past: - raise AirflowException( - f"You cannot backfill backwards because one or more " - f'tasks depend_on_past: {",".join(tasks_that_depend_on_past)}' - ) - dagrun_infos = dagrun_infos[::-1] - - if not dagrun_infos: - if not self.run_at_least_once: - self.log.info("No run dates were found for the given dates and dag interval.") - return - dagrun_infos = [DagRunInfo.interval(dagrun_start_date, dagrun_end_date)] - - running_dagruns = DagRun.find( - dag_id=self.dag.dag_id, - execution_start_date=self.bf_start_date, - execution_end_date=self.bf_end_date, - no_backfills=True, - state=DagRunState.RUNNING, - ) - - if running_dagruns: - for run in running_dagruns: - self.log.error( - "Backfill cannot be created for DagRun %s in %s, as there's already %s in a RUNNING " - "state.", - run.run_id, - run.execution_date.strftime("%Y-%m-%dT%H:%M:%S"), - run.run_type, - ) - self.log.error( - "Changing DagRun into BACKFILL would cause scheduler to lose track of executing " - "tasks. Not changing DagRun type into BACKFILL, and trying insert another DagRun into " - "database would cause database constraint violation for dag_id + execution_date " - "combination. Please adjust backfill dates or wait for this DagRun to finish.", - ) - return - pickle_id = None - - _support_pickling = [] - - for executor in self.job.executors: - _support_pickling.append(executor.supports_pickling) - - executor.job_id = self.job.id - executor.start() - - if not self.donot_pickle and all(_support_pickling): - pickle = DagPickle(self.dag) - session.add(pickle) - session.commit() - pickle_id = pickle.id - - ti_status.total_runs = len(dagrun_infos) # total dag runs in backfill - - try: - remaining_dates = ti_status.total_runs - while remaining_dates > 0: - dagrun_infos_to_process = [ - dagrun_info - for dagrun_info in dagrun_infos - if dagrun_info.logical_date not in ti_status.executed_dag_run_dates - ] - self._execute_dagruns( - dagrun_infos=dagrun_infos_to_process, - ti_status=ti_status, - pickle_id=pickle_id, - start_date=start_date, - session=session, - ) - - remaining_dates = ti_status.total_runs - len(ti_status.executed_dag_run_dates) - err = "".join(self._collect_errors(ti_status=ti_status, session=session)) - if err: - if not self.continue_on_failures or ti_status.deadlocked: - raise BackfillUnfinished(err, ti_status) - - if remaining_dates > 0: - self.log.info( - "max_active_runs limit for dag %s has been reached " - " - waiting for other dag runs to finish", - self.dag_id, - ) - time.sleep(self.delay_on_limit_secs) - except (KeyboardInterrupt, SystemExit): - self.log.warning("Backfill terminated by user.") - - # TODO: we will need to terminate running task instances and set the - # state to failed. - self._set_unfinished_dag_runs_to_failed(ti_status.active_runs) - except OperationalError: - self.log.exception( - "Backfill job dead-locked. The job will retry the job so it is likely " - "to heal itself. If your backfill succeeds you can ignore this exception.", - ) - raise - finally: - session.commit() - for executor in self.job.executors: - executor.end() - - self.log.info("Backfill done for DAG %s. Exiting.", self.dag) - - @provide_session - def reset_state_for_orphaned_tasks( - self, - filter_by_dag_run: DagRun | None = None, - session: Session = NEW_SESSION, - ) -> int | None: - """ - Reset state of orphaned tasks. - - This function checks if there are any tasks in the dagrun (or all) that - have a schedule or queued states but are not known by the executor. If - it finds those it will reset the state to None so they will get picked - up again. The batch option is for performance reasons as the queries - are made in sequence. - - :param filter_by_dag_run: the dag_run we want to process, None if all - :return: the number of TIs reset - """ - queued_tis = [] - running_tis = [] - for executor in self.job.executors: - queued_tis.append(executor.queued_tasks) - # also consider running as the state might not have changed in the db yet - running_tis.append(executor.running) - - # Can't use an update here since it doesn't support joins. - resettable_states = [TaskInstanceState.SCHEDULED, TaskInstanceState.QUEUED] - if filter_by_dag_run is None: - resettable_tis = ( - session.scalars( - select(TaskInstance) - .join(TaskInstance.dag_run) - .where( - DagRun.state == DagRunState.RUNNING, - DagRun.run_type != DagRunType.BACKFILL_JOB, - TaskInstance.state.in_(resettable_states), - ) - ) - ).all() - else: - resettable_tis = filter_by_dag_run.get_task_instances(state=resettable_states, session=session) - - tis_to_reset = [ti for ti in resettable_tis if ti.key not in queued_tis and ti.key not in running_tis] - if not tis_to_reset: - return 0 - - def query(result, items): - if not items: - return result - - filter_for_tis = TaskInstance.filter_for_tis(items) - reset_tis = session.scalars( - select(TaskInstance) - .where(filter_for_tis, TaskInstance.state.in_(resettable_states)) - .with_for_update() - ).all() - - for ti in reset_tis: - ti.state = None - session.merge(ti) - - return result + reset_tis - - reset_tis = helpers.reduce_in_chunks(query, tis_to_reset, [], self.job.max_tis_per_query) - - task_instance_str = "\n".join(f"\t{x!r}" for x in reset_tis) - session.flush() - - self.log.info("Reset the following %s TaskInstances:\n%s", len(reset_tis), task_instance_str) - return len(reset_tis) diff --git a/airflow/jobs/job.py b/airflow/jobs/job.py index 03bf92d4e3d53..0c2db219ef957 100644 --- a/airflow/jobs/job.py +++ b/airflow/jobs/job.py @@ -77,8 +77,6 @@ class Job(Base, LoggingMixin): The ORM class representing Job stored in the database. Jobs are processing items with state and duration that aren't task instances. - For instance a BackfillJob is a collection of task instance runs, - but should have its own state, start and end time. """ __tablename__ = "job" @@ -117,7 +115,7 @@ class Job(Base, LoggingMixin): """ TaskInstances which have been enqueued by this Job. - Only makes sense for SchedulerJob and BackfillJob instances. + Only makes sense for SchedulerJob. """ def __init__(self, executor: BaseExecutor | None = None, heartrate=None, **kwargs): diff --git a/airflow/models/dag.py b/airflow/models/dag.py index f5def92ea92a7..1dfb3b2e9114d 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2381,8 +2381,7 @@ def add_logger_if_needed(ti: TaskInstance): tasks = self.task_dict self.log.debug("starting dagrun") # Instead of starting a scheduler, we run the minimal loop possible to check - # for task readiness and dependency management. This is notably faster - # than creating a BackfillJob and allows us to surface logs to the user + # for task readiness and dependency management. # ``Dag.test()`` works in two different modes depending on ``use_executor``: # - if ``use_executor`` is False, runs the task locally with no executor using ``_run_task`` diff --git a/airflow/models/dagpickle.py b/airflow/models/dagpickle.py index e6f4561d8e1bf..c06ef09709f1c 100644 --- a/airflow/models/dagpickle.py +++ b/airflow/models/dagpickle.py @@ -32,7 +32,7 @@ class DagPickle(Base): """ - Represents a version of a DAG and becomes a source of truth for a BackfillJob execution. + Represents a version of a DAG and becomes a source of truth for an execution. Dags can originate from different places (user repos, main repo, ...) and also get executed in different places (different executors). A pickle is a native python serialized object, diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index 0373bc667bfc6..cad82e72b8b2d 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -1111,7 +1111,7 @@ def notify_dagrun_state_changed(self, msg: str = ""): elif self.state == DagRunState.FAILED: get_listener_manager().hook.on_dag_run_failed(dag_run=self, msg=msg) # deliberately not notifying on QUEUED - # we can't get all the state changes on SchedulerJob, BackfillJob + # we can't get all the state changes on SchedulerJob, # or LocalTaskJob, so we don't want to "falsely advertise" we notify about that def _get_ready_tis( diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index e75ad83923bf2..5b51bb0d24df8 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -2640,7 +2640,7 @@ def _check_and_change_state_before_execution( :param mark_success: Don't run the task, mark its state as success :param test_mode: Doesn't record success or failure in the DB :param hostname: The hostname of the worker running the task instance. - :param job_id: Job (BackfillJob / LocalTaskJob / SchedulerJob) ID + :param job_id: Job (LocalTaskJob / SchedulerJob) ID :param pool: specifies the pool to use to run the task instance :param external_executor_id: The identifier of the celery executor :param session: SQLAlchemy ORM Session diff --git a/airflow/ti_deps/dependencies_deps.py b/airflow/ti_deps/dependencies_deps.py index 44d6bfc5c7db7..c167cdb346325 100644 --- a/airflow/ti_deps/dependencies_deps.py +++ b/airflow/ti_deps/dependencies_deps.py @@ -17,7 +17,6 @@ from __future__ import annotations from airflow.ti_deps.dependencies_states import ( - BACKFILL_QUEUEABLE_STATES, QUEUEABLE_STATES, RUNNABLE_STATES, ) @@ -48,13 +47,6 @@ TaskNotRunningDep(), } -BACKFILL_QUEUED_DEPS = { - RunnableExecDateDep(), - ValidStateDep(BACKFILL_QUEUEABLE_STATES), - DagrunRunningDep(), - TaskNotRunningDep(), -} - # TODO(aoen): SCHEDULER_QUEUED_DEPS is not coupled to actual scheduling/execution # in any way and could easily be modified or removed from the scheduler causing # this dependency to become outdated and incorrect. This coupling should be created diff --git a/airflow/ti_deps/dependencies_states.py b/airflow/ti_deps/dependencies_states.py index fd25d62f6d75e..ebf581ab48e18 100644 --- a/airflow/ti_deps/dependencies_states.py +++ b/airflow/ti_deps/dependencies_states.py @@ -42,12 +42,3 @@ QUEUEABLE_STATES = { TaskInstanceState.SCHEDULED, } - -BACKFILL_QUEUEABLE_STATES = { - # For cases like unit tests and run manually - None, - TaskInstanceState.UP_FOR_RESCHEDULE, - TaskInstanceState.UP_FOR_RETRY, - # For normal backfill cases - TaskInstanceState.SCHEDULED, -} diff --git a/docs/apache-airflow/administration-and-deployment/listeners.rst b/docs/apache-airflow/administration-and-deployment/listeners.rst index 1fca915a6f1df..8ca3ed93fc091 100644 --- a/docs/apache-airflow/administration-and-deployment/listeners.rst +++ b/docs/apache-airflow/administration-and-deployment/listeners.rst @@ -34,7 +34,7 @@ Lifecycle Events - ``on_starting`` - ``before_stopping`` -Lifecycle events allow you to react to start and stop events for an Airflow ``Job``, like ``SchedulerJob`` or ``BackfillJob``. +Lifecycle events allow you to react to start and stop events for an Airflow ``Job``, like ``SchedulerJob``. DagRun State Change Events -------------------------- diff --git a/docs/apache-airflow/howto/listener-plugin.rst b/docs/apache-airflow/howto/listener-plugin.rst index 7b46a9de8a969..7d5fb430fb486 100644 --- a/docs/apache-airflow/howto/listener-plugin.rst +++ b/docs/apache-airflow/howto/listener-plugin.rst @@ -44,8 +44,8 @@ Using this plugin, following events can be listened: * dag run is in running state. * dag run is in success state. * dag run is in failure state. - * on start before event like airflow job, scheduler or backfilljob - * before stop for event like airflow job, scheduler or backfilljob + * on start before event like airflow job, scheduler + * before stop for event like airflow job, scheduler Listener Registration --------------------- diff --git a/tests/core/test_impersonation_tests.py b/tests/core/test_impersonation_tests.py index cf9359c863372..8350e95a8f4fd 100644 --- a/tests/core/test_impersonation_tests.py +++ b/tests/core/test_impersonation_tests.py @@ -28,13 +28,10 @@ import pytest from airflow.configuration import conf -from airflow.jobs.backfill_job_runner import BackfillJobRunner -from airflow.jobs.job import Job, run_job -from airflow.models import DagBag, DagRun, TaskInstance +from airflow.models import DagBag, TaskInstance from airflow.utils.db import add_default_pool_if_not_exists from airflow.utils.state import State from airflow.utils.timezone import datetime -from airflow.utils.types import DagRunType from dev.tests_common.test_utils import db @@ -175,17 +172,14 @@ def get_dagbag(dag_folder): logger.info(dagbag.dagbag_report()) return dagbag - def run_backfill(self, dag_id, task_id): + def run_dag(self, dag_id, task_id): dag = self.dagbag.get_dag(dag_id) dag.clear() - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag, start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - run_job(job=job, execute_callable=job_runner._execute) - run_id = DagRun.generate_run_id(DagRunType.BACKFILL_JOB, execution_date=DEFAULT_DATE) - ti = TaskInstance(task=dag.get_task(task_id), run_id=run_id) - ti.refresh_from_db() + dr = dag.test(use_executor=True) + ti = TaskInstance(task=dag.get_task(task_id), run_id=dr.run_id) + ti.refresh_from_db() assert ti.state == State.SUCCESS @@ -198,14 +192,14 @@ def test_impersonation(self): """ Tests that impersonating a unix user works """ - self.run_backfill("test_impersonation", "test_impersonated_user") + self.run_dag("test_impersonation", "test_impersonated_user") def test_no_impersonation(self): """ If default_impersonation=None, tests that the job is run as the current user (which will be a sudoer) """ - self.run_backfill( + self.run_dag( "test_no_impersonation", "test_superuser", ) @@ -216,7 +210,7 @@ def test_default_impersonation(self, monkeypatch): to running as TEST_USER for a test without 'run_as_user' set. """ monkeypatch.setenv("AIRFLOW__CORE__DEFAULT_IMPERSONATION", TEST_USER) - self.run_backfill("test_default_impersonation", "test_deelevated_user") + self.run_dag("test_default_impersonation", "test_deelevated_user") class TestImpersonationWithCustomPythonPath(BaseImpersonationTest): @@ -233,4 +227,4 @@ def test_impersonation_custom(self, monkeypatch): """ monkeypatch.setenv("PYTHONPATH", TEST_UTILS_FOLDER) assert TEST_UTILS_FOLDER not in sys.path - self.run_backfill("impersonation_with_custom_pkg", "exec_python_fn") + self.run_dag("impersonation_with_custom_pkg", "exec_python_fn") diff --git a/tests/jobs/test_backfill_job.py b/tests/jobs/test_backfill_job.py deleted file mode 100644 index 616f328ee4177..0000000000000 --- a/tests/jobs/test_backfill_job.py +++ /dev/null @@ -1,2094 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import datetime -import json -import logging -import threading -from collections import defaultdict -from importlib import reload -from unittest import mock -from unittest.mock import Mock, patch - -import pytest - -from airflow import settings -from airflow.exceptions import ( - AirflowException, - BackfillUnfinished, - DagConcurrencyLimitReached, - NoAvailablePoolSlot, - TaskConcurrencyLimitReached, - UnknownExecutorException, -) -from airflow.executors.debug_executor import DebugExecutor -from airflow.executors.executor_loader import ExecutorLoader -from airflow.executors.sequential_executor import SequentialExecutor -from airflow.jobs.backfill_job_runner import BackfillJobRunner -from airflow.jobs.job import Job, run_job -from airflow.listeners.listener import get_listener_manager -from airflow.models import DagBag, Pool, TaskInstance as TI -from airflow.models.dagrun import DagRun -from airflow.models.serialized_dag import SerializedDagModel -from airflow.models.taskinstancekey import TaskInstanceKey -from airflow.models.taskmap import TaskMap -from airflow.operators.empty import EmptyOperator -from airflow.utils import timezone -from airflow.utils.session import create_session -from airflow.utils.state import DagRunState, State, TaskInstanceState -from airflow.utils.trigger_rule import TriggerRule -from airflow.utils.types import DagRunType -from tests.listeners import dag_listener -from tests.models import TEST_DAGS_FOLDER - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import ( - clear_db_dags, - clear_db_pools, - clear_db_runs, - clear_db_xcom, - set_default_pool_slots, -) -from dev.tests_common.test_utils.mock_executor import MockExecutor - -if AIRFLOW_V_3_0_PLUS: - from airflow.utils.types import DagRunTriggeredByType - -pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] - -logger = logging.getLogger(__name__) - -DEFAULT_DATE = timezone.datetime(2016, 1, 1) -DEFAULT_DAG_RUN_ID = "test1" - - -@pytest.fixture(scope="module") -def dag_bag(): - return DagBag(include_examples=True) - - -class SecondaryMockExecutor(MockExecutor): - """Copy of MockExecutor class with a new name for testing with hybrid executors (which currently - disallows using the same executor concurrently)""" - - -def _mock_executor(executor=None): - if not executor: - default_executor = MockExecutor() - else: - if isinstance(executor, type): - default_executor = executor() - else: - default_executor = executor - - default_executor.name = mock.MagicMock( - alias="default_exec", - module_path=f"{default_executor.__module__}.{default_executor.__class__.__qualname__}", - ) - with mock.patch("airflow.jobs.job.Job.executors", new_callable=mock.PropertyMock) as executors_mock: - with mock.patch("airflow.jobs.job.Job.executor", new_callable=mock.PropertyMock) as executor_mock: - with mock.patch("airflow.executors.executor_loader.ExecutorLoader.load_executor") as loader_mock: - executor_mock.return_value = default_executor - executors_mock.return_value = [default_executor] - # The executor is mocked, so cannot be loaded/imported. Mock load_executor and return the - # correct object for the given input executor name. - loader_mock.side_effect = lambda *x: { - ("default_exec",): default_executor, - ("default.exec.module.path",): default_executor, - (None,): default_executor, - }[x] - - # Reload the job runner so that it gets a fresh instances of the mocked executor loader - from airflow.jobs import backfill_job_runner - - reload(backfill_job_runner) - - yield default_executor - - -@pytest.mark.execution_timeout(120) -class TestBackfillJob: - @pytest.fixture - def mock_executor(self): - yield from _mock_executor() - - def _mock_executors(self): - default_executor = MockExecutor() - _default_executor = Mock(wraps=default_executor) - default_alias = "default_exec" - default_module_path = f"{default_executor.__module__}.{default_executor.__class__.__qualname__}" - _default_executor.name = mock.MagicMock(alias=default_alias, module_path=default_module_path) - - secondary_executor = SecondaryMockExecutor() - _secondary_executor = Mock(wraps=secondary_executor) - secondary_alias = "secondary_exec" - secondary_module_path = f"{secondary_executor.__module__}.{secondary_executor.__class__.__qualname__}" - _secondary_executor.name = mock.MagicMock(alias=secondary_alias, module_path=secondary_module_path) - - with mock.patch( - "airflow.jobs.job.Job.executors", new_callable=mock.PropertyMock - ) as executors_mock, mock.patch( - "airflow.jobs.job.Job.executor", new_callable=mock.PropertyMock - ) as executor_mock, mock.patch( - "airflow.executors.executor_loader.ExecutorLoader.load_executor" - ) as loader_mock, conf_vars( - { - ( - "core", - "executor", - ): f"{default_alias}:{default_module_path},{secondary_alias}:{secondary_module_path}" - } - ): - # The executor is mocked, so cannot be loaded/imported. Mock load_executor and return the - # correct object for the given input executor name. - loader_mock.side_effect = lambda *x: { - (_secondary_executor.name.alias,): _secondary_executor, - (_secondary_executor.name.module_path,): _secondary_executor, - (default_alias,): _default_executor, - (default_module_path,): _default_executor, - (None,): _default_executor, - }[x] - - executor_mock.return_value = _default_executor - executors_mock.return_value = [_default_executor, _secondary_executor] - - yield (_default_executor, _secondary_executor) - - @pytest.fixture - def mock_executors(self): - yield from self._mock_executors() - - @staticmethod - def clean_db(): - clear_db_dags() - clear_db_runs() - clear_db_xcom() - clear_db_pools() - - @pytest.fixture(autouse=True) - def set_instance_attrs(self, dag_bag): - self.clean_db() - self.dagbag = dag_bag - # `airflow tasks run` relies on serialized_dag - for dag in self.dagbag.dags.values(): - SerializedDagModel.write_dag(dag) - - def _get_dummy_dag( - self, - dag_maker_fixture, - dag_id="test_dag", - pool=Pool.DEFAULT_POOL_NAME, - max_active_tis_per_dag=None, - task_id="op", - **kwargs, - ): - with dag_maker_fixture(dag_id=dag_id, schedule="@daily", **kwargs) as dag: - EmptyOperator(task_id=task_id, pool=pool, max_active_tis_per_dag=max_active_tis_per_dag) - - return dag - - def _times_called_with(self, method, class_): - count = 0 - for args in method.call_args_list: - if isinstance(args[0][0], class_): - count += 1 - return count - - def test_unfinished_dag_runs_set_to_failed(self, dag_maker): - dag = self._get_dummy_dag(dag_maker) - dag_run = dag_maker.create_dagrun(state=None) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=8), - ignore_first_depends_on_past=True, - ) - - job_runner._set_unfinished_dag_runs_to_failed([dag_run]) - dag_run.refresh_from_db() - - assert State.FAILED == dag_run.state - - def test_dag_run_with_finished_tasks_set_to_success(self, dag_maker, mock_executor): - dag = self._get_dummy_dag(dag_maker) - dag_run = dag_maker.create_dagrun(state=None) - - for ti in dag_run.get_task_instances(): - ti.set_state(State.SUCCESS) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=8), - ignore_first_depends_on_past=True, - ) - job_runner._set_unfinished_dag_runs_to_failed([dag_run]) - - dag_run.refresh_from_db() - - assert State.SUCCESS == dag_run.state - - @pytest.mark.backend("postgres", "mysql") - def test_trigger_controller_dag(self, session): - dag = self.dagbag.get_dag("example_trigger_controller_dag") - target_dag = self.dagbag.get_dag("example_trigger_target_dag") - target_dag.sync_to_db() - - target_dag_run = session.query(DagRun).filter(DagRun.dag_id == target_dag.dag_id).one_or_none() - assert target_dag_run is None - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - ignore_first_depends_on_past=True, - ) - - run_job(job=job, execute_callable=job_runner._execute) - - dag_run = session.query(DagRun).filter(DagRun.dag_id == dag.dag_id).one_or_none() - assert dag_run is not None - - task_instances_list = job_runner._task_instances_for_dag_run(dag=dag, dag_run=dag_run) - - assert task_instances_list - - @pytest.mark.backend("postgres", "mysql") - def test_backfill_multi_dates(self, mock_executor): - dag = self.dagbag.get_dag("miscellaneous_test_dag") - - end_date = DEFAULT_DATE + datetime.timedelta(days=1) - - job = Job() - executor = job.executor - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=end_date, - ignore_first_depends_on_past=True, - ) - - run_job(job=job, execute_callable=job_runner._execute) - expected_execution_order = [ - ("runme_0", DEFAULT_DATE), - ("runme_1", DEFAULT_DATE), - ("runme_2", DEFAULT_DATE), - ("runme_0", end_date), - ("runme_1", end_date), - ("runme_2", end_date), - ("also_run_this", DEFAULT_DATE), - ("also_run_this", end_date), - ("run_after_loop", DEFAULT_DATE), - ("run_after_loop", end_date), - ("run_this_last", DEFAULT_DATE), - ("run_this_last", end_date), - ] - actual = [(tuple(x), y) for x, y in executor.sorted_tasks] - expected = [ - ( - (dag.dag_id, task_id, f"backfill__{when.isoformat()}", 1, -1), - (State.SUCCESS, None), - ) - for (task_id, when) in expected_execution_order - ] - assert actual == expected - session = settings.Session() - drs = session.query(DagRun).filter(DagRun.dag_id == dag.dag_id).order_by(DagRun.execution_date).all() - - assert drs[0].execution_date == DEFAULT_DATE - assert drs[0].state == State.SUCCESS - assert drs[1].execution_date == DEFAULT_DATE + datetime.timedelta(days=1) - assert drs[1].state == State.SUCCESS - - dag.clear() - session.close() - - @pytest.mark.backend("postgres", "mysql") - @pytest.mark.parametrize( - "dag_id, expected_execution_order", - [ - [ - "example_branch_operator", - ( - "run_this_first", - "branching", - "branch_a", - "branch_b", - "branch_c", - "branch_d", - "follow_a", - "follow_b", - "follow_c", - "follow_d", - "join", - "branching_ext_python", - "ext_py_a", - "ext_py_b", - "ext_py_c", - "ext_py_d", - "join_ext_python", - "branching_venv", - "venv_a", - "venv_b", - "venv_c", - "venv_d", - "join_venv", - ), - ], - [ - "miscellaneous_test_dag", - ("runme_0", "runme_1", "runme_2", "also_run_this", "run_after_loop", "run_this_last"), - ], - [ - "example_skip_dag", - ( - "always_true_1", - "always_true_2", - "skip_operator_1", - "skip_operator_2", - "all_success", - "one_success", - "final_1", - "final_2", - ), - ], - ["latest_only", ("latest_only", "task1")], - ], - ) - def test_backfill_examples(self, dag_id, expected_execution_order, mock_executor): - """ - Test backfilling example dags - - Try to backfill some of the example dags. Be careful, not all dags are suitable - for doing this. For example, a dag that sleeps forever, or does not have a - schedule won't work here since you simply can't backfill them. - """ - dag = self.dagbag.get_dag(dag_id) - - logger.info("*** Running example DAG: %s", dag.dag_id) - job = Job() - executor = job.executor - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - ignore_first_depends_on_past=True, - ) - - run_job(job=job, execute_callable=job_runner._execute) - assert [ - ((dag_id, task_id, f"backfill__{DEFAULT_DATE.isoformat()}", 1, -1), (State.SUCCESS, None)) - for task_id in expected_execution_order - ] == executor.sorted_tasks - - def test_backfill_conf(self, dag_maker, mock_executor): - dag = self._get_dummy_dag(dag_maker, dag_id="test_backfill_conf") - dag_maker.create_dagrun(state=None) - - conf_ = json.loads("""{"key": "value"}""") - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - conf=conf_, - ) - run_job(job=job, execute_callable=job_runner._execute) - - # We ignore the first dag_run created by fixture - dr = DagRun.find( - dag_id="test_backfill_conf", execution_start_date=DEFAULT_DATE + datetime.timedelta(days=1) - ) - - assert conf_ == dr[0].conf - - def test_backfill_respect_max_active_tis_per_dag_limit(self, dag_maker, mock_executor): - max_active_tis_per_dag = 2 - dag = self._get_dummy_dag( - dag_maker, - dag_id="test_backfill_respect_max_active_tis_per_dag_limit", - max_active_tis_per_dag=max_active_tis_per_dag, - ) - dag_maker.create_dagrun(state=None) - - job = Job() - executor = job.executor - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=7), - ) - - mock_log = Mock() - job_runner._log = mock_log - - run_job(job=job, execute_callable=job_runner._execute) - - assert len(executor.history) > 0 - - task_concurrency_limit_reached_at_least_once = False - - num_running_task_instances = 0 - for running_task_instances in executor.history: - assert len(running_task_instances) <= max_active_tis_per_dag - num_running_task_instances += len(running_task_instances) - if len(running_task_instances) == max_active_tis_per_dag: - task_concurrency_limit_reached_at_least_once = True - - assert 8 == num_running_task_instances - assert task_concurrency_limit_reached_at_least_once - - times_dag_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - DagConcurrencyLimitReached, - ) - - times_pool_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - NoAvailablePoolSlot, - ) - - times_task_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - TaskConcurrencyLimitReached, - ) - - assert 0 == times_pool_limit_reached_in_debug - assert 0 == times_dag_concurrency_limit_reached_in_debug - assert times_task_concurrency_limit_reached_in_debug > 0 - - @pytest.mark.parametrize("with_max_active_tis_per_dag", [False, True]) - def test_backfill_respect_max_active_tis_per_dagrun_limit( - self, dag_maker, with_max_active_tis_per_dag, mock_executor - ): - max_active_tis_per_dag = 3 - max_active_tis_per_dagrun = 2 - kwargs = {"max_active_tis_per_dagrun": max_active_tis_per_dagrun} - if with_max_active_tis_per_dag: - kwargs["max_active_tis_per_dag"] = max_active_tis_per_dag - - with dag_maker(dag_id="test_backfill_respect_max_active_tis_per_dag_limit", schedule="@daily") as dag: - EmptyOperator.partial(task_id="task1", **kwargs).expand_kwargs([{"x": i} for i in range(10)]) - - dag_maker.create_dagrun(state=None) - - job = Job() - executor = job.executor - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=7), - ) - - mock_log = Mock() - job_runner._log = mock_log - - run_job(job=job, execute_callable=job_runner._execute) - - assert len(executor.history) > 0 - - task_concurrency_limit_reached_at_least_once = False - - def get_running_tis_per_dagrun(running_tis): - running_tis_per_dagrun_dict = defaultdict(int) - for running_ti in running_tis: - running_tis_per_dagrun_dict[running_ti[3].dag_run.id] += 1 - return running_tis_per_dagrun_dict - - num_running_task_instances = 0 - for running_task_instances in executor.history: - if with_max_active_tis_per_dag: - assert len(running_task_instances) <= max_active_tis_per_dag - running_tis_per_dagrun_dict = get_running_tis_per_dagrun(running_task_instances) - assert all( - [ - num_running_tis <= max_active_tis_per_dagrun - for num_running_tis in running_tis_per_dagrun_dict.values() - ] - ) - num_running_task_instances += len(running_task_instances) - task_concurrency_limit_reached_at_least_once = ( - task_concurrency_limit_reached_at_least_once - or any( - [ - num_running_tis == max_active_tis_per_dagrun - for num_running_tis in running_tis_per_dagrun_dict.values() - ] - ) - ) - - assert 80 == num_running_task_instances # (7 backfill run + 1 manual run ) * 10 mapped task per run - assert task_concurrency_limit_reached_at_least_once - - times_dag_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - DagConcurrencyLimitReached, - ) - - times_pool_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - NoAvailablePoolSlot, - ) - - times_task_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - TaskConcurrencyLimitReached, - ) - - assert 0 == times_pool_limit_reached_in_debug - assert 0 == times_dag_concurrency_limit_reached_in_debug - assert times_task_concurrency_limit_reached_in_debug > 0 - - def test_backfill_respect_dag_concurrency_limit(self, dag_maker, mock_executor): - dag = self._get_dummy_dag(dag_maker, dag_id="test_backfill_respect_concurrency_limit") - dag_maker.create_dagrun(state=None) - dag.max_active_tasks = 2 - - job = Job() - executor = job.executor - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=7), - ) - - mock_log = Mock() - job_runner._log = mock_log - - run_job(job=job, execute_callable=job_runner._execute) - - assert len(executor.history) > 0 - - concurrency_limit_reached_at_least_once = False - - num_running_task_instances = 0 - - for running_task_instances in executor.history: - assert len(running_task_instances) <= dag.max_active_tasks - num_running_task_instances += len(running_task_instances) - if len(running_task_instances) == dag.max_active_tasks: - concurrency_limit_reached_at_least_once = True - - assert 8 == num_running_task_instances - assert concurrency_limit_reached_at_least_once - - times_dag_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - DagConcurrencyLimitReached, - ) - - times_pool_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - NoAvailablePoolSlot, - ) - - times_task_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - TaskConcurrencyLimitReached, - ) - - assert 0 == times_pool_limit_reached_in_debug - assert 0 == times_task_concurrency_limit_reached_in_debug - assert times_dag_concurrency_limit_reached_in_debug > 0 - - def test_backfill_respect_default_pool_limit(self, dag_maker, mock_executor): - default_pool_slots = 2 - set_default_pool_slots(default_pool_slots) - - dag = self._get_dummy_dag(dag_maker, dag_id="test_backfill_with_no_pool_limit") - dag_maker.create_dagrun(state=None) - - job = Job() - executor = job.executor - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=7), - ) - - mock_log = Mock() - job_runner._log = mock_log - - run_job(job=job, execute_callable=job_runner._execute) - - assert len(executor.history) > 0 - - default_pool_task_slot_count_reached_at_least_once = False - - num_running_task_instances = 0 - - # if no pool is specified, the number of tasks running in - # parallel per backfill should be less than - # default_pool slots at any point of time. - for running_task_instances in executor.history: - assert len(running_task_instances) <= default_pool_slots - num_running_task_instances += len(running_task_instances) - if len(running_task_instances) == default_pool_slots: - default_pool_task_slot_count_reached_at_least_once = True - - assert 8 == num_running_task_instances - assert default_pool_task_slot_count_reached_at_least_once - - times_dag_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - DagConcurrencyLimitReached, - ) - - times_pool_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - NoAvailablePoolSlot, - ) - - times_task_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - TaskConcurrencyLimitReached, - ) - - assert 0 == times_dag_concurrency_limit_reached_in_debug - assert 0 == times_task_concurrency_limit_reached_in_debug - assert times_pool_limit_reached_in_debug > 0 - - def test_backfill_pool_not_found(self, dag_maker, mock_executor): - dag = self._get_dummy_dag( - dag_maker, - dag_id="test_backfill_pool_not_found", - pool="king_pool", - ) - dag_maker.create_dagrun(state=None) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=7), - ) - - try: - run_job(job=job, execute_callable=job_runner._execute) - except AirflowException: - return - - def test_backfill_respect_pool_limit(self, dag_maker, mock_executor): - session = settings.Session() - - slots = 2 - pool = Pool( - pool="pool_with_two_slots", - slots=slots, - include_deferred=False, - ) - session.add(pool) - session.commit() - - dag = self._get_dummy_dag( - dag_maker, - dag_id="test_backfill_respect_pool_limit", - pool=pool.pool, - ) - dag_maker.create_dagrun(state=None) - - job = Job() - executor = job.executor - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=7), - ) - - mock_log = Mock() - job_runner._log = mock_log - - run_job(job=job, execute_callable=job_runner._execute) - - assert len(executor.history) > 0 - - pool_was_full_at_least_once = False - num_running_task_instances = 0 - - for running_task_instances in executor.history: - assert len(running_task_instances) <= slots - num_running_task_instances += len(running_task_instances) - if len(running_task_instances) == slots: - pool_was_full_at_least_once = True - - assert 8 == num_running_task_instances - assert pool_was_full_at_least_once - - times_dag_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - DagConcurrencyLimitReached, - ) - - times_pool_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - NoAvailablePoolSlot, - ) - - times_task_concurrency_limit_reached_in_debug = self._times_called_with( - mock_log.debug, - TaskConcurrencyLimitReached, - ) - - assert 0 == times_task_concurrency_limit_reached_in_debug - assert 0 == times_dag_concurrency_limit_reached_in_debug - assert times_pool_limit_reached_in_debug > 0 - - def test_backfill_run_rescheduled(self, dag_maker, mock_executor): - dag = self._get_dummy_dag( - dag_maker, dag_id="test_backfill_run_rescheduled", task_id="test_backfill_run_rescheduled_task-1" - ) - dag_maker.create_dagrun(state=None, run_id=DEFAULT_DAG_RUN_ID) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - ) - run_job(job=job, execute_callable=job_runner._execute) - - ti = TI(task=dag.get_task("test_backfill_run_rescheduled_task-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - ti.set_state(State.UP_FOR_RESCHEDULE) - - for _ in _mock_executor(): - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - rerun_failed_tasks=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - ti = TI(task=dag.get_task("test_backfill_run_rescheduled_task-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - assert ti.state == State.SUCCESS - - def test_backfill_override_conf(self, dag_maker, mock_executor): - dag = self._get_dummy_dag( - dag_maker, dag_id="test_backfill_override_conf", task_id="test_backfill_override_conf-1" - ) - dr = dag_maker.create_dagrun( - state=None, - start_date=DEFAULT_DATE, - ) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - conf={"a": 1}, - ) - - with patch.object( - job_runner, - "_task_instances_for_dag_run", - wraps=job_runner._task_instances_for_dag_run, - ) as wrapped_task_instances_for_dag_run: - run_job(job=job, execute_callable=job_runner._execute) - dr = wrapped_task_instances_for_dag_run.call_args_list[0][0][1] - assert dr.conf == {"a": 1} - - def test_backfill_skip_active_scheduled_dagrun(self, dag_maker, caplog, mock_executor): - dag = self._get_dummy_dag( - dag_maker, - dag_id="test_backfill_skip_active_scheduled_dagrun", - task_id="test_backfill_skip_active_scheduled_dagrun-1", - ) - dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED, state=State.RUNNING, run_id=DEFAULT_DAG_RUN_ID) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - ) - with caplog.at_level(logging.ERROR, logger="airflow.jobs.backfill_job_runner.BackfillJob"): - caplog.clear() - run_job(job=job, execute_callable=job_runner._execute) - assert "Backfill cannot be created for DagRun" in caplog.messages[0] - - ti = TI(task=dag.get_task("test_backfill_skip_active_scheduled_dagrun-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - # since DAG backfill is skipped, task state should be none - assert ti.state == State.NONE - - def test_backfill_rerun_failed_tasks(self, dag_maker, mock_executor): - dag = self._get_dummy_dag( - dag_maker, dag_id="test_backfill_rerun_failed", task_id="test_backfill_rerun_failed_task-1" - ) - dag_maker.create_dagrun(state=None, run_id=DEFAULT_DAG_RUN_ID) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - ) - run_job(job=job, execute_callable=job_runner._execute) - - ti = TI(task=dag.get_task("test_backfill_rerun_failed_task-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - ti.set_state(State.FAILED) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - rerun_failed_tasks=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - ti = TI(task=dag.get_task("test_backfill_rerun_failed_task-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - assert ti.state == State.SUCCESS - - def test_backfill_rerun_upstream_failed_tasks(self, dag_maker, mock_executor): - with dag_maker(dag_id="test_backfill_rerun_upstream_failed", schedule="@daily") as dag: - op1 = EmptyOperator(task_id="test_backfill_rerun_upstream_failed_task-1") - op2 = EmptyOperator(task_id="test_backfill_rerun_upstream_failed_task-2") - op1.set_upstream(op2) - dag_maker.create_dagrun(state=None, run_id=DEFAULT_DAG_RUN_ID) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - ) - run_job(job=job, execute_callable=job_runner._execute) - - ti = TI(task=dag.get_task("test_backfill_rerun_upstream_failed_task-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - ti.set_state(State.UPSTREAM_FAILED) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - rerun_failed_tasks=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - ti = TI(task=dag.get_task("test_backfill_rerun_upstream_failed_task-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - assert ti.state == State.SUCCESS - - def test_backfill_rerun_failed_tasks_without_flag(self, dag_maker, mock_executor): - dag = self._get_dummy_dag( - dag_maker, dag_id="test_backfill_rerun_failed", task_id="test_backfill_rerun_failed_task-1" - ) - dag_maker.create_dagrun(state=None, run_id=DEFAULT_DAG_RUN_ID) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - ) - run_job(job=job, execute_callable=job_runner._execute) - - ti = TI(task=dag.get_task("test_backfill_rerun_failed_task-1"), run_id=DEFAULT_DAG_RUN_ID) - ti.refresh_from_db() - ti.set_state(State.FAILED) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - rerun_failed_tasks=False, - ) - - with pytest.raises(AirflowException): - run_job(job=job, execute_callable=job_runner._execute) - - def test_backfill_retry_intermittent_failed_task(self, dag_maker, mock_executor): - with dag_maker( - dag_id="test_intermittent_failure_job", - schedule="@daily", - default_args={ - "retries": 2, - "retry_delay": datetime.timedelta(seconds=0), - }, - ) as dag: - task1 = EmptyOperator(task_id="task1") - dag_maker.create_dagrun(state=None) - - executor = mock_executor - executor.mock_task_results[TaskInstanceKey(dag.dag_id, task1.task_id, DEFAULT_DATE, try_number=1)] = ( - State.UP_FOR_RETRY - ) - executor.mock_task_results[TaskInstanceKey(dag.dag_id, task1.task_id, DEFAULT_DATE, try_number=2)] = ( - State.UP_FOR_RETRY - ) - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - ) - run_job(job=job, execute_callable=job_runner._execute) - - def test_backfill_retry_always_failed_task(self, dag_maker, mock_executor): - with dag_maker( - dag_id="test_always_failure_job", - schedule="@daily", - default_args={ - "retries": 1, - "retry_delay": datetime.timedelta(seconds=0), - }, - ) as dag: - task1 = EmptyOperator(task_id="task1") - dr = dag_maker.create_dagrun(state=None) - - executor = mock_executor - executor.mock_task_results[TaskInstanceKey(dag.dag_id, task1.task_id, dr.run_id, try_number=0)] = ( - State.UP_FOR_RETRY - ) - executor.mock_task_fail(dag.dag_id, task1.task_id, dr.run_id, try_number=1) - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - ) - with pytest.raises(BackfillUnfinished): - run_job(job=job, execute_callable=job_runner._execute) - - def test_backfill_ordered_concurrent_execute(self, dag_maker, mock_executor): - with dag_maker( - dag_id="test_backfill_ordered_concurrent_execute", - schedule="@daily", - ) as dag: - op1 = EmptyOperator(task_id="leave1") - op2 = EmptyOperator(task_id="leave2") - op3 = EmptyOperator(task_id="upstream_level_1") - op4 = EmptyOperator(task_id="upstream_level_2") - op5 = EmptyOperator(task_id="upstream_level_3") - # order randomly - op2.set_downstream(op3) - op1.set_downstream(op3) - op4.set_downstream(op5) - op3.set_downstream(op4) - runid0 = f"backfill__{DEFAULT_DATE.isoformat()}" - dag_maker.create_dagrun(run_id=runid0) - - executor = mock_executor - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=2), - ) - run_job(job=job, execute_callable=job_runner._execute) - - runid1 = f"backfill__{(DEFAULT_DATE + datetime.timedelta(days=1)).isoformat()}" - runid2 = f"backfill__{(DEFAULT_DATE + datetime.timedelta(days=2)).isoformat()}" - - actual = [] - for batch in executor.history: - this_batch = [] - for cmd, idx, queue, ti in batch: # noqa: B007 - key = ti.key - this_batch.append((key.task_id, key.run_id)) - actual.append(sorted(this_batch)) - assert actual == [ - [ - ("leave1", runid0), - ("leave1", runid1), - ("leave1", runid2), - ("leave2", runid0), - ("leave2", runid1), - ("leave2", runid2), - ], - [ - ("upstream_level_1", runid0), - ("upstream_level_1", runid1), - ("upstream_level_1", runid2), - ], - [ - ("upstream_level_2", runid0), - ("upstream_level_2", runid1), - ("upstream_level_2", runid2), - ], - [ - ("upstream_level_3", runid0), - ("upstream_level_3", runid1), - ("upstream_level_3", runid2), - ], - ] - - @pytest.mark.parametrize("ignore_depends_on_past", [True, False]) - def test_backfill_depends_on_past_works_independently_on_ignore_depends_on_past( - self, ignore_depends_on_past, mock_executor - ): - dag = self.dagbag.get_dag("test_depends_on_past") - dag.clear() - run_date = DEFAULT_DATE + datetime.timedelta(days=5) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=run_date, - end_date=run_date, - ignore_first_depends_on_past=ignore_depends_on_past, - ) - run_job(job=job, execute_callable=job_runner._execute) - - run_id = f"backfill__{run_date.isoformat()}" - # ti should have succeeded - ti = TI(dag.tasks[0], run_id=run_id) - ti.refresh_from_db() - assert ti.state == State.SUCCESS - - def test_backfill_depends_on_past_backwards(self, mock_executor): - """ - Test that CLI respects -B argument and raises on interaction with depends_on_past - """ - dag_id = "test_depends_on_past" - start_date = DEFAULT_DATE + datetime.timedelta(days=1) - end_date = start_date + datetime.timedelta(days=1) - kwargs = dict( - start_date=start_date, - end_date=end_date, - ) - dag = self.dagbag.get_dag(dag_id) - dag.clear() - - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag, ignore_first_depends_on_past=True, **kwargs) - run_job(job=job, execute_callable=job_runner._execute) - - run_id = f"backfill__{end_date.isoformat()}" - ti = TI(dag.get_task("test_dop_task"), run_id=run_id) - ti.refresh_from_db() - # runs fine forwards - assert ti.state == State.SUCCESS - - # raises backwards - expected_msg = "You cannot backfill backwards because one or more tasks depend_on_past: test_dop_task" - - for _ in _mock_executor(): - # Mock again to get a new executor - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag, run_backwards=True, **kwargs) - with pytest.raises(AirflowException, match=expected_msg): - run_job(job=job, execute_callable=job_runner._execute) - - def _get_dag_test_max_active_limits( - self, dag_maker_fixture, dag_id="test_dag", max_active_runs=1, **kwargs - ): - with dag_maker_fixture( - dag_id=dag_id, - schedule="@hourly", - max_active_runs=max_active_runs, - **kwargs, - ) as dag: - op1 = EmptyOperator(task_id="leave1") - op2 = EmptyOperator(task_id="leave2") - op3 = EmptyOperator(task_id="upstream_level_1") - op4 = EmptyOperator(task_id="upstream_level_2") - - op1 >> op2 >> op3 - op4 >> op3 - return dag - - def test_backfill_max_limit_check_within_limit(self, dag_maker, mock_executor): - dag = self._get_dag_test_max_active_limits( - dag_maker, dag_id="test_backfill_max_limit_check_within_limit", max_active_runs=16 - ) - dag_maker.create_dagrun(state=None) - start_date = DEFAULT_DATE - datetime.timedelta(hours=1) - end_date = DEFAULT_DATE - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=start_date, - end_date=end_date, - donot_pickle=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - - dagruns = DagRun.find(dag_id=dag.dag_id) - assert 2 == len(dagruns) - assert all(run.state == State.SUCCESS for run in dagruns) - - def test_backfill_notifies_dagrun_listener(self, dag_maker, mock_executor): - dag = self._get_dummy_dag(dag_maker) - dag_run = dag_maker.create_dagrun(state=None) - dag_listener.clear() - get_listener_manager().add_listener(dag_listener) - - start_date = DEFAULT_DATE - datetime.timedelta(hours=1) - end_date = DEFAULT_DATE - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=start_date, - end_date=end_date, - donot_pickle=True, - ) - job.notification_threadpool = mock.MagicMock() - run_job(job=job, execute_callable=job_runner._execute) - - assert len(dag_listener.running) == 1 - assert len(dag_listener.success) == 1 - assert dag_listener.running[0].dag.dag_id == dag_run.dag.dag_id - assert dag_listener.running[0].run_id == dag_run.run_id - assert dag_listener.running[0].state == DagRunState.RUNNING - - assert dag_listener.success[0].dag.dag_id == dag_run.dag.dag_id - assert dag_listener.success[0].run_id == dag_run.run_id - assert dag_listener.success[0].state == DagRunState.SUCCESS - - def test_backfill_max_limit_check(self, dag_maker, mock_executor): - dag_id = "test_backfill_max_limit_check" - run_id = "test_dag_run" - start_date = DEFAULT_DATE - datetime.timedelta(hours=1) - end_date = DEFAULT_DATE - - dag_run_created_cond = threading.Condition() - - def run_backfill(cond): - cond.acquire() - # this session object is different than the one in the main thread - with create_session() as thread_session: - try: - dag = self._get_dag_test_max_active_limits( - dag_maker, - dag_id=dag_id, - ) - dag_maker.create_dagrun( - state=State.RUNNING, - # Existing dagrun that is not within the backfill range - run_id=run_id, - execution_date=DEFAULT_DATE + datetime.timedelta(hours=1), - ) - thread_session.commit() - cond.notify() - except Exception: - logger.exception("Exception when creating DagRun") - finally: - cond.release() - thread_session.close() - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=start_date, - end_date=end_date, - donot_pickle=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - - backfill_job_thread = threading.Thread( - target=run_backfill, name="run_backfill", args=(dag_run_created_cond,) - ) - - dag_run_created_cond.acquire() - with create_session() as session: - backfill_job_thread.start() - try: - # at this point backfill can't run since the max_active_runs has been - # reached, so it is waiting - dag_run_created_cond.wait(timeout=1.5) - dagruns = DagRun.find(dag_id=dag_id) - logger.info("The dag runs retrieved: %s", dagruns) - assert 1 == len(dagruns) - dr = dagruns[0] - assert dr.run_id == run_id - - # allow the backfill to execute - # by setting the existing dag run to SUCCESS, - # backfill will execute dag runs 1 by 1 - dr.set_state(State.SUCCESS) - session.merge(dr) - session.commit() - - backfill_job_thread.join() - - dagruns = DagRun.find(dag_id=dag_id) - assert 3 == len(dagruns) # 2 from backfill + 1 existing - assert dagruns[-1].run_id == dr.run_id - finally: - dag_run_created_cond.release() - - def test_backfill_max_limit_check_no_count_existing(self, dag_maker, mock_executor): - start_date = DEFAULT_DATE - end_date = DEFAULT_DATE - # Existing dagrun that is within the backfill range - dag = self._get_dag_test_max_active_limits( - dag_maker, dag_id="test_backfill_max_limit_check_no_count_existing" - ) - dag_maker.create_dagrun(state=None) - - job = Job() - job_runner = BackfillJobRunner( - job=job, dag=dag, start_date=start_date, end_date=end_date, donot_pickle=True - ) - run_job(job=job, execute_callable=job_runner._execute) - - # BackfillJobRunner will run since the existing DagRun does not count for the max - # active limit since it's within the backfill date range. - dagruns = DagRun.find(dag_id=dag.dag_id) - # will only be able to run 1 (the existing one) since there's just - # one dag run slot left given the max_active_runs limit - assert 1 == len(dagruns) - assert State.SUCCESS == dagruns[0].state - - def test_backfill_max_limit_check_complete_loop(self, dag_maker, mock_executor): - dag = self._get_dag_test_max_active_limits( - dag_maker, dag_id="test_backfill_max_limit_check_complete_loop" - ) - dag_maker.create_dagrun(state=None) - start_date = DEFAULT_DATE - datetime.timedelta(hours=1) - end_date = DEFAULT_DATE - - # Given the max limit to be 1 in active dag runs, we need to run the - # backfill job 3 times - success_expected = 2 - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=start_date, - end_date=end_date, - donot_pickle=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - - success_dagruns = len(DagRun.find(dag_id=dag.dag_id, state=State.SUCCESS)) - running_dagruns = len(DagRun.find(dag_id=dag.dag_id, state=State.RUNNING)) - assert success_expected == success_dagruns - assert 0 == running_dagruns # no dag_runs in running state are left - - def test_sub_set_subdag(self, dag_maker, mock_executor): - with dag_maker( - "test_sub_set_subdag", - on_success_callback=lambda _: None, - on_failure_callback=lambda _: None, - ) as dag: - op1 = EmptyOperator(task_id="leave1") - op2 = EmptyOperator(task_id="leave2") - op3 = EmptyOperator(task_id="upstream_level_1") - op4 = EmptyOperator(task_id="upstream_level_2") - op5 = EmptyOperator(task_id="upstream_level_3") - # order randomly - op2.set_downstream(op3) - op1.set_downstream(op3) - op4.set_downstream(op5) - op3.set_downstream(op4) - - dr = dag_maker.create_dagrun(state=None) - - sub_dag = dag.partial_subset( - task_ids_or_regex="leave*", include_downstream=False, include_upstream=False - ) - job = Job() - job_runner = BackfillJobRunner(job=job, dag=sub_dag, start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - run_job(job=job, execute_callable=job_runner._execute) - - for ti in dr.get_task_instances(): - if ti.task_id == "leave1" or ti.task_id == "leave2": - assert State.SUCCESS == ti.state - else: - assert State.NONE == ti.state - - def test_backfill_fill_blanks(self, dag_maker, mock_executor): - with dag_maker( - "test_backfill_fill_blanks", - ) as dag: - op1 = EmptyOperator(task_id="op1") - op2 = EmptyOperator(task_id="op2") - op3 = EmptyOperator(task_id="op3") - op4 = EmptyOperator(task_id="op4") - op5 = EmptyOperator(task_id="op5") - op6 = EmptyOperator(task_id="op6") - - dr = dag_maker.create_dagrun(state=None) - - session = settings.Session() - - tis = dr.get_task_instances() - for ti in tis: - if ti.task_id == op1.task_id: - ti.state = State.UP_FOR_RETRY - ti.end_date = DEFAULT_DATE - elif ti.task_id == op2.task_id: - ti.state = State.FAILED - elif ti.task_id == op3.task_id: - ti.state = State.SKIPPED - elif ti.task_id == op4.task_id: - ti.state = State.SCHEDULED - elif ti.task_id == op5.task_id: - ti.state = State.UPSTREAM_FAILED - # op6 = None - session.merge(ti) - session.commit() - session.close() - - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag, start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - with pytest.raises(AirflowException, match="Some task instances failed"): - run_job(job=job, execute_callable=job_runner._execute) - - dr.refresh_from_db() - - assert dr.state == State.FAILED - - tis = dr.get_task_instances() - for ti in tis: - if ti.task_id in (op1.task_id, op4.task_id, op6.task_id): - assert ti.state == State.SUCCESS - elif ti.task_id == op2.task_id: - assert ti.state == State.FAILED - elif ti.task_id == op3.task_id: - assert ti.state == State.SKIPPED - elif ti.task_id == op5.task_id: - assert ti.state == State.UPSTREAM_FAILED - - def test_update_counters(self, dag_maker, session): - with dag_maker(dag_id="test_manage_executor_state", start_date=DEFAULT_DATE, session=session) as dag: - task1 = EmptyOperator(task_id="dummy", owner="airflow") - dr = dag_maker.create_dagrun(state=None) - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag) - ti = TI(task1, run_id=dr.run_id) - ti.refresh_from_db() - - ti_status = BackfillJobRunner._DagRunTaskStatus() - - # Test for success - # The in-memory task key in ti_status.running contains a try_number - # that is not in sync with the DB. To test that _update_counters method - # handles this, we mark the task as running in-memory and then increase - # the try number as it would be before the raw task is executed. - # When updating the counters the in-memory key will be used which will - # match what's in the in-memory ti_status.running map. This is the same - # for skipped, failed and retry states. - ti_status.running[ti.key] = ti # Task is queued and marked as running - ti.try_number += 1 - ti.set_state(State.SUCCESS, session) # Task finishes with success state - job_runner._update_counters(ti_status=ti_status, session=session) # Update counters - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 1 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 0 - - ti_status.succeeded.clear() - - # Test for success when DB try_number is off from in-memory expectations - ti_status.running[ti.key] = ti - ti.try_number += 2 - ti.set_state(State.SUCCESS, session) - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 1 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 0 - - ti_status.succeeded.clear() - - # Test for skipped - ti_status.running[ti.key] = ti - ti.try_number += 1 - ti.set_state(State.SKIPPED, session) - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 0 - assert len(ti_status.skipped) == 1 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 0 - - ti_status.skipped.clear() - - # Test for failed - ti_status.running[ti.key] = ti - ti.try_number += 1 - ti.set_state(State.FAILED, session) - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 0 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 1 - assert len(ti_status.to_run) == 0 - - ti_status.failed.clear() - - # Test for retry - ti_status.running[ti.key] = ti - ti.try_number += 1 - ti.set_state(State.UP_FOR_RETRY, session) - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 0 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 1 - - ti_status.to_run.clear() - - # Test for reschedule - # Logic in taskinstance reduces the try number for a task that's been - # rescheduled (which makes sense because it's the _same_ try, but it's - # just being rescheduled to a later time). This now makes the in-memory - # and DB representation of the task try_number the _same_, which is unlike - # the above cases. But this is okay because the in-memory key is used. - ti_status.running[ti.key] = ti # Task queued and marked as running - ti.set_state(State.UP_FOR_RESCHEDULE, session) # Task finishes with reschedule state - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 0 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 1 - - ti_status.to_run.clear() - - # test for none - ti.set_state(State.NONE, session) - session.merge(ti) - session.commit() - ti_status.running[ti.key] = ti - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 0 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 1 - - ti_status.to_run.clear() - - # test for scheduled - ti.set_state(State.SCHEDULED) - # Deferred tasks are put into scheduled by the triggerer - # Check that they are put into to_run - ti_status.running[ti.key] = ti - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 0 - assert len(ti_status.succeeded) == 0 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 1 - - ti_status.to_run.clear() - # test for deferred - # if a task is deferred and it's not yet time for the triggerer - # to reschedule it, we should leave it in ti_status.running - ti.set_state(State.DEFERRED) - ti_status.running[ti.key] = ti - job_runner._update_counters(ti_status=ti_status, session=session) - assert len(ti_status.running) == 1 - assert len(ti_status.succeeded) == 0 - assert len(ti_status.skipped) == 0 - assert len(ti_status.failed) == 0 - assert len(ti_status.to_run) == 0 - session.close() - - def test_dag_dagrun_infos_between(self, dag_maker): - with dag_maker( - dag_id="dagrun_infos_between", start_date=DEFAULT_DATE, schedule="@hourly" - ) as test_dag: - EmptyOperator( - task_id="dummy", - owner="airflow", - ) - - assert [DEFAULT_DATE] == [ - info.logical_date - for info in test_dag.iter_dagrun_infos_between( - earliest=DEFAULT_DATE, - latest=DEFAULT_DATE, - ) - ] - assert [ - DEFAULT_DATE - datetime.timedelta(hours=3), - DEFAULT_DATE - datetime.timedelta(hours=2), - DEFAULT_DATE - datetime.timedelta(hours=1), - DEFAULT_DATE, - ] == [ - info.logical_date - for info in test_dag.iter_dagrun_infos_between( - earliest=DEFAULT_DATE - datetime.timedelta(hours=3), - latest=DEFAULT_DATE, - ) - ] - - def test_backfill_run_backwards(self, mock_executor): - dag = self.dagbag.get_dag("test_start_date_scheduling") - dag.clear() - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=1), - run_backwards=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - - session = settings.Session() - tis = ( - session.query(TI) - .join(TI.dag_run) - .filter(TI.dag_id == "test_start_date_scheduling" and TI.task_id == "dummy") - .order_by(DagRun.execution_date) - .all() - ) - - queued_times = [ti.queued_dttm for ti in tis] - assert queued_times == sorted(queued_times, reverse=True) - assert all(ti.state == State.SUCCESS for ti in tis) - - dag.clear() - session.close() - - def test_reset_orphaned_tasks_with_orphans(self, dag_maker): - """Create dagruns and ensure only ones with correct states are reset.""" - prefix = "backfill_job_test_test_reset_orphaned_tasks" - states = [State.QUEUED, State.SCHEDULED, State.NONE, State.RUNNING, State.SUCCESS] - states_to_reset = [State.QUEUED, State.SCHEDULED, State.NONE] - triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - - tasks = [] - with dag_maker(dag_id=prefix) as dag: - for i in range(len(states)): - task_id = f"{prefix}_task_{i}" - task = EmptyOperator(task_id=task_id) - tasks.append(task) - - session = settings.Session() - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag) - # create dagruns - dr1 = dag_maker.create_dagrun(run_id=DEFAULT_DAG_RUN_ID, state=State.RUNNING) - dr2 = dag.create_dagrun(run_id="test2", state=State.SUCCESS, **triggered_by_kwargs) - - # create taskinstances and set states - dr1_tis = [] - dr2_tis = [] - for task, state in zip(tasks, states): - ti1 = TI(task, run_id=dr1.run_id) - ti2 = TI(task, run_id=dr2.run_id) - ti1.refresh_from_db() - ti2.refresh_from_db() - ti1.state = state - ti2.state = state - dr1_tis.append(ti1) - dr2_tis.append(ti2) - session.merge(ti1) - session.merge(ti2) - session.commit() - - assert 2 == job_runner.reset_state_for_orphaned_tasks() - - for ti in dr1_tis + dr2_tis: - ti.refresh_from_db() - - # running dagrun should be reset - for state, ti in zip(states, dr1_tis): - if state in states_to_reset: - assert ti.state is None - else: - assert state == ti.state - - # otherwise not - for state, ti in zip(states, dr2_tis): - assert state == ti.state - - for state, ti in zip(states, dr1_tis): - ti.state = state - session.commit() - - job_runner.reset_state_for_orphaned_tasks(filter_by_dag_run=dr1, session=session) - - # check same for dag_run version - for state, ti in zip(states, dr2_tis): - assert state == ti.state - - def test_reset_orphaned_tasks_specified_dagrun(self, session, dag_maker): - """Try to reset when we specify a dagrun and ensure nothing else is.""" - dag_id = "test_reset_orphaned_tasks_specified_dagrun" - task_id = dag_id + "_task" - with dag_maker( - dag_id=dag_id, - start_date=DEFAULT_DATE, - schedule="@daily", - session=session, - ) as dag: - EmptyOperator(task_id=task_id, dag=dag) - - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag) - # make two dagruns, only reset for one - triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr1 = dag_maker.create_dagrun(state=State.SUCCESS, **triggered_by_kwargs) - dr2 = dag.create_dagrun( - run_id="test2", - state=State.RUNNING, - session=session, - **triggered_by_kwargs, - ) - ti1 = dr1.get_task_instances(session=session)[0] - ti2 = dr2.get_task_instances(session=session)[0] - ti1.state = State.SCHEDULED - ti2.state = State.SCHEDULED - - session.merge(ti1) - session.merge(ti2) - session.merge(dr1) - session.merge(dr2) - session.flush() - - num_reset_tis = job_runner.reset_state_for_orphaned_tasks(filter_by_dag_run=dr2, session=session) - assert 1 == num_reset_tis - ti1.refresh_from_db(session=session) - ti2.refresh_from_db(session=session) - assert State.SCHEDULED == ti1.state - assert State.NONE == ti2.state - - def test_job_id_is_assigned_to_dag_run(self, dag_maker, mock_executor): - dag_id = "test_job_id_is_assigned_to_dag_run" - with dag_maker(dag_id=dag_id, start_date=DEFAULT_DATE, schedule="@daily") as dag: - EmptyOperator(task_id="dummy_task", dag=dag) - - job = Job() - job_runner = BackfillJobRunner( - job=job, dag=dag, start_date=timezone.utcnow() - datetime.timedelta(days=1) - ) - run_job(job=job, execute_callable=job_runner._execute) - dr: DagRun = dag.get_last_dagrun() - assert dr.creating_job_id == job.id - - def test_executor_lifecycle(self, dag_maker, mock_executors): - """Ensure that all executors go through the full lifecycle of start, heartbeat, end, etc""" - dag_id = "test_executor_lifecycle" - with dag_maker(dag_id=dag_id, start_date=DEFAULT_DATE, schedule="@daily") as dag: - EmptyOperator(task_id="dummy_task", dag=dag) - - job = Job() - job_runner = BackfillJobRunner( - job=job, dag=dag, start_date=timezone.utcnow() - datetime.timedelta(days=1) - ) - run_job(job=job, execute_callable=job_runner._execute) - - for executor_mock in mock_executors: - assert executor_mock.job_id == job.id - executor_mock.start.assert_called_once() - executor_mock.heartbeat.assert_called_once() - executor_mock.end.assert_called_once() - - def test_non_existing_executor(self, dag_maker, mock_executors): - dag_id = "test_non_existing_executor" - with dag_maker(dag_id=dag_id, start_date=DEFAULT_DATE, schedule="@daily") as dag: - EmptyOperator(task_id="dummy_task", dag=dag, executor="foobar") - - job = Job() - job_runner = BackfillJobRunner( - job=job, dag=dag, start_date=timezone.utcnow() - datetime.timedelta(days=1) - ) - # Executor "foobar" does not exist, so the Backfill job should fail to run those tasks and - # throw an UnknownExecutorException - with pytest.raises(UnknownExecutorException): - run_job(job=job, execute_callable=job_runner._execute) - - def test_hybrid_executors(self, dag_maker, mock_executors, session): - dag_id = "test_hybrid_executors" - with dag_maker(dag_id=dag_id, start_date=DEFAULT_DATE, schedule="@daily") as dag: - EmptyOperator(task_id="default_exec", dag=dag) - EmptyOperator(task_id="default_exec_explicit", dag=dag, executor=mock_executors[0].name.alias) - EmptyOperator(task_id="secondary_exec", dag=dag, executor=mock_executors[1].name.alias) - - job = Job() - job_runner = BackfillJobRunner( - job=job, dag=dag, start_date=timezone.utcnow() - datetime.timedelta(days=1) - ) - - with mock.patch("airflow.executors.executor_loader.ExecutorLoader.lookup_executor_name_by_str"): - run_job(job=job, execute_callable=job_runner._execute) - - dr = DagRun.find(dag_id=dag.dag_id, session=session)[0] - - call_list = mock_executors[0].queue_task_instance.call_args_list - assert len(call_list) == 2 - assert call_list[0].args[0].task_id == "default_exec" - assert call_list[1].args[0].task_id == "default_exec_explicit" - - call_list = mock_executors[1].queue_task_instance.call_args_list - assert len(call_list) == 1 - assert call_list[0].args[0].task_id == "secondary_exec" - - assert dr - assert dr.state == DagRunState.SUCCESS - - # Check that every task has a start and end date - for ti in dr.task_instances: - assert ti.state == TaskInstanceState.SUCCESS - assert ti.start_date is not None - assert ti.end_date is not None - - def test_backfill_has_job_id_int(self, mock_executor): - """Make sure that backfill jobs are assigned job_ids and that the job_id is an int.""" - dag = self.dagbag.get_dag("test_start_date_scheduling") - dag.clear() - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE + datetime.timedelta(days=1), - run_backwards=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - assert isinstance(job.executor.job_id, int) - - @pytest.mark.long_running - @pytest.mark.parametrize("executor", [SequentialExecutor, DebugExecutor]) - @pytest.mark.parametrize("dag_id", ["test_mapped_classic", "test_mapped_taskflow", "test_sensor"]) - def test_backfilling_dags(self, dag_id, executor, session): - """ - End-to-end test for backfilling dags with various executors. - - We test with multiple executors as they have different "execution environments" -- for instance - DebugExecutor runs a lot more in the same process than other Executors. - - """ - # This test needs a real executor to run, so that the `make_list` task can write out the TaskMap - for _ in _mock_executor(executor): - self.dagbag.process_file(str(TEST_DAGS_FOLDER / f"{dag_id}.py")) - dag = self.dagbag.get_dag(dag_id) - - when = timezone.datetime(2022, 1, 1) - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=when, - end_date=when, - donot_pickle=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - - dr = DagRun.find(dag_id=dag.dag_id, execution_date=when, session=session)[0] - assert dr - assert dr.state == DagRunState.SUCCESS - - # Check that every task has a start and end date - for ti in dr.task_instances: - assert ti.state == TaskInstanceState.SUCCESS - assert ti.start_date is not None - assert ti.end_date is not None - - def test_mapped_dag_pre_existing_tis(self, dag_maker, session, mock_executor): - """If the DagRun already has some mapped TIs, ensure that we re-run them successfully""" - from airflow.decorators import task - from airflow.operators.python import PythonOperator - - list_result = [[1], [2], [{"a": "b"}]] - - @task - def make_arg_lists(): - return list_result - - def consumer(value): - print(repr(value)) - - with dag_maker(session=session) as dag: - consumer_op = PythonOperator.partial(task_id="consumer", python_callable=consumer).expand( - op_args=make_arg_lists() - ) - PythonOperator.partial(task_id="consumer_literal", python_callable=consumer).expand( - op_args=[[1], [2], [3]], - ) - - dr = dag_maker.create_dagrun() - - # Create the existing mapped TIs -- this the crucial part of this test - ti = dr.get_task_instance("consumer", session=session) - ti.map_index = 0 - for map_index in range(1, 3): - ti = TI(consumer_op, run_id=dr.run_id, map_index=map_index) - session.add(ti) - ti.dag_run = dr - session.flush() - - executor = mock_executor - - ti_status = BackfillJobRunner._DagRunTaskStatus() - ti_status.active_runs.add(dr) - ti_status.to_run = {ti.key: ti for ti in dr.task_instances} - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=dr.execution_date, - end_date=dr.execution_date, - donot_pickle=True, - ) - - executor_change_state = executor.change_state - - def on_change_state(key, state, info=None): - if key.task_id == "make_arg_lists": - session.add( - TaskMap( - length=len(list_result), - keys=None, - dag_id=key.dag_id, - run_id=key.run_id, - task_id=key.task_id, - map_index=key.map_index, - ) - ) - session.flush() - executor_change_state(key, state, info) - - with patch.object(executor, "change_state", side_effect=on_change_state): - job_runner._process_backfill_task_instances( - ti_status=ti_status, - start_date=dr.execution_date, - pickle_id=None, - session=session, - ) - assert ti_status.failed == set() - assert ti_status.succeeded == { - TaskInstanceKey(dag_id=dr.dag_id, task_id="consumer", run_id="test", try_number=0, map_index=0), - TaskInstanceKey(dag_id=dr.dag_id, task_id="consumer", run_id="test", try_number=0, map_index=1), - TaskInstanceKey(dag_id=dr.dag_id, task_id="consumer", run_id="test", try_number=0, map_index=2), - TaskInstanceKey( - dag_id=dr.dag_id, task_id="consumer_literal", run_id="test", try_number=0, map_index=0 - ), - TaskInstanceKey( - dag_id=dr.dag_id, task_id="consumer_literal", run_id="test", try_number=0, map_index=1 - ), - TaskInstanceKey( - dag_id=dr.dag_id, task_id="consumer_literal", run_id="test", try_number=0, map_index=2 - ), - TaskInstanceKey( - dag_id=dr.dag_id, task_id="make_arg_lists", run_id="test", try_number=0, map_index=-1 - ), - } - - def test_mapped_dag_unexpandable(self, dag_maker, session, mock_executor): - with dag_maker(session=session) as dag: - - @dag.task - def get_things(): - return [1, 2] - - @dag.task - def this_fails() -> None: - raise RuntimeError("sorry!") - - @dag.task(trigger_rule=TriggerRule.ALL_DONE) - def consumer(a, b): - print(a, b) - - consumer.expand(a=get_things(), b=this_fails()) - - when = timezone.datetime(2022, 1, 1) - job = Job() - job_runner = BackfillJobRunner(job=job, dag=dag, start_date=when, end_date=when, donot_pickle=True) - run_job(job=job, execute_callable=job_runner._execute) - (dr,) = DagRun.find(dag_id=dag.dag_id, execution_date=when, session=session) - assert dr.state == DagRunState.FAILED - - # Check that every task has a start and end date - tis = {(ti.task_id, ti.map_index): ti for ti in dr.task_instances} - assert len(tis) == 3 - tis[("get_things", -1)].state == TaskInstanceState.SUCCESS - tis[("this_fails", -1)].state == TaskInstanceState.FAILED - tis[("consumer", -1)].state == TaskInstanceState.UPSTREAM_FAILED - - def test_start_date_set_for_resetted_dagruns(self, dag_maker, session, caplog, mock_executor): - with dag_maker() as dag: - EmptyOperator(task_id="task1") - - dr = dag_maker.create_dagrun() - dr.state = State.SUCCESS - session.merge(dr) - session.flush() - dag.clear() - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - donot_pickle=True, - ) - run_job(job=job, execute_callable=job_runner._execute) - - (dr,) = DagRun.find(dag_id=dag.dag_id, execution_date=DEFAULT_DATE, session=session) - assert dr.start_date - assert f"Failed to record duration of {dr}" not in caplog.text - - def test_task_instances_are_not_set_to_scheduled_when_dagrun_reset( - self, dag_maker, session, mock_executor - ): - """Test that when dagrun is reset, task instances are not set to scheduled""" - - with dag_maker() as dag: - task1 = EmptyOperator(task_id="task1") - task2 = EmptyOperator(task_id="task2") - task3 = EmptyOperator(task_id="task3") - task1 >> task2 >> task3 - - for i in range(1, 4): - dag_maker.create_dagrun( - run_id=f"test_dagrun_{i}", execution_date=DEFAULT_DATE + datetime.timedelta(days=i) - ) - - dag.clear() - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE + datetime.timedelta(days=1), - end_date=DEFAULT_DATE + datetime.timedelta(days=4), - donot_pickle=True, - ) - for dr in DagRun.find(dag_id=dag.dag_id, session=session): - tasks_to_run = job_runner._task_instances_for_dag_run(dag, dr, session=session) - states = [ti.state for _, ti in tasks_to_run.items()] - assert TaskInstanceState.SCHEDULED in states - assert State.NONE in states - - @pytest.mark.parametrize( - ["disable_retry", "try_number", "exception"], - ( - (True, 1, BackfillUnfinished), - (False, 2, AirflowException), - ), - ) - def test_backfill_disable_retry(self, dag_maker, disable_retry, try_number, exception, mock_executor): - with dag_maker( - dag_id="test_disable_retry", - schedule="@daily", - default_args={ - "retries": 2, - "retry_delay": datetime.timedelta(seconds=3), - }, - ) as dag: - task1 = EmptyOperator(task_id="task1") - dag_run = dag_maker.create_dagrun(state=None) - - executor = mock_executor - executor.parallelism = 16 - executor.mock_task_results[ - TaskInstanceKey(dag.dag_id, task1.task_id, dag_run.run_id, try_number=1) - ] = TaskInstanceState.UP_FOR_RETRY - executor.mock_task_results[ - TaskInstanceKey(dag.dag_id, task1.task_id, dag_run.run_id, try_number=2) - ] = TaskInstanceState.FAILED - - job = Job() - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - disable_retry=disable_retry, - ) - with pytest.raises(exception): - run_job(job=job, execute_callable=job_runner._execute) - ti = dag_run.get_task_instance(task_id=task1.task_id) - - assert ti.try_number == try_number - - dag_run.refresh_from_db() - - assert dag_run.state == DagRunState.FAILED - - dag.clear() - - # Qyarantined issue tracked in https://github.com/apache/airflow/issues/39858 - @pytest.mark.quarantined - def test_backfill_failed_dag_with_upstream_failed_task(self, dag_maker): - self.dagbag.process_file(str(TEST_DAGS_FOLDER / "test_backfill_with_upstream_failed_task.py")) - dag = self.dagbag.get_dag("test_backfill_with_upstream_failed_task") - - # We have to use the "fake" version of perform_heartbeat due to the 'is_unit_test' check in - # the original one. However, instead of using the original version of perform_heartbeat, - # we can simply wait for a LocalExecutor's worker cycle. The approach with sleep works well now, - # but it can be replaced with checking the state of the LocalTaskJob. - def fake_perform_heartbeat(*args, **kwargs): - import time - - time.sleep(1) - - with mock.patch("airflow.jobs.backfill_job_runner.perform_heartbeat", fake_perform_heartbeat): - job = Job(executor=ExecutorLoader.load_executor("LocalExecutor")) - job_runner = BackfillJobRunner( - job=job, - dag=dag, - start_date=DEFAULT_DATE, - end_date=DEFAULT_DATE, - rerun_failed_tasks=True, - ) - with pytest.raises(BackfillUnfinished): - run_job(job=job, execute_callable=job_runner._execute) - - dr: DagRun = dag.get_last_dagrun() - assert dr.state == State.FAILED diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 78c3acdce0de6..18ca7c63320f0 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -48,7 +48,6 @@ from airflow.executors.base_executor import BaseExecutor from airflow.executors.executor_constants import MOCK_EXECUTOR from airflow.executors.executor_loader import ExecutorLoader -from airflow.jobs.backfill_job_runner import BackfillJobRunner from airflow.jobs.job import Job, run_job from airflow.jobs.local_task_job_runner import LocalTaskJobRunner from airflow.jobs.scheduler_job_runner import SchedulerJobRunner @@ -60,16 +59,16 @@ from airflow.models.db_callback_request import DbCallbackRequest from airflow.models.pool import Pool from airflow.models.serialized_dag import SerializedDagModel -from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance, TaskInstanceKey +from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance from airflow.operators.empty import EmptyOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG +from airflow.timetables.base import DataInterval from airflow.utils import timezone from airflow.utils.file import list_py_file_paths from airflow.utils.session import create_session, provide_session from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState from airflow.utils.types import DagRunType -from tests.jobs.test_backfill_job import _mock_executor from tests.listeners import dag_listener from tests.listeners.test_listeners import get_listener_manager from tests.models import TEST_DAGS_FOLDER @@ -2916,22 +2915,16 @@ def test_scheduler_start_date(self, configs): # because it would take the most recent run and start from there # That behavior still exists, but now it will only do so if after the # start date - bf_exec = MockExecutor() - for _ in _mock_executor(bf_exec): - backfill_job = Job() - job_runner = BackfillJobRunner( - job=backfill_job, dag=dag, start_date=DEFAULT_DATE, end_date=DEFAULT_DATE - ) - run_job(job=backfill_job, execute_callable=job_runner._execute) - - # one task ran + dag.create_dagrun( + state="success", + triggered_by=DagRunTriggeredByType.TIMETABLE, + run_id="abc123", + execution_date=DEFAULT_DATE, + run_type=DagRunType.BACKFILL_JOB, + data_interval=DataInterval(DEFAULT_DATE, DEFAULT_DATE + timedelta(days=1)), + ) + # one task "ran" assert len(session.query(TaskInstance).filter(TaskInstance.dag_id == dag_id).all()) == 1 - assert [ - ( - TaskInstanceKey(dag.dag_id, "dummy", f"backfill__{DEFAULT_DATE.isoformat()}", 1), - (State.SUCCESS, None), - ), - ] == bf_exec.sorted_tasks session.commit() scheduler_job = Job( From 147e59b9f4cc4124464f14db8209cc48c1c82969 Mon Sep 17 00:00:00 2001 From: olegkachur-e Date: Mon, 14 Oct 2024 03:49:35 +0200 Subject: [PATCH 097/199] Add early job_id xcom_push for google provider Beam Pipeline operators (#42982) - To let GCP Beam Sensor operators 'sense' the pipeline changes, by having dataflow job_id been xcom_push as soon as it available. Related issue: https://github.com/apache/airflow/issues/30007. Co-authored-by: Oleg Kachur --- .../providers/apache/beam/operators/beam.py | 19 +++++++++++++++-- .../tests/apache/beam/operators/test_beam.py | 21 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/providers/src/airflow/providers/apache/beam/operators/beam.py b/providers/src/airflow/providers/apache/beam/operators/beam.py index 41c55ede2a5bc..65f23336589d2 100644 --- a/providers/src/airflow/providers/apache/beam/operators/beam.py +++ b/providers/src/airflow/providers/apache/beam/operators/beam.py @@ -187,7 +187,20 @@ def __init__( self.gcp_conn_id = gcp_conn_id self.beam_hook: BeamHook self.dataflow_hook: DataflowHook | None = None - self.dataflow_job_id: str | None = None + self._dataflow_job_id: str | None = None + self._execute_context: Context | None = None + + @property + def dataflow_job_id(self): + return self._dataflow_job_id + + @dataflow_job_id.setter + def dataflow_job_id(self, new_value): + if all([new_value, not self._dataflow_job_id, self._execute_context]): + # push job_id as soon as it's ready, to let Sensors work before the job finished + # and job_id pushed as returned value item. + self.xcom_push(context=self._execute_context, key="dataflow_job_id", value=new_value) + self._dataflow_job_id = new_value def _cast_dataflow_config(self): if isinstance(self.dataflow_config, dict): @@ -346,6 +359,7 @@ def __init__( def execute(self, context: Context): """Execute the Apache Beam Python Pipeline.""" + self._execute_context = context self._cast_dataflow_config() self.pipeline_options.setdefault("labels", {}).update( {"airflow-version": "v" + version.replace(".", "-").replace("+", "-")} @@ -540,6 +554,7 @@ def __init__( def execute(self, context: Context): """Execute the Apache Beam Python Pipeline.""" + self._execute_context = context self._cast_dataflow_config() ( self.is_dataflow, @@ -738,7 +753,7 @@ def execute(self, context: Context): """Execute the Apache Beam Pipeline.""" if not exactly_one(self.go_file, self.launcher_binary): raise ValueError("Exactly one of `go_file` and `launcher_binary` must be set") - + self._execute_context = context self._cast_dataflow_config() if self.dataflow_config.impersonation_chain: self.log.warning( diff --git a/providers/tests/apache/beam/operators/test_beam.py b/providers/tests/apache/beam/operators/test_beam.py index 6d1b4b5d1b958..fd2e706c29414 100644 --- a/providers/tests/apache/beam/operators/test_beam.py +++ b/providers/tests/apache/beam/operators/test_beam.py @@ -110,6 +110,27 @@ def test_async_execute_logging_should_execute_successfully(self, caplog): ) assert f"{TASK_ID} completed with response Pipeline has finished SUCCESSFULLY" in caplog.text + def test_early_dataflow_id_xcom_push(self, default_options, pipeline_options): + with mock.patch.object(BeamBasePipelineOperator, "xcom_push") as mock_xcom_push: + op = BeamBasePipelineOperator( + **self.default_op_kwargs, + default_pipeline_options=copy.deepcopy(default_options), + pipeline_options=copy.deepcopy(pipeline_options), + dataflow_config={}, + ) + sample_df_job_id = "sample_df_job_id_value" + op._execute_context = MagicMock() + + assert op.dataflow_job_id is None + + op.dataflow_job_id = sample_df_job_id + mock_xcom_push.assert_called_once_with( + context=op._execute_context, key="dataflow_job_id", value=sample_df_job_id + ) + mock_xcom_push.reset_mock() + op.dataflow_job_id = "sample_df_job_same_value_id" + mock_xcom_push.assert_not_called() + class TestBeamRunPythonPipelineOperator: @pytest.fixture(autouse=True) From 8e2cbba8e7e46dd6f91f08325d8927559f3af058 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Sun, 13 Oct 2024 19:40:48 -0700 Subject: [PATCH 098/199] Add logic to mark backfills as complete (#42683) Periodically check for backfills that should be marked as complete. --- airflow/jobs/scheduler_job_runner.py | 30 +++++++++++++++++++++++++++- tests/jobs/test_scheduler_job.py | 29 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index 2999ed391bc94..f6821f57aa83c 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Collection, Iterable, Iterator -from sqlalchemy import and_, delete, func, not_, or_, select, text, update +from sqlalchemy import and_, delete, exists, func, not_, or_, select, text, update from sqlalchemy.exc import OperationalError from sqlalchemy.orm import lazyload, load_only, make_transient, selectinload from sqlalchemy.sql import expression @@ -51,6 +51,7 @@ DagScheduleAssetReference, TaskOutletAssetReference, ) +from airflow.models.backfill import Backfill from airflow.models.dag import DAG, DagModel from airflow.models.dagbag import DagBag from airflow.models.dagrun import DagRun @@ -1063,6 +1064,11 @@ def _run_scheduler_loop(self) -> None: self.check_trigger_timeouts, ) + timers.call_regular_interval( + 30, + self._mark_backfills_complete, + ) + timers.call_regular_interval( conf.getfloat("scheduler", "pool_metrics_interval", fallback=5.0), self._emit_pool_metrics, @@ -1288,6 +1294,28 @@ def _create_dagruns_for_dags(self, guard: CommitProhibitorGuard, session: Sessio guard.commit() # END: create dagruns + @provide_session + def _mark_backfills_complete(self, session: Session = NEW_SESSION) -> None: + """Mark completed backfills as completed.""" + self.log.debug("checking for completed backfills.") + unfinished_states = (DagRunState.RUNNING, DagRunState.QUEUED) + now = timezone.utcnow() + # todo: AIP-78 simplify this function to an update statement + query = select(Backfill).where( + Backfill.completed_at.is_(None), + ~exists( + select(DagRun.id).where( + and_(DagRun.backfill_id == Backfill.id, DagRun.state.in_(unfinished_states)) + ) + ), + ) + backfills = session.scalars(query).all() + if not backfills: + return + self.log.info("marking %s backfills as complete", len(backfills)) + for b in backfills: + b.completed_at = now + @add_span def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) -> None: """Create a DAG run and update the dag_model to control if/when the next DAGRun should be created.""" diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 18ca7c63320f0..7eae1639d0e1b 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -6399,3 +6399,32 @@ def test_process_dags_queries_count( prefix = "Collected database query count mismatches:" joined = "\n\n".join(failures) raise AssertionError(f"{prefix}\n\n{joined}") + + +def test_mark_backfills_completed(dag_maker, session): + clear_db_backfills() + with dag_maker(serialized=True, dag_id="test_mark_backfills_completed", schedule="@daily") as dag: + BashOperator(task_id="hi", bash_command="echo hi") + b = _create_backfill( + dag_id=dag.dag_id, + from_date=pendulum.parse("2021-01-01"), + to_date=pendulum.parse("2021-01-03"), + max_active_runs=10, + reverse=False, + dag_run_conf={}, + ) + session.expunge_all() + runner = SchedulerJobRunner( + job=Job(job_type=SchedulerJobRunner.job_type, executor=MockExecutor(do_update=False)) + ) + runner._mark_backfills_complete() + b = session.get(Backfill, b.id) + assert b.completed_at is None + session.expunge_all() + drs = session.scalars(select(DagRun).where(DagRun.dag_id == dag.dag_id)) + for dr in drs: + dr.state = DagRunState.SUCCESS + session.commit() + runner._mark_backfills_complete() + b = session.get(Backfill, b.id) + assert b.completed_at.timestamp() > 0 From 8c13f86733d7f389cced1c8e2cb53a952f72e8f3 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Mon, 14 Oct 2024 14:46:59 +0530 Subject: [PATCH 099/199] fix mypy check failure on main (#42976) * fix mypy error * Apply suggestions from code review * Update conftest.py Co-authored-by: Jarek Potiuk --------- Co-authored-by: Kaxil Naik Co-authored-by: Jarek Potiuk --- task_sdk/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task_sdk/tests/conftest.py b/task_sdk/tests/conftest.py index 209fad46d7d62..7839b299eef2c 100644 --- a/task_sdk/tests/conftest.py +++ b/task_sdk/tests/conftest.py @@ -23,4 +23,4 @@ @pytest.hookimpl(tryfirst=True) def pytest_configure(config: pytest.Config) -> None: - config.inicfg["airflow_deprecations_ignore"] = () + config.inicfg["airflow_deprecations_ignore"] = [] From a4d7125f23d97d4ebcc3c9e80546d90f796bcb9c Mon Sep 17 00:00:00 2001 From: GPK Date: Mon, 14 Oct 2024 10:30:37 +0100 Subject: [PATCH 100/199] mark test_setup_constraint_mapped_task_upstream_removed_and_success as flaky (#42997) --- tests/ti_deps/deps/test_trigger_rule_dep.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ti_deps/deps/test_trigger_rule_dep.py b/tests/ti_deps/deps/test_trigger_rule_dep.py index 80cb400a46d77..9f3699d09c33d 100644 --- a/tests/ti_deps/deps/test_trigger_rule_dep.py +++ b/tests/ti_deps/deps/test_trigger_rule_dep.py @@ -1494,6 +1494,7 @@ def w2(): assert self.get_ti(dr, "w2").state == expected +@pytest.mark.flaky(reruns=5) @pytest.mark.parametrize( "map_index, flag_upstream_failed, expected_ti_state", [(2, True, None), (3, True, REMOVED), (4, True, REMOVED), (3, False, None)], From f4f4a0ea2f53f491c867e4d057dbcbd9b88b38b7 Mon Sep 17 00:00:00 2001 From: Elad Kalif <45845474+eladkal@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:12:31 +0700 Subject: [PATCH 101/199] Update providers metadata 2024-10-14 (#42995) --- generated/provider_metadata.json | 358 ++++++++++++++++++++----------- 1 file changed, 227 insertions(+), 131 deletions(-) diff --git a/generated/provider_metadata.json b/generated/provider_metadata.json index 56199e2f82c3d..a692eb72ae9e5 100644 --- a/generated/provider_metadata.json +++ b/generated/provider_metadata.json @@ -1,11 +1,11 @@ { "airbyte": { "1.0.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -205,15 +205,15 @@ "date_released": "2021-02-27T14:47:19Z" }, "1.3.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-04-06T23:45:20Z" }, "1.4.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -447,23 +447,27 @@ "8.29.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "9.0.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:52Z" } }, "apache.beam": { "1.0.0": { - "associated_airflow_version": "2.0.1", + "associated_airflow_version": "2.1.0", "date_released": "2021-02-04T09:11:00Z" }, "1.0.1": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.0", "date_released": "2021-03-08T19:27:03Z" }, "2.0.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "3.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "3.0.1": { @@ -581,6 +585,10 @@ "5.8.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-08-22T10:37:58Z" + }, + "5.8.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:52Z" } }, "apache.cassandra": { @@ -593,7 +601,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -777,11 +785,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-03-08T19:27:03Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -957,7 +965,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -1071,39 +1079,39 @@ "date_released": "2021-02-27T14:47:19Z" }, "1.0.3": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { - "associated_airflow_version": "2.1.3", + "associated_airflow_version": "2.1.2", "date_released": "2021-07-26T20:06:28Z" }, "2.0.2": { - "associated_airflow_version": "2.1.4", + "associated_airflow_version": "2.1.2", "date_released": "2021-08-30T21:27:32Z" }, "2.0.3": { - "associated_airflow_version": "2.2.2", + "associated_airflow_version": "2.1.2", "date_released": "2021-10-29T22:33:08Z" }, "2.1.0": { - "associated_airflow_version": "2.2.3", + "associated_airflow_version": "2.1.2", "date_released": "2021-11-30T10:33:58Z" }, "2.2.0": { - "associated_airflow_version": "2.2.4", + "associated_airflow_version": "2.1.2", "date_released": "2022-02-08T21:57:49Z" }, "2.3.0": { - "associated_airflow_version": "2.2.4", + "associated_airflow_version": "2.1.2", "date_released": "2022-03-07T20:27:47Z" }, "2.3.1": { - "associated_airflow_version": "2.2.4", + "associated_airflow_version": "2.1.2", "date_released": "2022-03-14T22:17:57Z" }, "2.3.2": { @@ -1347,6 +1355,10 @@ "1.6.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-08-22T10:37:58Z" + }, + "1.6.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:52Z" } }, "apache.kylin": { @@ -1359,7 +1371,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -1433,11 +1445,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -1547,7 +1559,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -1613,11 +1625,11 @@ "date_released": "2020-12-09T21:48:29Z" }, "1.0.1": { - "associated_airflow_version": "2.0.1", + "associated_airflow_version": "2.1.0", "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -1719,15 +1731,15 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-02-27T14:47:19Z" }, "1.0.3": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -1849,6 +1861,10 @@ "4.11.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "4.11.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "apprise": { @@ -1953,7 +1969,7 @@ }, "asana": { "1.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "1.1.0": { @@ -2089,7 +2105,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -2207,6 +2223,10 @@ "3.8.2": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "3.8.3": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "cloudant": { @@ -2219,7 +2239,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -2285,6 +2305,10 @@ "4.0.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "4.0.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "cncf.kubernetes": { @@ -2301,23 +2325,23 @@ "date_released": "2021-02-27T14:47:19Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-04-06T23:45:20Z" }, "1.2.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.3", "date_released": "2021-07-26T20:06:28Z" }, "2.0.2": { - "associated_airflow_version": "2.1.3", + "associated_airflow_version": "2.1.4", "date_released": "2021-08-27T18:18:45Z" }, "2.0.3": { @@ -2543,6 +2567,10 @@ "8.4.2": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "9.0.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "cohere": { @@ -2591,6 +2619,10 @@ "1.2.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-08-22T10:37:57Z" + }, + "1.2.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "common.io": { @@ -2629,6 +2661,10 @@ "1.4.1": { "associated_airflow_version": "2.10.0", "date_released": "2024-09-24T13:49:56Z" + }, + "1.4.2": { + "associated_airflow_version": "2.10.0", + "date_released": "2024-10-14T07:10:53Z" } }, "common.sql": { @@ -2767,6 +2803,10 @@ "1.17.1": { "associated_airflow_version": "2.10.1", "date_released": "2024-10-01T09:05:14Z" + }, + "1.18.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:52Z" } }, "databricks": { @@ -2779,7 +2819,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -2937,6 +2977,10 @@ "6.10.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "6.11.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:52Z" } }, "datadog": { @@ -2949,7 +2993,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -3137,6 +3181,10 @@ "3.10.1": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "3.11.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "dingding": { @@ -3149,11 +3197,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -3219,7 +3267,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -3305,15 +3353,15 @@ "date_released": "2021-02-27T14:47:19Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-04-06T23:45:20Z" }, "1.2.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -3487,11 +3535,11 @@ "date_released": "2021-02-27T14:47:19Z" }, "1.0.3": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-03-16T23:19:47Z" }, "1.0.4": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.1": { @@ -3637,6 +3685,10 @@ "5.5.1": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "5.5.2": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "exasol": { @@ -3649,11 +3701,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.1": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -3821,6 +3873,10 @@ "1.4.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "1.4.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:52Z" } }, "facebook": { @@ -3833,11 +3889,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -3923,11 +3979,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -4111,23 +4167,23 @@ "date_released": "2020-12-09T21:48:29Z" }, "2.0.0": { - "associated_airflow_version": "2.0.0", + "associated_airflow_version": "2.1.0", "date_released": "2021-02-04T09:11:00Z" }, "2.1.0": { - "associated_airflow_version": "2.0.0", + "associated_airflow_version": "2.1.0", "date_released": "2021-02-27T14:47:19Z" }, "2.2.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.0", "date_released": "2021-04-06T23:45:20Z" }, "3.0.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "4.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "5.0.0": { @@ -4349,6 +4405,10 @@ "10.23.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "10.24.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "grpc": { @@ -4361,11 +4421,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -4443,11 +4503,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -4561,11 +4621,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.1": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -4699,7 +4759,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -4871,7 +4931,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -4965,6 +5025,10 @@ "4.5.1": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "4.5.2": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "jenkins": { @@ -4977,11 +5041,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -5087,15 +5151,15 @@ "date_released": "2021-03-08T19:27:03Z" }, "1.3.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "3.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "3.1.0": { @@ -5305,6 +5369,10 @@ "10.5.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "10.5.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:52Z" } }, "microsoft.mssql": { @@ -5317,11 +5385,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -5513,15 +5581,15 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-04-06T23:45:20Z" }, "1.2.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -5599,7 +5667,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -5705,11 +5773,11 @@ "date_released": "2021-02-27T14:47:19Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -5847,19 +5915,23 @@ "5.7.1": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "5.7.2": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "neo4j": { "1.0.0": { - "associated_airflow_version": "2.0.1", + "associated_airflow_version": "2.1.0", "date_released": "2021-02-04T09:11:00Z" }, "1.0.1": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -5949,7 +6021,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -6093,19 +6165,19 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.1": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2022-03-07T20:27:47Z" }, "2.0.2": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2022-03-14T22:17:57Z" }, "2.0.3": { @@ -6233,6 +6305,10 @@ "1.12.1": { "associated_airflow_version": "2.10.1", "date_released": "2024-10-01T09:05:14Z" + }, + "1.12.2": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "opensearch": { @@ -6267,6 +6343,10 @@ "1.4.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-08-22T10:37:58Z" + }, + "1.5.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "opsgenie": { @@ -6279,11 +6359,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -6365,11 +6445,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -6495,7 +6575,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -6589,11 +6669,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -6747,15 +6827,15 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-07-26T20:06:28Z" }, "2.2.0": { @@ -6889,6 +6969,10 @@ "5.13.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "5.13.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "presto": { @@ -6901,11 +6985,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -7061,7 +7145,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -7151,11 +7235,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "3.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "3.1.0": { @@ -7277,7 +7361,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "3.0.0": { @@ -7359,7 +7443,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -7425,11 +7509,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -7499,15 +7583,15 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.1": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-02-27T14:47:19Z" }, "1.2.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -7661,11 +7745,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.1.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -7735,11 +7819,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "3.0.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "4.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "4.0.1": { @@ -7939,15 +8023,15 @@ "date_released": "2021-03-08T19:27:03Z" }, "1.2.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.0.0", "date_released": "2021-04-06T23:45:20Z" }, "1.3.0": { - "associated_airflow_version": "2.1.0", + "associated_airflow_version": "2.1.1", "date_released": "2021-05-01T09:04:42Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -8129,6 +8213,10 @@ "5.7.1": { "associated_airflow_version": "2.10.1", "date_released": "2024-09-24T13:49:56Z" + }, + "5.8.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "sqlite": { @@ -8141,11 +8229,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -8259,11 +8347,11 @@ "date_released": "2021-02-27T14:47:19Z" }, "1.3.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -8393,11 +8481,11 @@ }, "tableau": { "1.0.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -8511,11 +8599,11 @@ "date_released": "2021-02-04T09:11:00Z" }, "1.0.2": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-02-27T14:47:19Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -8627,11 +8715,11 @@ }, "trino": { "1.0.0": { - "associated_airflow_version": "2.0.2", + "associated_airflow_version": "2.1.1", "date_released": "2021-04-06T23:45:20Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -8769,6 +8857,10 @@ "5.8.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-08-22T10:37:58Z" + }, + "5.8.1": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "vertica": { @@ -8781,7 +8873,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { @@ -8937,7 +9029,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.1.0": { @@ -9041,6 +9133,10 @@ "1.3.0": { "associated_airflow_version": "2.10.1", "date_released": "2024-08-22T10:37:59Z" + }, + "1.4.0": { + "associated_airflow_version": "2.10.1", + "date_released": "2024-10-14T07:10:53Z" } }, "zendesk": { @@ -9053,7 +9149,7 @@ "date_released": "2021-02-04T09:11:00Z" }, "2.0.0": { - "associated_airflow_version": "2.1.1", + "associated_airflow_version": "2.1.2", "date_released": "2021-06-23T12:50:28Z" }, "2.0.1": { From eae01b7b0a06af2869c710100fd8c1240fca2dcc Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 14 Oct 2024 15:26:05 +0100 Subject: [PATCH 102/199] Update json schema pre-commit to have draft7 schema in file (#43005) We request this file each time the CI is running pre-commit and sometimes it fails with 403 error. This PR adds the draft7 schema file locally for use by pre-commit to avoid this error --- .pre-commit-config.yaml | 4 +- scripts/ci/pre_commit/draft7_schema.json | 172 +++++++++++++++++++++++ 2 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 scripts/ci/pre_commit/draft7_schema.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d492747856a92..a2f42e65192bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -966,8 +966,8 @@ repos: name: Lint JSON Schema files entry: ./scripts/ci/pre_commit/json_schema.py args: - - --spec-url - - https://json-schema.org/draft-07/schema + - --spec-file + - scripts/ci/pre_commit/draft7_schema.json language: python pass_filenames: true files: .*\.schema\.json$ diff --git a/scripts/ci/pre_commit/draft7_schema.json b/scripts/ci/pre_commit/draft7_schema.json new file mode 100644 index 0000000000000..fb92c7f756b55 --- /dev/null +++ b/scripts/ci/pre_commit/draft7_schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": { "$ref": "#" }, + "then": { "$ref": "#" }, + "else": { "$ref": "#" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "default": true +} From 141e562512fd28a448a55862d15de4a02c28005e Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Mon, 14 Oct 2024 23:08:15 +0800 Subject: [PATCH 103/199] AIP-84 Patch Variable (#42929) --- .../endpoints/variable_endpoint.py | 1 + airflow/api_fastapi/openapi/v1-generated.yaml | 94 ++++++++++++++++++- airflow/api_fastapi/serializers/dags.py | 2 +- airflow/api_fastapi/serializers/variables.py | 17 +++- airflow/api_fastapi/views/public/variables.py | 28 +++++- airflow/ui/openapi-gen/queries/common.ts | 3 + airflow/ui/openapi-gen/queries/queries.ts | 49 +++++++++- .../ui/openapi-gen/requests/schemas.gen.ts | 41 +++++++- .../ui/openapi-gen/requests/services.gen.ts | 36 +++++++ airflow/ui/openapi-gen/requests/types.gen.ts | 50 +++++++++- .../views/public/test_variables.py | 88 +++++++++++++++++ 11 files changed, 394 insertions(+), 15 deletions(-) diff --git a/airflow/api_connexion/endpoints/variable_endpoint.py b/airflow/api_connexion/endpoints/variable_endpoint.py index 8efddb58419c4..b1d9e2f5c8b6f 100644 --- a/airflow/api_connexion/endpoints/variable_endpoint.py +++ b/airflow/api_connexion/endpoints/variable_endpoint.py @@ -95,6 +95,7 @@ def get_variables( ) +@mark_fastapi_migration_done @security.requires_access_variable("PUT") @provide_session @action_logging( diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index 6c35ca212a2a6..759ab7fdd8e9b 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -629,6 +629,72 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + patch: + tags: + - Variable + summary: Patch Variable + description: Update a variable by key. + operationId: patch_variable + parameters: + - name: variable_key + in: path + required: true + schema: + type: string + title: Variable Key + - name: update_mask + in: query + required: false + schema: + anyOf: + - type: array + items: + type: string + - type: 'null' + title: Update Mask + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/VariableBody' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/VariableResponse' + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Bad Request + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /public/dags/{dag_id}/dagRuns/{dag_run_id}: get: tags: @@ -1045,7 +1111,7 @@ components: required: - is_paused title: DAGPatchBody - description: Dag Serializer for updatable body. + description: Dag Serializer for updatable bodies. DAGResponse: properties: dag_id: @@ -1492,25 +1558,47 @@ components: - msg - type title: ValidationError - VariableResponse: + VariableBody: properties: key: type: string title: Key + description: + anyOf: + - type: string + - type: 'null' + title: Description value: anyOf: - type: string - type: 'null' title: Value + type: object + required: + - key + - description + - value + title: VariableBody + description: Variable serializer for bodies. + VariableResponse: + properties: + key: + type: string + title: Key description: anyOf: - type: string - type: 'null' title: Description + value: + anyOf: + - type: string + - type: 'null' + title: Value type: object required: - key - - value - description + - value title: VariableResponse description: Variable serializer for responses. diff --git a/airflow/api_fastapi/serializers/dags.py b/airflow/api_fastapi/serializers/dags.py index 9879badf25048..c9d48aac222eb 100644 --- a/airflow/api_fastapi/serializers/dags.py +++ b/airflow/api_fastapi/serializers/dags.py @@ -95,7 +95,7 @@ def file_token(self) -> str: class DAGPatchBody(BaseModel): - """Dag Serializer for updatable body.""" + """Dag Serializer for updatable bodies.""" is_paused: bool diff --git a/airflow/api_fastapi/serializers/variables.py b/airflow/api_fastapi/serializers/variables.py index ded268432b89d..1ecc87425a24f 100644 --- a/airflow/api_fastapi/serializers/variables.py +++ b/airflow/api_fastapi/serializers/variables.py @@ -25,15 +25,20 @@ from airflow.utils.log.secrets_masker import redact -class VariableResponse(BaseModel): - """Variable serializer for responses.""" +class VariableBase(BaseModel): + """Base Variable serializer.""" model_config = ConfigDict(populate_by_name=True) key: str - val: str | None = Field(alias="value") description: str | None + +class VariableResponse(VariableBase): + """Variable serializer for responses.""" + + val: str | None = Field(alias="value") + @model_validator(mode="after") def redact_val(self) -> Self: if self.val is None: @@ -47,3 +52,9 @@ def redact_val(self) -> Self: # value is not a serialized string representation of a dict. self.val = redact(self.val, self.key) return self + + +class VariableBody(VariableBase): + """Variable serializer for bodies.""" + + value: str | None diff --git a/airflow/api_fastapi/views/public/variables.py b/airflow/api_fastapi/views/public/variables.py index e6cbb136f1cac..b4c07e23de293 100644 --- a/airflow/api_fastapi/views/public/variables.py +++ b/airflow/api_fastapi/views/public/variables.py @@ -16,14 +16,14 @@ # under the License. from __future__ import annotations -from fastapi import Depends, HTTPException +from fastapi import Depends, HTTPException, Query from sqlalchemy import select from sqlalchemy.orm import Session from typing_extensions import Annotated from airflow.api_fastapi.db.common import get_session from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.serializers.variables import VariableResponse +from airflow.api_fastapi.serializers.variables import VariableBody, VariableResponse from airflow.api_fastapi.views.router import AirflowRouter from airflow.models.variable import Variable @@ -56,3 +56,27 @@ async def get_variable( raise HTTPException(404, f"The Variable with key: `{variable_key}` was not found") return VariableResponse.model_validate(variable, from_attributes=True) + + +@variables_router.patch("/{variable_key}", responses=create_openapi_http_exception_doc([400, 401, 403, 404])) +async def patch_variable( + variable_key: str, + patch_body: VariableBody, + session: Annotated[Session, Depends(get_session)], + update_mask: list[str] | None = Query(None), +) -> VariableResponse: + """Update a variable by key.""" + if patch_body.key != variable_key: + raise HTTPException(400, "Invalid body, key from request body doesn't match uri parameter") + non_update_fields = {"key"} + variable = session.scalar(select(Variable).filter_by(key=variable_key).limit(1)) + if not variable: + raise HTTPException(404, f"The Variable with key: `{variable_key}` was not found") + if update_mask: + data = patch_body.dict(include=set(update_mask) - non_update_fields) + else: + data = patch_body.dict(exclude=non_update_fields) + for key, val in data.items(): + setattr(variable, key, val) + session.add(variable) + return variable diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 393fad520a679..e3c0ef3ab4daf 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -191,6 +191,9 @@ export type DagServicePatchDagsMutationResult = Awaited< export type DagServicePatchDagMutationResult = Awaited< ReturnType >; +export type VariableServicePatchVariableMutationResult = Awaited< + ReturnType +>; export type ConnectionServiceDeleteConnectionMutationResult = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 01e919cce53bf..b4c8cf9fea5c9 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -14,7 +14,7 @@ import { DashboardService, VariableService, } from "../requests/services.gen"; -import { DAGPatchBody, DagRunState } from "../requests/types.gen"; +import { DAGPatchBody, DagRunState, VariableBody } from "../requests/types.gen"; import * as Common from "./common"; /** @@ -428,6 +428,53 @@ export const useDagServicePatchDag = < }) as unknown as Promise, ...options, }); +/** + * Patch Variable + * Update a variable by key. + * @param data The data for the request. + * @param data.variableKey + * @param data.requestBody + * @param data.updateMask + * @returns VariableResponse Successful Response + * @throws ApiError + */ +export const useVariableServicePatchVariable = < + TData = Common.VariableServicePatchVariableMutationResult, + TError = unknown, + TContext = unknown, +>( + options?: Omit< + UseMutationOptions< + TData, + TError, + { + requestBody: VariableBody; + updateMask?: string[]; + variableKey: string; + }, + TContext + >, + "mutationFn" + >, +) => + useMutation< + TData, + TError, + { + requestBody: VariableBody; + updateMask?: string[]; + variableKey: string; + }, + TContext + >({ + mutationFn: ({ requestBody, updateMask, variableKey }) => + VariableService.patchVariable({ + requestBody, + updateMask, + variableKey, + }) as unknown as Promise, + ...options, + }); /** * Delete Connection * Delete a connection entry. diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 18df5284651b7..e42a3f6572ca9 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -526,7 +526,7 @@ export const $DAGPatchBody = { type: "object", required: ["is_paused"], title: "DAGPatchBody", - description: "Dag Serializer for updatable body.", + description: "Dag Serializer for updatable bodies.", } as const; export const $DAGResponse = { @@ -1182,12 +1182,23 @@ export const $ValidationError = { title: "ValidationError", } as const; -export const $VariableResponse = { +export const $VariableBody = { properties: { key: { type: "string", title: "Key", }, + description: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Description", + }, value: { anyOf: [ { @@ -1199,6 +1210,19 @@ export const $VariableResponse = { ], title: "Value", }, + }, + type: "object", + required: ["key", "description", "value"], + title: "VariableBody", + description: "Variable serializer for bodies.", +} as const; + +export const $VariableResponse = { + properties: { + key: { + type: "string", + title: "Key", + }, description: { anyOf: [ { @@ -1210,9 +1234,20 @@ export const $VariableResponse = { ], title: "Description", }, + value: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Value", + }, }, type: "object", - required: ["key", "value", "description"], + required: ["key", "description", "value"], title: "VariableResponse", description: "Variable serializer for responses.", } as const; diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index f0cbc099370f8..43d9e8d940df4 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -25,6 +25,8 @@ import type { DeleteVariableResponse, GetVariableData, GetVariableResponse, + PatchVariableData, + PatchVariableResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, @@ -364,6 +366,40 @@ export class VariableService { }, }); } + + /** + * Patch Variable + * Update a variable by key. + * @param data The data for the request. + * @param data.variableKey + * @param data.requestBody + * @param data.updateMask + * @returns VariableResponse Successful Response + * @throws ApiError + */ + public static patchVariable( + data: PatchVariableData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "PATCH", + url: "/public/variables/{variable_key}", + path: { + variable_key: data.variableKey, + }, + query: { + update_mask: data.updateMask, + }, + body: data.requestBody, + mediaType: "application/json", + errors: { + 400: "Bad Request", + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Validation Error", + }, + }); + } } export class DagRunService { diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 5fe3615c7d4e6..3deba451fced4 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -79,7 +79,7 @@ export type DAGDetailsResponse = { }; /** - * Dag Serializer for updatable body. + * Dag Serializer for updatable bodies. */ export type DAGPatchBody = { is_paused: boolean; @@ -250,13 +250,22 @@ export type ValidationError = { type: string; }; +/** + * Variable serializer for bodies. + */ +export type VariableBody = { + key: string; + description: string | null; + value: string | null; +}; + /** * Variable serializer for responses. */ export type VariableResponse = { key: string; - value: string | null; description: string | null; + value: string | null; }; export type NextRunAssetsData = { @@ -348,6 +357,14 @@ export type GetVariableData = { export type GetVariableResponse = VariableResponse; +export type PatchVariableData = { + requestBody: VariableBody; + updateMask?: Array | null; + variableKey: string; +}; + +export type PatchVariableResponse = VariableResponse; + export type GetDagRunData = { dagId: string; dagRunId: string; @@ -637,6 +654,35 @@ export type $OpenApiTs = { 422: HTTPValidationError; }; }; + patch: { + req: PatchVariableData; + res: { + /** + * Successful Response + */ + 200: VariableResponse; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; }; "/public/dags/{dag_id}/dagRuns/{dag_run_id}": { get: { diff --git a/tests/api_fastapi/views/public/test_variables.py b/tests/api_fastapi/views/public/test_variables.py index 7957d0bf22249..cf5c78fd562cf 100644 --- a/tests/api_fastapi/views/public/test_variables.py +++ b/tests/api_fastapi/views/public/test_variables.py @@ -135,3 +135,91 @@ def test_get_should_respond_404(self, test_client): assert response.status_code == 404 body = response.json() assert f"The Variable with key: `{TEST_VARIABLE_KEY}` was not found" == body["detail"] + + +class TestPatchVariable(TestVariableEndpoint): + @pytest.mark.enable_redact + @pytest.mark.parametrize( + "key, body, params, expected_response", + [ + ( + TEST_VARIABLE_KEY, + { + "key": TEST_VARIABLE_KEY, + "value": "The new value", + "description": "The new description", + }, + None, + { + "key": TEST_VARIABLE_KEY, + "value": "The new value", + "description": "The new description", + }, + ), + ( + TEST_VARIABLE_KEY, + { + "key": TEST_VARIABLE_KEY, + "value": "The new value", + "description": "The new description", + }, + {"update_mask": ["value"]}, + { + "key": TEST_VARIABLE_KEY, + "value": "The new value", + "description": TEST_VARIABLE_DESCRIPTION, + }, + ), + ( + TEST_VARIABLE_KEY2, + { + "key": TEST_VARIABLE_KEY2, + "value": "some_other_value", + "description": TEST_VARIABLE_DESCRIPTION2, + }, + None, + { + "key": TEST_VARIABLE_KEY2, + "value": "***", + "description": TEST_VARIABLE_DESCRIPTION2, + }, + ), + ( + TEST_VARIABLE_KEY3, + { + "key": TEST_VARIABLE_KEY3, + "value": '{"password": "new_password"}', + "description": "new description", + }, + {"update_mask": ["value", "description"]}, + { + "key": TEST_VARIABLE_KEY3, + "value": '{"password": "***"}', + "description": "new description", + }, + ), + ], + ) + def test_patch_should_respond_200(self, test_client, session, key, body, params, expected_response): + self.create_variable() + response = test_client.patch(f"/public/variables/{key}", json=body, params=params) + assert response.status_code == 200 + assert response.json() == expected_response + + def test_patch_should_respond_400(self, test_client): + response = test_client.patch( + f"/public/variables/{TEST_VARIABLE_KEY}", + json={"key": "different_key", "value": "some_value", "description": None}, + ) + assert response.status_code == 400 + body = response.json() + assert "Invalid body, key from request body doesn't match uri parameter" == body["detail"] + + def test_patch_should_respond_404(self, test_client): + response = test_client.patch( + f"/public/variables/{TEST_VARIABLE_KEY}", + json={"key": TEST_VARIABLE_KEY, "value": "some_value", "description": None}, + ) + assert response.status_code == 404 + body = response.json() + assert f"The Variable with key: `{TEST_VARIABLE_KEY}` was not found" == body["detail"] From da4ed783e48dff9989859e81900514783e88cba7 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 14 Oct 2024 17:00:07 +0100 Subject: [PATCH 104/199] Commit the session between writing and deletion of RTIF (#42928) * Use different sessions in writing and deletion of RTIF Previously, this was how it was done, but now, a session was used for both the writing and deletion of RTIF, which we suspect caused StaleDataError. The related PR: https://github.com/apache/airflow/pull/38565 This PR brings back the old behaviour of using different sessions for writing/deleting RTIFs * fixup! Use different sessions in writing and deletion of RTIF * add test and use flush --- airflow/models/taskinstance.py | 3 +- tests/models/test_renderedtifields.py | 50 ++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 5b51bb0d24df8..c1373e5d6a12d 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1632,11 +1632,12 @@ def _get_previous_ti( @internal_api_call @provide_session -def _update_rtif(ti, rendered_fields, session: Session | None = None): +def _update_rtif(ti, rendered_fields, session: Session = NEW_SESSION): from airflow.models.renderedtifields import RenderedTaskInstanceFields rtif = RenderedTaskInstanceFields(ti=ti, render_templates=False, rendered_fields=rendered_fields) RenderedTaskInstanceFields.write(rtif, session=session) + session.flush() RenderedTaskInstanceFields.delete_old_records(ti.task_id, ti.dag_id, session=session) diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index ea22d31871db9..6ff87b28a89b6 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -24,13 +24,16 @@ from datetime import date, timedelta from unittest import mock +import pendulum import pytest +from sqlalchemy import select from airflow import settings from airflow.configuration import conf from airflow.decorators import task as task_decorator -from airflow.models import Variable +from airflow.models import DagRun, Variable from airflow.models.renderedtifields import RenderedTaskInstanceFields as RTIF +from airflow.operators.python import PythonOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.timezone import datetime @@ -386,3 +389,48 @@ def test_redact(self, redact, dag_maker): "env": "val 2", "cwd": "val 3", } + + @pytest.mark.skip_if_database_isolation_mode + def test_rtif_deletion_stale_data_error(self, dag_maker, session): + """ + Here we verify bad behavior. When we rerun a task whose RTIF + will get removed, we get a stale data error. + """ + with dag_maker(dag_id="test_retry_handling"): + task = PythonOperator( + task_id="test_retry_handling_op", + python_callable=lambda a, b: print(f"{a}\n{b}\n"), + op_args=[ + "dag {{dag.dag_id}};", + "try_number {{ti.try_number}};yo", + ], + ) + + def run_task(date): + run_id = f"abc_{date.to_date_string()}" + dr = session.scalar(select(DagRun).where(DagRun.execution_date == date, DagRun.run_id == run_id)) + if not dr: + dr = dag_maker.create_dagrun(execution_date=date, run_id=run_id) + ti = dr.task_instances[0] + ti.state = None + ti.try_number += 1 + session.commit() + ti.task = task + ti.run() + return dr + + base_date = pendulum.datetime(2021, 1, 1) + exec_dates = [base_date.add(days=x) for x in range(40)] + for date_ in exec_dates: + run_task(date=date_) + + session.commit() + session.expunge_all() + + # find oldest date + date = session.scalar( + select(DagRun.execution_date).join(RTIF.dag_run).order_by(DagRun.execution_date).limit(1) + ) + date = pendulum.instance(date) + # rerun the old date. this will fail + run_task(date=date) From 1840ec65eed4e1feb6f02815a2e8a76d5bee0d74 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Mon, 14 Oct 2024 22:28:28 +0530 Subject: [PATCH 105/199] Refactor FastApi Dag and DagRun endpoints tests (#42949) * wip - failing * update tests * update * update tests * update tests * fix * fix * fix mypy error * fix --- .../api_fastapi/views/public/test_dag_run.py | 52 +- tests/api_fastapi/views/public/test_dags.py | 477 +++++++++--------- 2 files changed, 270 insertions(+), 259 deletions(-) diff --git a/tests/api_fastapi/views/public/test_dag_run.py b/tests/api_fastapi/views/public/test_dag_run.py index f8d6780e50a3a..377aca4553d86 100644 --- a/tests/api_fastapi/views/public/test_dag_run.py +++ b/tests/api_fastapi/views/public/test_dag_run.py @@ -110,32 +110,32 @@ def setup(dag_maker, session=None): session.commit() -@pytest.mark.parametrize( - "dag_id, run_id, state, run_type, triggered_by, dag_run_note", - [ - (DAG1_ID, DAG1_RUN1_ID, DAG1_RUN1_STATE, DAG1_RUN1_RUN_TYPE, DAG1_RUN1_TRIGGERED_BY, DAG1_NOTE), - (DAG1_ID, DAG1_RUN2_ID, DAG1_RUN2_STATE, DAG1_RUN2_RUN_TYPE, DAG1_RUN2_TRIGGERED_BY, None), - (DAG2_ID, DAG2_RUN1_ID, DAG2_RUN1_STATE, DAG2_RUN1_RUN_TYPE, DAG2_RUN1_TRIGGERED_BY, None), - (DAG2_ID, DAG2_RUN2_ID, DAG2_RUN2_STATE, DAG2_RUN2_RUN_TYPE, DAG2_RUN2_TRIGGERED_BY, None), - ], -) -def test_get_dag_run(test_client, dag_id, run_id, state, run_type, triggered_by, dag_run_note): - response = test_client.get(f"/public/dags/{dag_id}/dagRuns/{run_id}") - assert response.status_code == 200 - body = response.json() - assert body["dag_id"] == dag_id - assert body["run_id"] == run_id - assert body["state"] == state - assert body["run_type"] == run_type - assert body["triggered_by"] == triggered_by.value - assert body["note"] == dag_run_note - - -def test_get_dag_run_not_found(test_client): - response = test_client.get(f"/public/dags/{DAG1_ID}/dagRuns/invalid") - assert response.status_code == 404 - body = response.json() - assert body["detail"] == "The DagRun with dag_id: `test_dag1` and run_id: `invalid` was not found" +class TestGetDagRun: + @pytest.mark.parametrize( + "dag_id, run_id, state, run_type, triggered_by, dag_run_note", + [ + (DAG1_ID, DAG1_RUN1_ID, DAG1_RUN1_STATE, DAG1_RUN1_RUN_TYPE, DAG1_RUN1_TRIGGERED_BY, DAG1_NOTE), + (DAG1_ID, DAG1_RUN2_ID, DAG1_RUN2_STATE, DAG1_RUN2_RUN_TYPE, DAG1_RUN2_TRIGGERED_BY, None), + (DAG2_ID, DAG2_RUN1_ID, DAG2_RUN1_STATE, DAG2_RUN1_RUN_TYPE, DAG2_RUN1_TRIGGERED_BY, None), + (DAG2_ID, DAG2_RUN2_ID, DAG2_RUN2_STATE, DAG2_RUN2_RUN_TYPE, DAG2_RUN2_TRIGGERED_BY, None), + ], + ) + def test_get_dag_run(self, test_client, dag_id, run_id, state, run_type, triggered_by, dag_run_note): + response = test_client.get(f"/public/dags/{dag_id}/dagRuns/{run_id}") + assert response.status_code == 200 + body = response.json() + assert body["dag_id"] == dag_id + assert body["run_id"] == run_id + assert body["state"] == state + assert body["run_type"] == run_type + assert body["triggered_by"] == triggered_by.value + assert body["note"] == dag_run_note + + def test_get_dag_run_not_found(self, test_client): + response = test_client.get(f"/public/dags/{DAG1_ID}/dagRuns/invalid") + assert response.status_code == 404 + body = response.json() + assert body["detail"] == "The DagRun with dag_id: `test_dag1` and run_id: `invalid` was not found" class TestDeleteDagRun: diff --git a/tests/api_fastapi/views/public/test_dags.py b/tests/api_fastapi/views/public/test_dags.py index 13520b37b3ff2..8bc10407e6287 100644 --- a/tests/api_fastapi/views/public/test_dags.py +++ b/tests/api_fastapi/views/public/test_dags.py @@ -114,238 +114,249 @@ def setup(dag_maker, session=None) -> None: _create_deactivated_paused_dag() -@pytest.mark.parametrize( - "query_params, expected_total_entries, expected_ids", - [ - # Filters - ({}, 2, [DAG1_ID, DAG2_ID]), - ({"limit": 1}, 2, [DAG1_ID]), - ({"offset": 1}, 2, [DAG2_ID]), - ({"tags": ["example"]}, 1, [DAG1_ID]), - ({"only_active": False}, 3, [DAG1_ID, DAG2_ID, DAG3_ID]), - ({"paused": True, "only_active": False}, 1, [DAG3_ID]), - ({"paused": False}, 2, [DAG1_ID, DAG2_ID]), - ({"owners": ["airflow"]}, 2, [DAG1_ID, DAG2_ID]), - ({"owners": ["test_owner"], "only_active": False}, 1, [DAG3_ID]), - ({"last_dag_run_state": "success", "only_active": False}, 1, [DAG3_ID]), - ({"last_dag_run_state": "failed", "only_active": False}, 1, [DAG1_ID]), - # # Sort - ({"order_by": "-dag_id"}, 2, [DAG2_ID, DAG1_ID]), - ({"order_by": "-dag_display_name"}, 2, [DAG2_ID, DAG1_ID]), - ({"order_by": "dag_display_name"}, 2, [DAG1_ID, DAG2_ID]), - ({"order_by": "next_dagrun", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), - ({"order_by": "last_run_state", "only_active": False}, 3, [DAG1_ID, DAG3_ID, DAG2_ID]), - ({"order_by": "-last_run_state", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), - ( - {"order_by": "last_run_start_date", "only_active": False}, - 3, - [DAG1_ID, DAG3_ID, DAG2_ID], - ), - ( - {"order_by": "-last_run_start_date", "only_active": False}, - 3, - [DAG3_ID, DAG1_ID, DAG2_ID], - ), - # Search - ({"dag_id_pattern": "1"}, 1, [DAG1_ID]), - ({"dag_display_name_pattern": "test_dag2"}, 1, [DAG2_ID]), - ], -) -def test_get_dags(test_client, query_params, expected_total_entries, expected_ids): - response = test_client.get("/public/dags", params=query_params) - - assert response.status_code == 200 - body = response.json() - - assert body["total_entries"] == expected_total_entries - assert [dag["dag_id"] for dag in body["dags"]] == expected_ids - - -@pytest.mark.parametrize( - "query_params, dag_id, body, expected_status_code, expected_is_paused", - [ - ({}, "fake_dag_id", {"is_paused": True}, 404, None), - ({"update_mask": ["field_1", "is_paused"]}, DAG1_ID, {"is_paused": True}, 400, None), - ({}, DAG1_ID, {"is_paused": True}, 200, True), - ({}, DAG1_ID, {"is_paused": False}, 200, False), - ({"update_mask": ["is_paused"]}, DAG1_ID, {"is_paused": True}, 200, True), - ({"update_mask": ["is_paused"]}, DAG1_ID, {"is_paused": False}, 200, False), - ], -) -def test_patch_dag(test_client, query_params, dag_id, body, expected_status_code, expected_is_paused): - response = test_client.patch(f"/public/dags/{dag_id}", json=body, params=query_params) - - assert response.status_code == expected_status_code - if expected_status_code == 200: - body = response.json() - assert body["is_paused"] == expected_is_paused - - -@pytest.mark.parametrize( - "query_params, body, expected_status_code, expected_ids, expected_paused_ids", - [ - ({"update_mask": ["field_1", "is_paused"]}, {"is_paused": True}, 400, None, None), - ( - {"only_active": False}, - {"is_paused": True}, - 200, - [], - [], - ), # no-op because the dag_id_pattern is not provided - ( - {"only_active": False, "dag_id_pattern": "~"}, - {"is_paused": True}, - 200, - [DAG1_ID, DAG2_ID, DAG3_ID], - [DAG1_ID, DAG2_ID, DAG3_ID], - ), - ( - {"only_active": False, "dag_id_pattern": "~"}, - {"is_paused": False}, - 200, - [DAG1_ID, DAG2_ID, DAG3_ID], - [], - ), - ( - {"dag_id_pattern": "~"}, - {"is_paused": True}, - 200, - [DAG1_ID, DAG2_ID], - [DAG1_ID, DAG2_ID], - ), - ( - {"dag_id_pattern": "dag1"}, - {"is_paused": True}, - 200, - [DAG1_ID], - [DAG1_ID], - ), - ], -) -def test_patch_dags(test_client, query_params, body, expected_status_code, expected_ids, expected_paused_ids): - response = test_client.patch("/public/dags", json=body, params=query_params) - - assert response.status_code == expected_status_code - if expected_status_code == 200: +class TestGetDags: + @pytest.mark.parametrize( + "query_params, expected_total_entries, expected_ids", + [ + # Filters + ({}, 2, [DAG1_ID, DAG2_ID]), + ({"limit": 1}, 2, [DAG1_ID]), + ({"offset": 1}, 2, [DAG2_ID]), + ({"tags": ["example"]}, 1, [DAG1_ID]), + ({"only_active": False}, 3, [DAG1_ID, DAG2_ID, DAG3_ID]), + ({"paused": True, "only_active": False}, 1, [DAG3_ID]), + ({"paused": False}, 2, [DAG1_ID, DAG2_ID]), + ({"owners": ["airflow"]}, 2, [DAG1_ID, DAG2_ID]), + ({"owners": ["test_owner"], "only_active": False}, 1, [DAG3_ID]), + ({"last_dag_run_state": "success", "only_active": False}, 1, [DAG3_ID]), + ({"last_dag_run_state": "failed", "only_active": False}, 1, [DAG1_ID]), + # # Sort + ({"order_by": "-dag_id"}, 2, [DAG2_ID, DAG1_ID]), + ({"order_by": "-dag_display_name"}, 2, [DAG2_ID, DAG1_ID]), + ({"order_by": "dag_display_name"}, 2, [DAG1_ID, DAG2_ID]), + ({"order_by": "next_dagrun", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), + ({"order_by": "last_run_state", "only_active": False}, 3, [DAG1_ID, DAG3_ID, DAG2_ID]), + ({"order_by": "-last_run_state", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), + ( + {"order_by": "last_run_start_date", "only_active": False}, + 3, + [DAG1_ID, DAG3_ID, DAG2_ID], + ), + ( + {"order_by": "-last_run_start_date", "only_active": False}, + 3, + [DAG3_ID, DAG1_ID, DAG2_ID], + ), + # Search + ({"dag_id_pattern": "1"}, 1, [DAG1_ID]), + ({"dag_display_name_pattern": "test_dag2"}, 1, [DAG2_ID]), + ], + ) + def test_get_dags(self, test_client, query_params, expected_total_entries, expected_ids): + response = test_client.get("/public/dags", params=query_params) + + assert response.status_code == 200 body = response.json() + + assert body["total_entries"] == expected_total_entries assert [dag["dag_id"] for dag in body["dags"]] == expected_ids - paused_dag_ids = [dag["dag_id"] for dag in body["dags"] if dag["is_paused"]] - assert paused_dag_ids == expected_paused_ids - - -@pytest.mark.parametrize( - "query_params, dag_id, expected_status_code, dag_display_name, start_date", - [ - ({}, "fake_dag_id", 404, "fake_dag", datetime(2023, 12, 31, tzinfo=timezone.utc)), - ({}, DAG2_ID, 200, DAG2_ID, DAG2_START_DATE), - ], -) -def test_dag_details(test_client, query_params, dag_id, expected_status_code, dag_display_name, start_date): - response = test_client.get(f"/public/dags/{dag_id}/details", params=query_params) - assert response.status_code == expected_status_code - if expected_status_code != 200: - return - - # Match expected and actual responses below. - res_json = response.json() - last_parsed = res_json["last_parsed"] - last_parsed_time = res_json["last_parsed_time"] - file_token = res_json["file_token"] - expected = { - "catchup": True, - "concurrency": 16, - "dag_id": dag_id, - "dag_display_name": dag_display_name, - "dag_run_timeout": None, - "dataset_expression": None, - "default_view": "grid", - "description": None, - "doc_md": "details", - "end_date": None, - "fileloc": "/opt/airflow/tests/api_fastapi/views/public/test_dags.py", - "file_token": file_token, - "has_import_errors": False, - "has_task_concurrency_limits": True, - "is_active": True, - "is_paused": False, - "is_paused_upon_creation": None, - "last_expired": None, - "last_parsed": last_parsed, - "last_parsed_time": last_parsed_time, - "last_pickled": None, - "max_active_runs": 16, - "max_active_tasks": 16, - "max_consecutive_failed_dag_runs": 0, - "next_dagrun": None, - "next_dagrun_create_after": None, - "next_dagrun_data_interval_end": None, - "next_dagrun_data_interval_start": None, - "orientation": "LR", - "owners": ["airflow"], - "params": { - "foo": { - "__class": "airflow.models.param.Param", - "description": None, - "schema": {}, - "value": 1, - } - }, - "pickle_id": None, - "render_template_as_native_obj": False, - "timetable_summary": None, - "scheduler_lock": None, - "start_date": start_date.replace(tzinfo=None).isoformat() + "Z", # pydantic datetime format - "tags": [], - "template_search_path": None, - "timetable_description": "Never, external triggers only", - "timezone": UTC_JSON_REPR, - } - assert res_json == expected - - -@pytest.mark.parametrize( - "query_params, dag_id, expected_status_code, dag_display_name", - [ - ({}, "fake_dag_id", 404, "fake_dag"), - ({}, DAG2_ID, 200, DAG2_ID), - ], -) -def test_get_dag(test_client, query_params, dag_id, expected_status_code, dag_display_name): - response = test_client.get(f"/public/dags/{dag_id}", params=query_params) - assert response.status_code == expected_status_code - if expected_status_code != 200: - return - - # Match expected and actual responses below. - res_json = response.json() - last_parsed_time = res_json["last_parsed_time"] - file_token = res_json["file_token"] - expected = { - "dag_id": dag_id, - "dag_display_name": dag_display_name, - "description": None, - "fileloc": "/opt/airflow/tests/api_fastapi/views/public/test_dags.py", - "file_token": file_token, - "is_paused": False, - "is_active": True, - "owners": ["airflow"], - "timetable_summary": None, - "tags": [], - "next_dagrun": None, - "has_task_concurrency_limits": True, - "next_dagrun_data_interval_start": None, - "next_dagrun_data_interval_end": None, - "max_active_runs": 16, - "max_consecutive_failed_dag_runs": 0, - "next_dagrun_create_after": None, - "last_expired": None, - "max_active_tasks": 16, - "last_pickled": None, - "default_view": "grid", - "last_parsed_time": last_parsed_time, - "scheduler_lock": None, - "timetable_description": "Never, external triggers only", - "has_import_errors": False, - "pickle_id": None, - } - assert res_json == expected + + +class TestPatchDag: + @pytest.mark.parametrize( + "query_params, dag_id, body, expected_status_code, expected_is_paused", + [ + ({}, "fake_dag_id", {"is_paused": True}, 404, None), + ({"update_mask": ["field_1", "is_paused"]}, DAG1_ID, {"is_paused": True}, 400, None), + ({}, DAG1_ID, {"is_paused": True}, 200, True), + ({}, DAG1_ID, {"is_paused": False}, 200, False), + ({"update_mask": ["is_paused"]}, DAG1_ID, {"is_paused": True}, 200, True), + ({"update_mask": ["is_paused"]}, DAG1_ID, {"is_paused": False}, 200, False), + ], + ) + def test_patch_dag( + self, test_client, query_params, dag_id, body, expected_status_code, expected_is_paused + ): + response = test_client.patch(f"/public/dags/{dag_id}", json=body, params=query_params) + + assert response.status_code == expected_status_code + if expected_status_code == 200: + body = response.json() + assert body["is_paused"] == expected_is_paused + + +class TestPatchDags: + @pytest.mark.parametrize( + "query_params, body, expected_status_code, expected_ids, expected_paused_ids", + [ + ({"update_mask": ["field_1", "is_paused"]}, {"is_paused": True}, 400, None, None), + ( + {"only_active": False}, + {"is_paused": True}, + 200, + [], + [], + ), # no-op because the dag_id_pattern is not provided + ( + {"only_active": False, "dag_id_pattern": "~"}, + {"is_paused": True}, + 200, + [DAG1_ID, DAG2_ID, DAG3_ID], + [DAG1_ID, DAG2_ID, DAG3_ID], + ), + ( + {"only_active": False, "dag_id_pattern": "~"}, + {"is_paused": False}, + 200, + [DAG1_ID, DAG2_ID, DAG3_ID], + [], + ), + ( + {"dag_id_pattern": "~"}, + {"is_paused": True}, + 200, + [DAG1_ID, DAG2_ID], + [DAG1_ID, DAG2_ID], + ), + ( + {"dag_id_pattern": "dag1"}, + {"is_paused": True}, + 200, + [DAG1_ID], + [DAG1_ID], + ), + ], + ) + def test_patch_dags( + self, test_client, query_params, body, expected_status_code, expected_ids, expected_paused_ids + ): + response = test_client.patch("/public/dags", json=body, params=query_params) + + assert response.status_code == expected_status_code + if expected_status_code == 200: + body = response.json() + assert [dag["dag_id"] for dag in body["dags"]] == expected_ids + paused_dag_ids = [dag["dag_id"] for dag in body["dags"] if dag["is_paused"]] + assert paused_dag_ids == expected_paused_ids + + +class TestDagDetails: + @pytest.mark.parametrize( + "query_params, dag_id, expected_status_code, dag_display_name, start_date", + [ + ({}, "fake_dag_id", 404, "fake_dag", datetime(2023, 12, 31, tzinfo=timezone.utc)), + ({}, DAG2_ID, 200, DAG2_ID, DAG2_START_DATE), + ], + ) + def test_dag_details( + self, test_client, query_params, dag_id, expected_status_code, dag_display_name, start_date + ): + response = test_client.get(f"/public/dags/{dag_id}/details", params=query_params) + assert response.status_code == expected_status_code + if expected_status_code != 200: + return + + # Match expected and actual responses below. + res_json = response.json() + last_parsed = res_json["last_parsed"] + last_parsed_time = res_json["last_parsed_time"] + file_token = res_json["file_token"] + expected = { + "catchup": True, + "concurrency": 16, + "dag_id": dag_id, + "dag_display_name": dag_display_name, + "dag_run_timeout": None, + "dataset_expression": None, + "default_view": "grid", + "description": None, + "doc_md": "details", + "end_date": None, + "fileloc": "/opt/airflow/tests/api_fastapi/views/public/test_dags.py", + "file_token": file_token, + "has_import_errors": False, + "has_task_concurrency_limits": True, + "is_active": True, + "is_paused": False, + "is_paused_upon_creation": None, + "last_expired": None, + "last_parsed": last_parsed, + "last_parsed_time": last_parsed_time, + "last_pickled": None, + "max_active_runs": 16, + "max_active_tasks": 16, + "max_consecutive_failed_dag_runs": 0, + "next_dagrun": None, + "next_dagrun_create_after": None, + "next_dagrun_data_interval_end": None, + "next_dagrun_data_interval_start": None, + "orientation": "LR", + "owners": ["airflow"], + "params": { + "foo": { + "__class": "airflow.models.param.Param", + "description": None, + "schema": {}, + "value": 1, + } + }, + "pickle_id": None, + "render_template_as_native_obj": False, + "timetable_summary": None, + "scheduler_lock": None, + "start_date": start_date.replace(tzinfo=None).isoformat() + "Z", # pydantic datetime format + "tags": [], + "template_search_path": None, + "timetable_description": "Never, external triggers only", + "timezone": UTC_JSON_REPR, + } + assert res_json == expected + + +class TestGetDag: + @pytest.mark.parametrize( + "query_params, dag_id, expected_status_code, dag_display_name", + [ + ({}, "fake_dag_id", 404, "fake_dag"), + ({}, DAG2_ID, 200, DAG2_ID), + ], + ) + def test_get_dag(self, test_client, query_params, dag_id, expected_status_code, dag_display_name): + response = test_client.get(f"/public/dags/{dag_id}", params=query_params) + assert response.status_code == expected_status_code + if expected_status_code != 200: + return + + # Match expected and actual responses below. + res_json = response.json() + last_parsed_time = res_json["last_parsed_time"] + file_token = res_json["file_token"] + expected = { + "dag_id": dag_id, + "dag_display_name": dag_display_name, + "description": None, + "fileloc": "/opt/airflow/tests/api_fastapi/views/public/test_dags.py", + "file_token": file_token, + "is_paused": False, + "is_active": True, + "owners": ["airflow"], + "timetable_summary": None, + "tags": [], + "next_dagrun": None, + "has_task_concurrency_limits": True, + "next_dagrun_data_interval_start": None, + "next_dagrun_data_interval_end": None, + "max_active_runs": 16, + "max_consecutive_failed_dag_runs": 0, + "next_dagrun_create_after": None, + "last_expired": None, + "max_active_tasks": 16, + "last_pickled": None, + "default_view": "grid", + "last_parsed_time": last_parsed_time, + "scheduler_lock": None, + "timetable_description": "Never, external triggers only", + "has_import_errors": False, + "pickle_id": None, + } + assert res_json == expected From e67b5b2f9100e63fbc0e732e2e5771e1687b179c Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Mon, 14 Oct 2024 18:10:12 +0100 Subject: [PATCH 106/199] Minor updates in UI contributing docs (#43013) --- contributing-docs/14_node_environment_setup.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contributing-docs/14_node_environment_setup.rst b/contributing-docs/14_node_environment_setup.rst index 81ced88240ac5..25635a89765e1 100644 --- a/contributing-docs/14_node_environment_setup.rst +++ b/contributing-docs/14_node_environment_setup.rst @@ -36,8 +36,8 @@ Committers will exercise their judgement on what endpoints should exist in the p Airflow UI ---------- -``airflow/ui`` is our React frontend powered. Dependencies are managed by pnpm and dev/build processes by `Vite `__ -Make sure you are using recent versions of ``pnpm\>=9`` and ``node/>=20``. ``breeze start-airflow`` will build the UI automatically. +``airflow/ui`` is our React frontend powered. Dependencies are managed by pnpm and dev/build processes by `Vite `__. +Make sure you are using recent versions of ``pnpm>=9`` and ``node>=20``. ``breeze start-airflow`` will build the UI automatically. Adding the ``--dev-mode`` flag will automatically run the vite dev server for hot reloading the UI during local development. pnpm commands From 651df6b6a54ef25ce124918c890f5dcd8cd2ffec Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Tue, 15 Oct 2024 03:26:35 +0800 Subject: [PATCH 107/199] AIP-84 post variable (#42948) --- .../endpoints/variable_endpoint.py | 1 + airflow/api_fastapi/openapi/v1-generated.yaml | 38 ++++++++++++++ airflow/api_fastapi/views/public/variables.py | 17 ++++++- airflow/ui/openapi-gen/queries/common.ts | 3 ++ airflow/ui/openapi-gen/queries/queries.ts | 39 +++++++++++++++ .../ui/openapi-gen/requests/services.gen.ts | 26 ++++++++++ airflow/ui/openapi-gen/requests/types.gen.ts | 29 +++++++++++ .../views/public/test_variables.py | 50 +++++++++++++++++++ 8 files changed, 201 insertions(+), 2 deletions(-) diff --git a/airflow/api_connexion/endpoints/variable_endpoint.py b/airflow/api_connexion/endpoints/variable_endpoint.py index b1d9e2f5c8b6f..20e7ce1edeabe 100644 --- a/airflow/api_connexion/endpoints/variable_endpoint.py +++ b/airflow/api_connexion/endpoints/variable_endpoint.py @@ -130,6 +130,7 @@ def patch_variable( return variable_schema.dump(variable) +@mark_fastapi_migration_done @security.requires_access_variable("POST") @action_logging( event=action_event_from_permission( diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index 759ab7fdd8e9b..235410a6d347a 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -695,6 +695,44 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/variables/: + post: + tags: + - Variable + summary: Post Variable + description: Create a variable. + operationId: post_variable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/VariableBody' + required: true + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/VariableResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + '403': + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /public/dags/{dag_id}/dagRuns/{dag_run_id}: get: tags: diff --git a/airflow/api_fastapi/views/public/variables.py b/airflow/api_fastapi/views/public/variables.py index b4c07e23de293..a61b9bb930023 100644 --- a/airflow/api_fastapi/views/public/variables.py +++ b/airflow/api_fastapi/views/public/variables.py @@ -73,10 +73,23 @@ async def patch_variable( if not variable: raise HTTPException(404, f"The Variable with key: `{variable_key}` was not found") if update_mask: - data = patch_body.dict(include=set(update_mask) - non_update_fields) + data = patch_body.model_dump(include=set(update_mask) - non_update_fields) else: - data = patch_body.dict(exclude=non_update_fields) + data = patch_body.model_dump(exclude=non_update_fields) for key, val in data.items(): setattr(variable, key, val) session.add(variable) return variable + + +@variables_router.post("/", status_code=201, responses=create_openapi_http_exception_doc([401, 403])) +async def post_variable( + post_body: VariableBody, + session: Annotated[Session, Depends(get_session)], +) -> VariableResponse: + """Create a variable.""" + Variable.set(**post_body.model_dump(), session=session) + + variable = session.scalar(select(Variable).where(Variable.key == post_body.key).limit(1)) + + return VariableResponse.model_validate(variable, from_attributes=True) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index e3c0ef3ab4daf..426e28447fd9c 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -185,6 +185,9 @@ export const UseDagRunServiceGetDagRunKeyFn = ( }, queryKey?: Array, ) => [useDagRunServiceGetDagRunKey, ...(queryKey ?? [{ dagId, dagRunId }])]; +export type VariableServicePostVariableMutationResult = Awaited< + ReturnType +>; export type DagServicePatchDagsMutationResult = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index b4c8cf9fea5c9..557a7ba8ffa17 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -295,6 +295,45 @@ export const useDagRunServiceGetDagRun = < queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, ...options, }); +/** + * Post Variable + * Create a variable. + * @param data The data for the request. + * @param data.requestBody + * @returns VariableResponse Successful Response + * @throws ApiError + */ +export const useVariableServicePostVariable = < + TData = Common.VariableServicePostVariableMutationResult, + TError = unknown, + TContext = unknown, +>( + options?: Omit< + UseMutationOptions< + TData, + TError, + { + requestBody: VariableBody; + }, + TContext + >, + "mutationFn" + >, +) => + useMutation< + TData, + TError, + { + requestBody: VariableBody; + }, + TContext + >({ + mutationFn: ({ requestBody }) => + VariableService.postVariable({ + requestBody, + }) as unknown as Promise, + ...options, + }); /** * Patch Dags * Patch multiple DAGs. diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 43d9e8d940df4..78b113c7f2f80 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -27,6 +27,8 @@ import type { GetVariableResponse, PatchVariableData, PatchVariableResponse, + PostVariableData, + PostVariableResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, @@ -400,6 +402,30 @@ export class VariableService { }, }); } + + /** + * Post Variable + * Create a variable. + * @param data The data for the request. + * @param data.requestBody + * @returns VariableResponse Successful Response + * @throws ApiError + */ + public static postVariable( + data: PostVariableData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "POST", + url: "/public/variables/", + body: data.requestBody, + mediaType: "application/json", + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 422: "Validation Error", + }, + }); + } } export class DagRunService { diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 3deba451fced4..856517d560fe4 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -365,6 +365,12 @@ export type PatchVariableData = { export type PatchVariableResponse = VariableResponse; +export type PostVariableData = { + requestBody: VariableBody; +}; + +export type PostVariableResponse = VariableResponse; + export type GetDagRunData = { dagId: string; dagRunId: string; @@ -684,6 +690,29 @@ export type $OpenApiTs = { }; }; }; + "/public/variables/": { + post: { + req: PostVariableData; + res: { + /** + * Successful Response + */ + 201: VariableResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; "/public/dags/{dag_id}/dagRuns/{dag_run_id}": { get: { req: GetDagRunData; diff --git a/tests/api_fastapi/views/public/test_variables.py b/tests/api_fastapi/views/public/test_variables.py index cf5c78fd562cf..58a09538a971a 100644 --- a/tests/api_fastapi/views/public/test_variables.py +++ b/tests/api_fastapi/views/public/test_variables.py @@ -223,3 +223,53 @@ def test_patch_should_respond_404(self, test_client): assert response.status_code == 404 body = response.json() assert f"The Variable with key: `{TEST_VARIABLE_KEY}` was not found" == body["detail"] + + +class TestPostVariable(TestVariableEndpoint): + @pytest.mark.enable_redact + @pytest.mark.parametrize( + "body, expected_response", + [ + ( + { + "key": "new variable key", + "value": "new variable value", + "description": "new variable description", + }, + { + "key": "new variable key", + "value": "new variable value", + "description": "new variable description", + }, + ), + ( + { + "key": "another_password", + "value": "password_value", + "description": "another password", + }, + { + "key": "another_password", + "value": "***", + "description": "another password", + }, + ), + ( + { + "key": "another value with sensitive information", + "value": '{"password": "new_password"}', + "description": "some description", + }, + { + "key": "another value with sensitive information", + "value": '{"password": "***"}', + "description": "some description", + }, + ), + ], + ) + def test_post_should_respond_201(self, test_client, session, body, expected_response): + self.create_variable() + response = test_client.post("/public/variables/", json=body) + assert response.status_code == 201 + assert response.json() == expected_response From 8fa8c51c35127ac39c140f0e102155ebe24bd849 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 15 Oct 2024 02:41:36 +0200 Subject: [PATCH 108/199] Feature: Added event_handler parameter in MSGraphAsyncOperator (#42539) * refactor: Added parameter in MSGraphAsyncOperator to allow overriding default event_handler * docs: Added docstring for event_handler parameter in MSGraphAsyncOperator * refactor: Fixed TestMSGraphAsyncOperator * refactor: Check if event is not None * refactor: Register the TextParseNodeFactory and JsonParseNodeFactory so error messages get handled correctly in RequestAdapter * refactor: Reorganized import for TestMSGraphAsyncOperator * refactor: Added missing kiota-serialization packages in azure provider * refactor: Updated provider dependencies * refactor: Reorganized import of TestKiotaRequestAdapterHook * refactor: Downgraded version of json kiota serialization * refactor: Updated provider dependencies * refactor: Put import of Context in TYPE_CHECKING block * refactor: Fixed lookup of tenant-id * refactor: Fixed kiota serialization dependencies to 1.0.0 to avoid pendulum dependency issues for backward compatibility * refactor: Updated provider dependencies * refactored: Fixed import of test_utils in test_dag_run --------- Co-authored-by: David Blain --- generated/provider_dependencies.json | 2 + .../microsoft/azure/hooks/msgraph.py | 7 +++ .../microsoft/azure/operators/msgraph.py | 18 ++++++-- .../providers/microsoft/azure/provider.yaml | 2 + .../microsoft/azure/hooks/test_msgraph.py | 45 ++++++++++++++++++- .../microsoft/azure/operators/test_msgraph.py | 30 +++++++++++++ 6 files changed, 99 insertions(+), 5 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 4d921fc1fb911..8efdd5eae78ad 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -828,6 +828,8 @@ "azure-synapse-artifacts>=0.17.0", "azure-synapse-spark>=0.2.0", "microsoft-kiota-http>=1.3.0,!=1.3.4", + "microsoft-kiota-serialization-json==1.0.0", + "microsoft-kiota-serialization-text==1.0.0", "msgraph-core>=1.0.0" ], "devel-deps": [ diff --git a/providers/src/airflow/providers/microsoft/azure/hooks/msgraph.py b/providers/src/airflow/providers/microsoft/azure/hooks/msgraph.py index 61e555f4caa78..4ab3aaf3ba37f 100644 --- a/providers/src/airflow/providers/microsoft/azure/hooks/msgraph.py +++ b/providers/src/airflow/providers/microsoft/azure/hooks/msgraph.py @@ -32,11 +32,14 @@ from kiota_abstractions.method import Method from kiota_abstractions.request_information import RequestInformation from kiota_abstractions.response_handler import ResponseHandler +from kiota_abstractions.serialization import ParseNodeFactoryRegistry from kiota_authentication_azure.azure_identity_authentication_provider import ( AzureIdentityAuthenticationProvider, ) from kiota_http.httpx_request_adapter import HttpxRequestAdapter from kiota_http.middleware.options import ResponseHandlerOption +from kiota_serialization_json.json_parse_node_factory import JsonParseNodeFactory +from kiota_serialization_text.text_parse_node_factory import TextParseNodeFactory from msgraph_core import APIVersion, GraphClientFactory from msgraph_core._enums import NationalClouds @@ -249,8 +252,12 @@ def get_conn(self) -> RequestAdapter: scopes=scopes, allowed_hosts=allowed_hosts, ) + parse_node_factory = ParseNodeFactoryRegistry() + parse_node_factory.CONTENT_TYPE_ASSOCIATED_FACTORIES["text/plain"] = TextParseNodeFactory() + parse_node_factory.CONTENT_TYPE_ASSOCIATED_FACTORIES["application/json"] = JsonParseNodeFactory() request_adapter = HttpxRequestAdapter( authentication_provider=auth_provider, + parse_node_factory=parse_node_factory, http_client=http_client, base_url=base_url, ) diff --git a/providers/src/airflow/providers/microsoft/azure/operators/msgraph.py b/providers/src/airflow/providers/microsoft/azure/operators/msgraph.py index b3d14b14a57ec..0d187ebd5144b 100644 --- a/providers/src/airflow/providers/microsoft/azure/operators/msgraph.py +++ b/providers/src/airflow/providers/microsoft/azure/operators/msgraph.py @@ -44,6 +44,14 @@ from airflow.utils.context import Context +def default_event_handler(context: Context, event: dict[Any, Any] | None = None) -> Any: + if event: + if event.get("status") == "failure": + raise AirflowException(event.get("message")) + + return event.get("response") + + class MSGraphAsyncOperator(BaseOperator): """ A Microsoft Graph API operator which allows you to execute REST call to the Microsoft Graph API. @@ -69,6 +77,9 @@ class MSGraphAsyncOperator(BaseOperator): :param result_processor: Function to further process the response from MS Graph API (default is lambda: context, response: response). When the response returned by the `KiotaRequestAdapterHook` are bytes, then those will be base64 encoded into a string. + :param event_handler: Function to process the event returned from `MSGraphTrigger`. By default, when the + event returned by the `MSGraphTrigger` has a failed status, an AirflowException is being raised with + the message from the event, otherwise the response from the event payload is returned. :param serializer: Class which handles response serialization (default is ResponseSerializer). Bytes will be base64 encoded into a string, so it can be stored as an XCom. """ @@ -102,6 +113,7 @@ def __init__( api_version: APIVersion | str | None = None, pagination_function: Callable[[MSGraphAsyncOperator, dict, Context], tuple[str, dict]] | None = None, result_processor: Callable[[Context, Any], Any] = lambda context, result: result, + event_handler: Callable[[Context, dict[Any, Any] | None], Any] | None = None, serializer: type[ResponseSerializer] = ResponseSerializer, **kwargs: Any, ): @@ -121,6 +133,7 @@ def __init__( self.api_version = api_version self.pagination_function = pagination_function or self.paginate self.result_processor = result_processor + self.event_handler = event_handler or default_event_handler self.serializer: ResponseSerializer = serializer() def execute(self, context: Context) -> None: @@ -158,10 +171,7 @@ def execute_complete( if event: self.log.debug("%s completed with %s: %s", self.task_id, event.get("status"), event) - if event.get("status") == "failure": - raise AirflowException(event.get("message")) - - response = event.get("response") + response = self.event_handler(context, event) self.log.debug("response: %s", response) diff --git a/providers/src/airflow/providers/microsoft/azure/provider.yaml b/providers/src/airflow/providers/microsoft/azure/provider.yaml index cf0b3f75ef9d8..c4831a641b333 100644 --- a/providers/src/airflow/providers/microsoft/azure/provider.yaml +++ b/providers/src/airflow/providers/microsoft/azure/provider.yaml @@ -111,6 +111,8 @@ dependencies: # msgraph-core has transient import failures with microsoft-kiota-http==1.3.4 # See https://github.com/microsoftgraph/msgraph-sdk-python-core/issues/706 - microsoft-kiota-http>=1.3.0,!=1.3.4 + - microsoft-kiota-serialization-json==1.0.0 + - microsoft-kiota-serialization-text==1.0.0 devel-dependencies: - pywinrm diff --git a/providers/tests/microsoft/azure/hooks/test_msgraph.py b/providers/tests/microsoft/azure/hooks/test_msgraph.py index 0ecad98548b51..aff5d0226a1c4 100644 --- a/providers/tests/microsoft/azure/hooks/test_msgraph.py +++ b/providers/tests/microsoft/azure/hooks/test_msgraph.py @@ -19,11 +19,15 @@ import asyncio from json import JSONDecodeError from typing import TYPE_CHECKING -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest +from httpx import Response from kiota_http.httpx_request_adapter import HttpxRequestAdapter +from kiota_serialization_json.json_parse_node import JsonParseNode +from kiota_serialization_text.text_parse_node import TextParseNode from msgraph_core import APIVersion, NationalClouds +from opentelemetry.trace import Span from airflow.exceptions import AirflowBadRequest, AirflowException, AirflowNotFoundException from airflow.providers.microsoft.azure.hooks.msgraph import ( @@ -175,6 +179,45 @@ def test_encoded_query_parameters(self): assert actual == {"%24expand": "reports,users,datasets,dataflows,dashboards", "%24top": 5000} + @pytest.mark.asyncio + async def test_throw_failed_responses_with_text_plain_content_type(self): + with patch( + "airflow.hooks.base.BaseHook.get_connection", + side_effect=get_airflow_connection, + ): + hook = KiotaRequestAdapterHook(conn_id="msgraph_api") + response = Mock(spec=Response) + response.headers = {"content-type": "text/plain"} + response.status_code = 429 + response.content = b"TenantThrottleThresholdExceeded" + response.is_success = False + span = Mock(spec=Span) + + actual = await hook.get_conn().get_root_parse_node(response, span, span) + + assert isinstance(actual, TextParseNode) + assert actual.get_str_value() == "TenantThrottleThresholdExceeded" + + @pytest.mark.asyncio + async def test_throw_failed_responses_with_application_json_content_type(self): + with patch( + "airflow.hooks.base.BaseHook.get_connection", + side_effect=get_airflow_connection, + ): + hook = KiotaRequestAdapterHook(conn_id="msgraph_api") + response = Mock(spec=Response) + response.headers = {"content-type": "application/json"} + response.status_code = 429 + response.content = b'{"error": {"code": "TenantThrottleThresholdExceeded"}}' + response.is_success = False + span = Mock(spec=Span) + + actual = await hook.get_conn().get_root_parse_node(response, span, span) + + assert isinstance(actual, JsonParseNode) + error_code = actual.get_child_node("error").get_child_node("code").get_str_value() + assert error_code == "TenantThrottleThresholdExceeded" + class TestResponseHandler: def test_default_response_handler_when_json(self): diff --git a/providers/tests/microsoft/azure/operators/test_msgraph.py b/providers/tests/microsoft/azure/operators/test_msgraph.py index 372152fe979ee..fe404e48e6f0a 100644 --- a/providers/tests/microsoft/azure/operators/test_msgraph.py +++ b/providers/tests/microsoft/azure/operators/test_msgraph.py @@ -19,6 +19,7 @@ import json import locale from base64 import b64encode +from typing import TYPE_CHECKING, Any import pytest @@ -35,6 +36,9 @@ mock_response, ) +if TYPE_CHECKING: + from airflow.utils.context import Context + class TestMSGraphAsyncOperator(Base): @pytest.mark.db_test @@ -101,6 +105,32 @@ def test_execute_when_an_exception_occurs(self): with pytest.raises(AirflowException): self.execute_operator(operator) + @pytest.mark.db_test + def test_execute_when_an_exception_occurs_on_custom_event_handler(self): + with self.patch_hook_and_request_adapter(AirflowException("An error occurred")): + + def custom_event_handler(context: Context, event: dict[Any, Any] | None = None): + if event: + if event.get("status") == "failure": + return None + + return event.get("response") + + operator = MSGraphAsyncOperator( + task_id="users_delta", + conn_id="msgraph_api", + url="users/delta", + event_handler=custom_event_handler, + ) + + results, events = self.execute_operator(operator) + + assert not results + assert len(events) == 1 + assert isinstance(events[0], TriggerEvent) + assert events[0].payload["status"] == "failure" + assert events[0].payload["message"] == "An error occurred" + @pytest.mark.db_test def test_execute_when_response_is_bytes(self): content = load_file("resources", "dummy.pdf", mode="rb", encoding=None) From f177da1bb54e54de5e42676b06da438f83bfb282 Mon Sep 17 00:00:00 2001 From: Gopal Dirisala <39794726+dirrao@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:25:16 +0530 Subject: [PATCH 109/199] uv version bump to 0.4.20 (#42905) --- Dockerfile | 2 +- Dockerfile.ci | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b5fa8d0a2b44..c762cae900f2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.10.2" ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.17 +ARG AIRFLOW_UV_VERSION=0.4.20 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index 7e0ee74556f9d..df1cdf8c27385 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1278,7 +1278,7 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.17 +ARG AIRFLOW_UV_VERSION=0.4.20 ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller From 07c180e1da80f93d433aa715331ead11c69998d5 Mon Sep 17 00:00:00 2001 From: Gopal Dirisala <39794726+dirrao@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:26:03 +0530 Subject: [PATCH 110/199] Removed deprecated Chainable type from BaseOperator (#42776) * Removed deprecated Chainable type from BaseOperator * Chainable word removed from spelling_wordlist.txt * Chainable word removed from spelling_wordlist.txt * Chainable word removed from spelling_wordlist.txt * news fragment added --- airflow/models/baseoperator.py | 5 ----- docs/spelling_wordlist.txt | 1 - newsfragments/42776.significant.rst | 1 + 3 files changed, 1 insertion(+), 6 deletions(-) create mode 100644 newsfragments/42776.significant.rst diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 0c3d119be1983..514553e05a2dd 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -45,7 +45,6 @@ NoReturn, Sequence, TypeVar, - Union, cast, ) @@ -1789,10 +1788,6 @@ def expand_start_trigger_args(self, *, context: Context, session: Session) -> St return self.start_trigger_args -# TODO: Deprecate for Airflow 3.0 -Chainable = Union[DependencyMixin, Sequence[DependencyMixin]] - - def chain(*tasks: DependencyMixin | Sequence[DependencyMixin]) -> None: r""" Given a number of tasks, builds a dependency chain. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 4413be1fc746a..78025bfe9fe22 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -227,7 +227,6 @@ cgi Cgroups cgroups cgroupspy -Chainable chakra Changelog changelog diff --git a/newsfragments/42776.significant.rst b/newsfragments/42776.significant.rst new file mode 100644 index 0000000000000..5fad7dfe79b66 --- /dev/null +++ b/newsfragments/42776.significant.rst @@ -0,0 +1 @@ +Removed deprecated ``Chainable`` type from ``BaseOperator``. From acb779ae227f5dc50c67a155f8f9df725bb0034c Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Tue, 15 Oct 2024 07:17:31 +0530 Subject: [PATCH 111/199] pin in providers (#43001) --- generated/provider_dependencies.json | 6 +++--- providers/src/airflow/providers/apache/livy/provider.yaml | 2 +- providers/src/airflow/providers/dbt/cloud/provider.yaml | 2 +- providers/src/airflow/providers/http/provider.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 8efdd5eae78ad..49b457cd81a70 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -243,7 +243,7 @@ "aiohttp>=3.9.2", "apache-airflow-providers-http", "apache-airflow>=2.8.0", - "asgiref" + "asgiref>=2.3.0" ], "devel-deps": [], "plugins": [], @@ -472,7 +472,7 @@ "aiohttp>=3.9.2", "apache-airflow-providers-http", "apache-airflow>=2.8.0", - "asgiref" + "asgiref>=2.3.0" ], "devel-deps": [], "plugins": [], @@ -746,7 +746,7 @@ "deps": [ "aiohttp>=3.9.2", "apache-airflow>=2.8.0", - "asgiref", + "asgiref>=2.3.0", "requests>=2.27.0,<3", "requests_toolbelt" ], diff --git a/providers/src/airflow/providers/apache/livy/provider.yaml b/providers/src/airflow/providers/apache/livy/provider.yaml index 2e149246aee74..48c5524808dc6 100644 --- a/providers/src/airflow/providers/apache/livy/provider.yaml +++ b/providers/src/airflow/providers/apache/livy/provider.yaml @@ -58,7 +58,7 @@ dependencies: - apache-airflow>=2.8.0 - apache-airflow-providers-http - aiohttp>=3.9.2 - - asgiref + - asgiref>=2.3.0 integrations: - integration-name: Apache Livy diff --git a/providers/src/airflow/providers/dbt/cloud/provider.yaml b/providers/src/airflow/providers/dbt/cloud/provider.yaml index 828aecf335435..4caa2eeba0019 100644 --- a/providers/src/airflow/providers/dbt/cloud/provider.yaml +++ b/providers/src/airflow/providers/dbt/cloud/provider.yaml @@ -59,7 +59,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - apache-airflow-providers-http - - asgiref + - asgiref>=2.3.0 - aiohttp>=3.9.2 additional-extras: diff --git a/providers/src/airflow/providers/http/provider.yaml b/providers/src/airflow/providers/http/provider.yaml index c329428987276..e7a9b9ef94f84 100644 --- a/providers/src/airflow/providers/http/provider.yaml +++ b/providers/src/airflow/providers/http/provider.yaml @@ -67,7 +67,7 @@ dependencies: - requests>=2.27.0,<3 - requests_toolbelt - aiohttp>=3.9.2 - - asgiref + - asgiref>=2.3.0 integrations: - integration-name: Hypertext Transfer Protocol (HTTP) From ce97729192e9bee7ba7a3dfb6153154a09bd6c5a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 15 Oct 2024 04:43:07 +0200 Subject: [PATCH 112/199] Move tests_common from "dev" to top-level. (#42985) * Move tests common without changes * Fix docstrings in tests_common * Move tests_common from "dev" to top-level. Follow-up after #42505 fixing teething problem with tests_common. Originally in #42505 common test code was moved to "dev" folder, but the "dev" folder is really dedicated to "build" scripts and the problem with moving "tests_common" to the folder was that the whole "dev" folder is replaced (for non-committer PRs) with the content from the target branch. This is done for security reasons, because we can accidentally use any of the scripts from dev in the CI build scripts and we might not notice, which will open us to a security issue where a file in "dev" coming from PR could be accidentally executed during the "pull_request_target" workflow - which would expose our secrets and GitHub Package write permissions to a contributor coming from a fork. This change moves the files, fixes pre-commit specification and docs, also fixes a number of "doc" issues detected by "ruff" in the tests_common folder as they were detected after the move. The tests_common folder is added to folders mounted when breeze is executed with local folders mounted (in order to avoid accidental mounting of randomly generated files to inside the breeze container). All imports for the common tests were updated to reflect this move. --- .dockerignore | 1 + .pre-commit-config.yaml | 8 +-- Dockerfile.ci | 1 + contributing-docs/testing/system_tests.rst | 2 +- contributing-docs/testing/unit_tests.rst | 6 +-- .../utils/docker_command_utils.py | 1 + .../airflow_breeze/utils/selective_checks.py | 2 +- dev/breeze/tests/test_selective_checks.py | 4 +- dev/perf/scheduler_dag_execution_timing.py | 2 +- .../airflow_aux/test_pod_template_file.py | 1 + .../cloud/log/test_oss_task_handler.py | 5 +- .../aws/auth_manager/avp/test_facade.py | 3 +- .../aws/auth_manager/cli/test_avp_commands.py | 5 +- .../test_aws_security_manager_override.py | 3 +- .../aws/auth_manager/test_aws_auth_manager.py | 7 ++- .../aws/auth_manager/views/test_auth.py | 5 +- .../executors/batch/test_batch_executor.py | 5 +- .../aws/executors/ecs/test_ecs_executor.py | 7 ++- .../tests/amazon/aws/hooks/test_base_aws.py | 3 +- providers/tests/amazon/aws/hooks/test_s3.py | 3 +- .../tests/amazon/aws/links/test_base_aws.py | 5 +- .../aws/log/test_cloudwatch_task_handler.py | 3 +- .../amazon/aws/log/test_s3_task_handler.py | 3 +- .../aws/secrets/test_systems_manager.py | 3 +- .../aws/transfers/test_redshift_to_s3.py | 3 +- .../aws/transfers/test_s3_to_redshift.py | 3 +- .../amazon/aws/transfers/test_s3_to_sftp.py | 3 +- .../amazon/aws/transfers/test_sftp_to_s3.py | 3 +- .../tests/apache/hive/hooks/test_hive.py | 2 +- .../tests/apache/livy/hooks/test_livy.py | 3 +- .../apache/spark/hooks/test_spark_sql.py | 3 +- .../tests/atlassian/jira/hooks/test_jira.py | 3 +- .../atlassian/jira/operators/test_jira.py | 3 +- .../tests/atlassian/jira/sensors/test_jira.py | 3 +- .../tests/celery/cli/test_celery_command.py | 5 +- .../celery/executors/test_celery_executor.py | 7 ++- .../celery/log_handlers/test_log_handlers.py | 5 +- .../kubernetes/cli/test_kubernetes_command.py | 3 +- .../executors/test_kubernetes_executor.py | 5 +- .../cncf/kubernetes/hooks/test_kubernetes.py | 5 +- .../log_handlers/test_log_handlers.py | 5 +- .../cncf/kubernetes/operators/test_pod.py | 3 +- .../operators/test_spark_kubernetes.py | 3 +- .../tests/cncf/kubernetes/test_client.py | 3 +- .../kubernetes/test_template_rendering.py | 3 +- .../common/io/operators/test_file_transfer.py | 4 +- .../tests/common/io/xcom/test_backend.py | 11 ++-- .../tests/common/sql/hooks/test_dbapi.py | 3 +- providers/tests/common/sql/hooks/test_sql.py | 2 +- .../tests/common/sql/hooks/test_sqlparse.py | 3 +- .../tests/common/sql/operators/test_sql.py | 3 +- .../common/sql/operators/test_sql_execute.py | 3 +- .../tests/common/sql/sensors/test_sql.py | 3 +- providers/tests/common/sql/test_utils.py | 3 +- providers/tests/conftest.py | 2 +- .../plugins/test_databricks_workflow.py | 3 +- .../api_endpoints/test_rpc_api_endpoint.py | 5 +- providers/tests/edge/cli/test_edge_command.py | 3 +- .../edge/plugins/test_edge_executor_plugin.py | 3 +- .../elasticsearch/log/test_es_task_handler.py | 4 +- .../api/auth/backend/test_basic_auth.py | 3 +- .../api/auth/backend/test_kerberos_auth.py | 2 +- .../api/auth/backend/test_session.py | 3 +- .../api_endpoints/api_connexion_utils.py | 2 +- .../api_endpoints/test_asset_endpoint.py | 6 +-- .../auth_manager/api_endpoints/test_auth.py | 11 ++-- .../api_endpoints/test_backfill_endpoint.py | 14 ++--- .../auth_manager/api_endpoints/test_cors.py | 7 ++- .../api_endpoints/test_dag_endpoint.py | 6 +-- .../api_endpoints/test_dag_run_endpoint.py | 4 +- .../api_endpoints/test_dag_source_endpoint.py | 4 +- .../test_dag_warning_endpoint.py | 4 +- .../api_endpoints/test_event_log_endpoint.py | 4 +- .../test_import_error_endpoint.py | 6 +-- .../test_role_and_permission_endpoint.py | 4 +- .../test_task_instance_endpoint.py | 4 +- .../api_endpoints/test_user_endpoint.py | 6 +-- .../api_endpoints/test_variable_endpoint.py | 4 +- .../api_endpoints/test_xcom_endpoint.py | 4 +- .../cli_commands/test_definition.py | 2 +- .../cli_commands/test_role_command.py | 5 +- .../cli_commands/test_sync_perm_command.py | 3 +- .../cli_commands/test_user_command.py | 3 +- .../auth_manager/cli_commands/test_utils.py | 3 +- providers/tests/fab/auth_manager/conftest.py | 5 +- .../fab/auth_manager/decorators/test_auth.py | 3 +- .../models/test_anonymous_user.py | 2 +- .../auth_manager/schemas/test_user_schema.py | 2 +- .../security_manager/test_constants.py | 2 +- .../security_manager/test_override.py | 2 +- .../fab/auth_manager/test_fab_auth_manager.py | 2 +- .../tests/fab/auth_manager/test_models.py | 3 +- .../tests/fab/auth_manager/test_security.py | 12 ++--- .../auth_manager/views/test_permissions.py | 4 +- .../fab/auth_manager/views/test_roles_list.py | 4 +- .../tests/fab/auth_manager/views/test_user.py | 4 +- .../fab/auth_manager/views/test_user_edit.py | 4 +- .../fab/auth_manager/views/test_user_stats.py | 4 +- .../cloud/hooks/test_bigquery_system.py | 2 +- .../google/cloud/hooks/test_kms_system.py | 2 +- .../cloud/hooks/test_secret_manager_system.py | 2 +- .../google/cloud/log/test_gcs_task_handler.py | 5 +- .../cloud/log/test_gcs_task_handler_system.py | 14 ++--- .../log/test_stackdriver_task_handler.py | 7 ++- .../test_stackdriver_task_handler_system.py | 14 ++--- .../google/cloud/operators/test_bigquery.py | 13 +++-- .../cloud/operators/test_bigquery_dts.py | 3 +- .../cloud/operators/test_dataprep_system.py | 7 ++- .../google/cloud/operators/test_dataproc.py | 5 +- .../cloud/operators/test_datastore_system.py | 4 +- .../google/cloud/operators/test_looker.py | 3 +- .../secrets/test_secret_manager_system.py | 2 +- .../test_facebook_ads_to_gcs_system.py | 10 ++-- .../test_salesforce_to_gcs_system.py | 12 ++--- .../google/cloud/utils/gcp_authenticator.py | 6 +-- .../common/auth_backend/test_google_openid.py | 9 ++-- .../operators/test_display_video_system.py | 12 ++--- providers/tests/hashicorp/hooks/test_vault.py | 3 +- providers/tests/imap/hooks/test_imap.py | 3 +- .../azure/log/test_wasb_task_handler.py | 5 +- providers/tests/mongo/hooks/test_mongo.py | 3 +- providers/tests/mysql/hooks/test_mysql.py | 4 +- .../tests/openlineage/extractors/test_bash.py | 3 +- .../openlineage/extractors/test_manager.py | 3 +- .../openlineage/extractors/test_python.py | 3 +- .../tests/openlineage/plugins/test_adapter.py | 5 +- .../openlineage/plugins/test_execution.py | 5 +- .../openlineage/plugins/test_listener.py | 5 +- .../openlineage/plugins/test_openlineage.py | 5 +- .../tests/openlineage/plugins/test_utils.py | 3 +- providers/tests/openlineage/test_conf.py | 3 +- .../tests/openlineage/utils/test_utils.py | 8 +-- .../opensearch/log/test_os_task_handler.py | 4 +- .../redis/log/test_redis_task_handler.py | 3 +- providers/tests/sftp/operators/test_sftp.py | 3 +- providers/tests/smtp/hooks/test_smtp.py | 3 +- .../tests/smtp/notifications/test_smtp.py | 5 +- providers/tests/ssh/operators/test_ssh.py | 3 +- .../tests/standard/operators/test_bash.py | 3 +- .../tests/standard/operators/test_datetime.py | 3 +- .../tests/standard/operators/test_weekday.py | 3 +- .../tests/standard/sensors/test_weekday.py | 3 +- .../airbyte/example_airbyte_trigger_job.py | 2 +- .../system/alibaba/example_adb_spark_batch.py | 4 +- .../system/alibaba/example_adb_spark_sql.py | 4 +- .../system/alibaba/example_oss_bucket.py | 4 +- .../system/alibaba/example_oss_object.py | 4 +- providers/tests/system/amazon/CONTRIBUTING.md | 2 +- .../system/amazon/aws/example_appflow.py | 4 +- .../system/amazon/aws/example_appflow_run.py | 4 +- .../tests/system/amazon/aws/example_athena.py | 4 +- .../amazon/aws/example_azure_blob_to_s3.py | 4 +- .../tests/system/amazon/aws/example_batch.py | 4 +- .../system/amazon/aws/example_bedrock.py | 4 +- .../example_bedrock_retrieve_and_generate.py | 4 +- .../amazon/aws/example_cloudformation.py | 4 +- .../system/amazon/aws/example_comprehend.py | 4 +- .../example_comprehend_document_classifier.py | 4 +- .../system/amazon/aws/example_datasync.py | 4 +- .../tests/system/amazon/aws/example_dms.py | 4 +- .../system/amazon/aws/example_dynamodb.py | 4 +- .../amazon/aws/example_dynamodb_to_s3.py | 4 +- .../tests/system/amazon/aws/example_ec2.py | 4 +- .../tests/system/amazon/aws/example_ecs.py | 4 +- .../system/amazon/aws/example_ecs_fargate.py | 4 +- .../amazon/aws/example_eks_templated.py | 4 +- .../example_eks_with_fargate_in_one_step.py | 4 +- .../aws/example_eks_with_fargate_profile.py | 4 +- .../example_eks_with_nodegroup_in_one_step.py | 4 +- .../amazon/aws/example_eks_with_nodegroups.py | 4 +- .../tests/system/amazon/aws/example_emr.py | 4 +- .../system/amazon/aws/example_emr_eks.py | 4 +- .../aws/example_emr_notebook_execution.py | 4 +- .../amazon/aws/example_emr_serverless.py | 4 +- .../system/amazon/aws/example_eventbridge.py | 2 +- .../system/amazon/aws/example_ftp_to_s3.py | 4 +- .../system/amazon/aws/example_gcs_to_s3.py | 4 +- .../amazon/aws/example_glacier_to_gcs.py | 4 +- .../tests/system/amazon/aws/example_glue.py | 4 +- .../amazon/aws/example_glue_data_quality.py | 4 +- ...e_glue_data_quality_with_recommendation.py | 4 +- .../amazon/aws/example_glue_databrew.py | 4 +- .../aws/example_google_api_sheets_to_s3.py | 4 +- .../aws/example_google_api_youtube_to_s3.py | 4 +- .../amazon/aws/example_hive_to_dynamodb.py | 4 +- .../system/amazon/aws/example_http_to_s3.py | 4 +- .../aws/example_imap_attachment_to_s3.py | 4 +- .../amazon/aws/example_kinesis_analytics.py | 4 +- .../tests/system/amazon/aws/example_lambda.py | 4 +- .../system/amazon/aws/example_local_to_s3.py | 4 +- .../system/amazon/aws/example_mongo_to_s3.py | 4 +- .../system/amazon/aws/example_neptune.py | 4 +- .../system/amazon/aws/example_quicksight.py | 4 +- .../system/amazon/aws/example_rds_event.py | 4 +- .../system/amazon/aws/example_rds_export.py | 4 +- .../system/amazon/aws/example_rds_instance.py | 4 +- .../system/amazon/aws/example_rds_snapshot.py | 4 +- .../system/amazon/aws/example_redshift.py | 4 +- .../aws/example_redshift_s3_transfers.py | 4 +- .../tests/system/amazon/aws/example_s3.py | 4 +- .../amazon/aws/example_s3_to_dynamodb.py | 4 +- .../system/amazon/aws/example_s3_to_ftp.py | 4 +- .../system/amazon/aws/example_s3_to_sftp.py | 4 +- .../system/amazon/aws/example_s3_to_sql.py | 5 +- .../system/amazon/aws/example_sagemaker.py | 4 +- .../amazon/aws/example_sagemaker_endpoint.py | 4 +- .../amazon/aws/example_sagemaker_notebook.py | 4 +- .../amazon/aws/example_sagemaker_pipeline.py | 4 +- .../amazon/aws/example_salesforce_to_s3.py | 4 +- .../system/amazon/aws/example_sftp_to_s3.py | 4 +- .../tests/system/amazon/aws/example_sns.py | 4 +- .../system/amazon/aws/example_sql_to_s3.py | 4 +- .../tests/system/amazon/aws/example_sqs.py | 4 +- .../amazon/aws/example_step_functions.py | 4 +- .../amazon/aws/tests/test_aws_auth_manager.py | 4 +- .../tests/system/apache/beam/example_beam.py | 2 +- .../apache/beam/example_beam_java_flink.py | 2 +- .../apache/beam/example_beam_java_spark.py | 2 +- .../tests/system/apache/beam/example_go.py | 2 +- .../system/apache/beam/example_go_dataflow.py | 2 +- .../apache/beam/example_java_dataflow.py | 2 +- .../system/apache/beam/example_python.py | 2 +- .../apache/beam/example_python_async.py | 2 +- .../apache/beam/example_python_dataflow.py | 2 +- .../apache/cassandra/example_cassandra_dag.py | 2 +- .../system/apache/drill/example_drill_dag.py | 2 +- .../system/apache/druid/example_druid_dag.py | 2 +- .../system/apache/hive/example_twitter_dag.py | 4 +- .../system/apache/iceberg/example_iceberg.py | 2 +- .../kafka/example_dag_event_listener.py | 2 +- .../apache/kafka/example_dag_hello_kafka.py | 2 +- .../system/apache/kylin/example_kylin_dag.py | 4 +- .../tests/system/apache/livy/example_livy.py | 4 +- .../tests/system/apache/pig/example_pig.py | 2 +- .../system/apache/pinot/example_pinot_dag.py | 2 +- .../system/apache/spark/example_pyspark.py | 2 +- .../system/apache/spark/example_spark_dag.py | 2 +- providers/tests/system/asana/example_asana.py | 4 +- .../cncf/kubernetes/example_kubernetes.py | 4 +- .../kubernetes/example_kubernetes_async.py | 4 +- .../example_kubernetes_decorator.py | 2 +- .../cncf/kubernetes/example_kubernetes_job.py | 4 +- .../kubernetes/example_kubernetes_resource.py | 4 +- .../kubernetes/example_spark_kubernetes.py | 4 +- .../example_cohere_embedding_operator.py | 2 +- .../io/example_file_transfer_local_to_s3.py | 4 +- .../sql/example_sql_column_table_check.py | 2 +- .../common/sql/example_sql_execute_query.py | 2 +- .../system/databricks/example_databricks.py | 4 +- .../databricks/example_databricks_repos.py | 4 +- .../databricks/example_databricks_sensors.py | 4 +- .../databricks/example_databricks_sql.py | 4 +- .../databricks/example_databricks_workflow.py | 4 +- .../system/dbt/cloud/example_dbt_cloud.py | 8 +-- .../tests/system/dingding/example_dingding.py | 4 +- .../tests/system/docker/example_docker.py | 2 +- .../system/docker/example_docker_copy_data.py | 2 +- .../system/docker/example_docker_swarm.py | 2 +- .../example_taskflow_api_docker_virtualenv.py | 8 ++- .../example_elasticsearch_query.py | 2 +- providers/tests/system/ftp/example_ftp.py | 4 +- .../tests/system/github/example_github.py | 2 +- .../tests/system/google/ads/example_ads.py | 4 +- .../cloud/automl/example_automl_dataset.py | 4 +- .../automl/example_automl_translation.py | 4 +- .../example_automl_video_classification.py | 4 +- .../automl/example_automl_video_tracking.py | 4 +- .../example_automl_vision_classification.py | 4 +- .../example_automl_vision_object_detection.py | 4 +- .../cloud/azure/example_azure_blob_to_gcs.py | 4 +- .../azure/example_azure_fileshare_to_gcs.py | 4 +- .../bigquery/example_bigquery_dataset.py | 4 +- .../cloud/bigquery/example_bigquery_dts.py | 4 +- .../bigquery/example_bigquery_operations.py | 4 +- .../example_bigquery_operations_location.py | 4 +- .../bigquery/example_bigquery_queries.py | 4 +- .../example_bigquery_queries_async.py | 4 +- .../bigquery/example_bigquery_sensors.py | 4 +- .../cloud/bigquery/example_bigquery_tables.py | 4 +- .../bigquery/example_bigquery_to_bigquery.py | 4 +- .../cloud/bigquery/example_bigquery_to_gcs.py | 4 +- .../bigquery/example_bigquery_to_gcs_async.py | 4 +- .../bigquery/example_bigquery_to_mssql.py | 4 +- .../bigquery/example_bigquery_to_mysql.py | 4 +- .../bigquery/example_bigquery_to_postgres.py | 4 +- .../bigquery/example_bigquery_transfer.py | 4 +- .../bigquery/example_bigquery_value_check.py | 6 +-- .../google/cloud/bigtable/example_bigtable.py | 4 +- .../cloud/cloud_batch/example_cloud_batch.py | 4 +- .../cloud/cloud_build/example_cloud_build.py | 4 +- .../example_cloud_build_trigger.py | 4 +- .../cloud_functions/example_functions.py | 4 +- .../example_cloud_memorystore_memcached.py | 4 +- .../example_cloud_memorystore_redis.py | 4 +- .../cloud/cloud_run/example_cloud_run.py | 4 +- .../cloud_run/example_cloud_run_service.py | 4 +- .../cloud/cloud_sql/example_cloud_sql.py | 4 +- .../cloud_sql/example_cloud_sql_query.py | 4 +- .../cloud_sql/example_cloud_sql_query_ssl.py | 4 +- .../cloud/composer/example_cloud_composer.py | 4 +- .../google/cloud/compute/example_compute.py | 4 +- .../cloud/compute/example_compute_igm.py | 4 +- .../cloud/compute/example_compute_ssh.py | 4 +- .../compute/example_compute_ssh_os_login.py | 4 +- .../compute/example_compute_ssh_parallel.py | 4 +- .../example_dlp_deidentify_content.py | 4 +- .../example_dlp_info_types.py | 4 +- .../example_dlp_inspect_template.py | 4 +- .../data_loss_prevention/example_dlp_job.py | 4 +- .../example_dlp_job_trigger.py | 4 +- .../cloud/dataflow/example_dataflow_go.py | 2 +- .../dataflow/example_dataflow_native_java.py | 4 +- .../example_dataflow_native_python.py | 4 +- .../example_dataflow_native_python_async.py | 4 +- .../dataflow/example_dataflow_pipeline.py | 4 +- .../example_dataflow_sensors_deferrable.py | 4 +- .../cloud/dataflow/example_dataflow_sql.py | 4 +- .../example_dataflow_streaming_python.py | 4 +- .../dataflow/example_dataflow_template.py | 4 +- .../cloud/dataflow/example_dataflow_yaml.py | 4 +- .../google/cloud/dataform/example_dataform.py | 4 +- .../cloud/datafusion/example_datafusion.py | 4 +- .../datapipelines/example_datapipeline.py | 4 +- .../google/cloud/dataplex/example_dataplex.py | 4 +- .../cloud/dataplex/example_dataplex_dp.py | 4 +- .../cloud/dataplex/example_dataplex_dq.py | 4 +- .../google/cloud/dataprep/example_dataprep.py | 4 +- .../cloud/dataproc/example_dataproc_batch.py | 4 +- .../example_dataproc_batch_deferrable.py | 4 +- .../example_dataproc_batch_persistent.py | 4 +- ...cluster_create_existing_stopped_cluster.py | 4 +- .../example_dataproc_cluster_deferrable.py | 4 +- .../example_dataproc_cluster_diagnose.py | 4 +- .../example_dataproc_cluster_generator.py | 4 +- .../example_dataproc_cluster_start_stop.py | 4 +- .../example_dataproc_cluster_update.py | 4 +- .../cloud/dataproc/example_dataproc_flink.py | 4 +- .../cloud/dataproc/example_dataproc_gke.py | 4 +- .../cloud/dataproc/example_dataproc_hadoop.py | 4 +- .../cloud/dataproc/example_dataproc_hive.py | 4 +- .../cloud/dataproc/example_dataproc_pig.py | 4 +- .../cloud/dataproc/example_dataproc_presto.py | 4 +- .../dataproc/example_dataproc_pyspark.py | 4 +- .../cloud/dataproc/example_dataproc_spark.py | 4 +- .../dataproc/example_dataproc_spark_async.py | 4 +- .../example_dataproc_spark_deferrable.py | 4 +- .../dataproc/example_dataproc_spark_sql.py | 4 +- .../cloud/dataproc/example_dataproc_sparkr.py | 4 +- .../cloud/dataproc/example_dataproc_trino.py | 4 +- .../dataproc/example_dataproc_workflow.py | 4 +- .../example_dataproc_workflow_deferrable.py | 4 +- .../example_dataproc_metastore.py | 4 +- .../example_dataproc_metastore_backup.py | 4 +- ...ataproc_metastore_hive_partition_sensor.py | 4 +- .../datastore/example_datastore_commit.py | 4 +- .../datastore/example_datastore_query.py | 4 +- .../datastore/example_datastore_rollback.py | 4 +- .../cloud/gcs/example_calendar_to_gcs.py | 4 +- .../google/cloud/gcs/example_firestore.py | 4 +- .../google/cloud/gcs/example_gcs_acl.py | 4 +- .../cloud/gcs/example_gcs_copy_delete.py | 4 +- .../google/cloud/gcs/example_gcs_sensor.py | 4 +- .../cloud/gcs/example_gcs_to_bigquery.py | 4 +- .../gcs/example_gcs_to_bigquery_async.py | 4 +- .../google/cloud/gcs/example_gcs_to_gcs.py | 4 +- .../google/cloud/gcs/example_gcs_to_gdrive.py | 4 +- .../google/cloud/gcs/example_gcs_to_sheets.py | 4 +- .../google/cloud/gcs/example_gcs_transform.py | 4 +- .../gcs/example_gcs_transform_timespan.py | 4 +- .../cloud/gcs/example_gcs_upload_download.py | 4 +- .../google/cloud/gcs/example_gdrive_to_gcs.py | 4 +- .../google/cloud/gcs/example_mssql_to_gcs.py | 4 +- .../google/cloud/gcs/example_mysql_to_gcs.py | 4 +- .../google/cloud/gcs/example_oracle_to_gcs.py | 4 +- .../google/cloud/gcs/example_s3_to_gcs.py | 4 +- .../google/cloud/gcs/example_sftp_to_gcs.py | 4 +- .../system/google/cloud/gcs/example_sheets.py | 4 +- .../google/cloud/gcs/example_sheets_to_gcs.py | 4 +- .../google/cloud/gcs/example_trino_to_gcs.py | 4 +- .../example_kubernetes_engine.py | 4 +- .../example_kubernetes_engine_async.py | 4 +- .../example_kubernetes_engine_job.py | 4 +- .../example_kubernetes_engine_kueue.py | 4 +- .../example_kubernetes_engine_resource.py | 4 +- .../life_sciences/example_life_sciences.py | 4 +- .../cloud/ml_engine/example_mlengine.py | 4 +- .../example_natural_language.py | 4 +- .../google/cloud/pubsub/example_pubsub.py | 4 +- .../cloud/pubsub/example_pubsub_deferrable.py | 4 +- .../google/cloud/spanner/example_spanner.py | 4 +- .../speech_to_text/example_speech_to_text.py | 4 +- .../sql_to_sheets/example_sql_to_sheets.py | 4 +- .../cloud/stackdriver/example_stackdriver.py | 4 +- ...mple_cloud_storage_transfer_service_aws.py | 4 +- ...mple_cloud_storage_transfer_service_gcp.py | 4 +- ...oud_storage_transfer_service_gcs_to_gcs.py | 4 +- .../google/cloud/tasks/example_queue.py | 4 +- .../google/cloud/tasks/example_tasks.py | 4 +- .../text_to_speech/example_text_to_speech.py | 4 +- .../cloud/transfers/example_gcs_to_sftp.py | 4 +- .../transfers/example_gdrive_to_local.py | 4 +- .../transfers/example_postgres_to_gcs.py | 4 +- .../cloud/translate/example_translate.py | 4 +- .../example_translate_speech.py | 4 +- ..._vertex_ai_auto_ml_forecasting_training.py | 4 +- ...xample_vertex_ai_auto_ml_image_training.py | 4 +- ...example_vertex_ai_auto_ml_list_training.py | 4 +- ...mple_vertex_ai_auto_ml_tabular_training.py | 4 +- ...xample_vertex_ai_auto_ml_video_training.py | 4 +- .../example_vertex_ai_batch_prediction_job.py | 4 +- .../example_vertex_ai_custom_container.py | 4 +- .../vertex_ai/example_vertex_ai_custom_job.py | 4 +- ...ple_vertex_ai_custom_job_python_package.py | 4 +- .../vertex_ai/example_vertex_ai_dataset.py | 4 +- .../vertex_ai/example_vertex_ai_endpoint.py | 4 +- .../example_vertex_ai_generative_model.py | 4 +- ...ample_vertex_ai_generative_model_tuning.py | 4 +- ...ple_vertex_ai_hyperparameter_tuning_job.py | 4 +- .../example_vertex_ai_list_custom_jobs.py | 4 +- .../example_vertex_ai_model_service.py | 4 +- .../example_vertex_ai_pipeline_job.py | 4 +- .../example_video_intelligence.py | 4 +- .../vision/example_vision_annotate_image.py | 4 +- .../vision/example_vision_autogenerated.py | 4 +- .../cloud/vision/example_vision_explicit.py | 4 +- .../cloud/workflows/example_workflows.py | 4 +- .../example_datacatalog_entries.py | 4 +- .../example_datacatalog_search_catalog.py | 4 +- .../example_datacatalog_tag_templates.py | 4 +- .../datacatalog/example_datacatalog_tags.py | 4 +- .../system/google/leveldb/example_leveldb.py | 4 +- .../example_analytics_admin.py | 4 +- .../example_campaign_manager.py | 4 +- .../marketing_platform/example_search_ads.py | 2 +- .../google/suite/example_local_to_drive.py | 4 +- providers/tests/system/http/example_http.py | 2 +- .../tests/system/influxdb/example_influxdb.py | 4 +- .../system/influxdb/example_influxdb_query.py | 2 +- .../tests/system/jdbc/example_jdbc_queries.py | 4 +- .../jenkins/example_jenkins_job_trigger.py | 2 +- .../azure/example_adf_run_pipeline.py | 4 +- .../microsoft/azure/example_adls_create.py | 4 +- .../microsoft/azure/example_adls_delete.py | 4 +- .../microsoft/azure/example_adls_list.py | 4 +- .../azure/example_azure_batch_operator.py | 2 +- .../example_azure_container_instances.py | 2 +- .../microsoft/azure/example_azure_cosmosdb.py | 4 +- .../azure/example_azure_service_bus.py | 4 +- .../microsoft/azure/example_azure_synapse.py | 2 +- .../microsoft/azure/example_fileshare.py | 4 +- .../microsoft/azure/example_local_to_adls.py | 4 +- .../microsoft/azure/example_local_to_wasb.py | 4 +- .../microsoft/azure/example_msfabric.py | 4 +- .../system/microsoft/azure/example_msgraph.py | 4 +- .../system/microsoft/azure/example_powerbi.py | 4 +- .../azure/example_powerbi_dataset_refresh.py | 4 +- .../microsoft/azure/example_s3_to_wasb.py | 4 +- .../microsoft/azure/example_sftp_to_wasb.py | 4 +- .../azure/example_synapse_run_pipeline.py | 4 +- .../microsoft/azure/example_wasb_sensors.py | 2 +- .../system/microsoft/mssql/example_mssql.py | 4 +- .../system/microsoft/winrm/example_winrm.py | 4 +- providers/tests/system/mysql/example_mysql.py | 2 +- providers/tests/system/neo4j/example_neo4j.py | 2 +- .../tests/system/openai/example_openai.py | 2 +- .../openai/example_trigger_batch_operator.py | 2 +- .../system/opensearch/example_opensearch.py | 4 +- .../system/opsgenie/example_opsgenie_alert.py | 2 +- .../opsgenie/example_opsgenie_notifier.py | 2 +- .../system/papermill/example_papermill.py | 2 +- .../example_papermill_remote_verify.py | 2 +- .../papermill/example_papermill_verify.py | 2 +- .../tests/system/pgvector/example_pgvector.py | 2 +- .../pgvector/example_pgvector_openai.py | 2 +- .../pinecone/example_create_pod_index.py | 2 +- .../example_create_serverless_index.py | 2 +- .../system/pinecone/example_dag_pinecone.py | 2 +- .../pinecone/example_pinecone_cohere.py | 2 +- .../pinecone/example_pinecone_openai.py | 2 +- .../tests/system/postgres/example_postgres.py | 4 +- .../system/presto/example_gcs_to_presto.py | 2 +- .../tests/system/qdrant/example_dag_qdrant.py | 2 +- .../system/redis/example_redis_publish.py | 4 +- .../tests/system/salesforce/example_bulk.py | 2 +- .../example_salesforce_apex_rest.py | 2 +- .../system/samba/example_gcs_to_samba.py | 4 +- .../tests/system/sftp/example_sftp_sensor.py | 4 +- .../system/singularity/example_singularity.py | 2 +- providers/tests/system/slack/example_slack.py | 2 +- .../system/slack/example_slack_webhook.py | 2 +- .../system/slack/example_sql_to_slack.py | 2 +- .../slack/example_sql_to_slack_webhook.py | 2 +- .../snowflake/example_copy_into_snowflake.py | 2 +- .../system/snowflake/example_snowflake.py | 2 +- .../snowflake/example_snowpark_decorator.py | 2 +- .../snowflake/example_snowpark_operator.py | 2 +- .../tests/system/sqlite/example_sqlite.py | 4 +- .../tests/system/tableau/example_tableau.py | 2 +- .../tests/system/telegram/example_telegram.py | 2 +- ...example_azure_blob_to_teradata_transfer.py | 4 +- .../example_s3_to_teradata_transfer.py | 4 +- .../system/teradata/example_ssl_teradata.py | 4 +- .../tests/system/teradata/example_teradata.py | 4 +- .../teradata/example_teradata_call_sp.py | 4 +- .../example_teradata_compute_cluster.py | 4 +- .../example_teradata_to_teradata_transfer.py | 4 +- .../system/trino/example_gcs_to_trino.py | 2 +- providers/tests/system/trino/example_trino.py | 2 +- .../weaviate/example_weaviate_cohere.py | 2 +- .../example_weaviate_dynamic_mapping_dag.py | 2 +- .../weaviate/example_weaviate_openai.py | 2 +- .../weaviate/example_weaviate_operator.py | 2 +- .../weaviate/example_weaviate_using_hook.py | 2 +- .../example_weaviate_vectorizer_dag.py | 2 +- ...example_weaviate_without_vectorizer_dag.py | 2 +- .../system/yandex/example_yandexcloud.py | 7 ++- .../yandex/example_yandexcloud_dataproc.py | 8 +-- ...xample_yandexcloud_dataproc_lightweight.py | 8 +-- .../system/yandex/example_yandexcloud_yq.py | 8 +-- providers/tests/system/ydb/example_ydb.py | 4 +- .../zendesk/example_zendesk_custom_get.py | 2 +- providers/tests/yandex/hooks/test_yandex.py | 4 +- providers/tests/yandex/links/test_yq.py | 5 +- providers/tests/yandex/operators/test_yq.py | 3 +- pyproject.toml | 6 +-- scripts/ci/docker-compose/local.yml | 3 ++ scripts/ci/pre_commit/check_system_tests.py | 8 +-- scripts/in_container/install_devel_deps.py | 2 +- task_sdk/tests/conftest.py | 2 +- tests/always/test_connection.py | 3 +- tests/always/test_example_dags.py | 3 +- tests/always/test_secrets.py | 5 +- tests/always/test_secrets_backends.py | 3 +- tests/always/test_secrets_local_filesystem.py | 3 +- tests/api_connexion/conftest.py | 7 ++- .../endpoints/test_asset_endpoint.py | 11 ++-- .../endpoints/test_backfill_endpoint.py | 5 +- .../endpoints/test_config_endpoint.py | 4 +- .../endpoints/test_connection_endpoint.py | 9 ++-- .../endpoints/test_dag_endpoint.py | 9 ++-- .../endpoints/test_dag_parsing.py | 5 +- .../endpoints/test_dag_run_endpoint.py | 11 ++-- .../endpoints/test_dag_source_endpoint.py | 5 +- .../endpoints/test_dag_stats_endpoint.py | 5 +- .../endpoints/test_dag_warning_endpoint.py | 5 +- .../endpoints/test_event_log_endpoint.py | 7 ++- .../endpoints/test_extra_link_endpoint.py | 11 ++-- .../endpoints/test_import_error_endpoint.py | 9 ++-- .../endpoints/test_log_endpoint.py | 5 +- .../test_mapped_task_instance_endpoint.py | 7 ++- .../endpoints/test_plugin_endpoint.py | 9 ++-- .../endpoints/test_pool_endpoint.py | 9 ++-- .../endpoints/test_provider_endpoint.py | 3 +- .../endpoints/test_task_endpoint.py | 5 +- .../endpoints/test_task_instance_endpoint.py | 7 ++- .../endpoints/test_variable_endpoint.py | 9 ++-- .../endpoints/test_xcom_endpoint.py | 7 ++- .../schemas/test_connection_schema.py | 3 +- .../schemas/test_dag_run_schema.py | 5 +- .../schemas/test_dataset_schema.py | 3 +- .../schemas/test_error_schema.py | 5 +- .../schemas/test_plugin_schema.py | 3 +- .../schemas/test_pool_schemas.py | 3 +- .../api_connexion/schemas/test_xcom_schema.py | 3 +- tests/api_connexion/test_auth.py | 7 ++- tests/api_connexion/test_parameters.py | 3 +- tests/api_connexion/test_security.py | 2 +- .../views/public/test_connections.py | 3 +- .../api_fastapi/views/public/test_dag_run.py | 3 +- tests/api_fastapi/views/public/test_dags.py | 3 +- .../views/public/test_variables.py | 3 +- tests/api_fastapi/views/ui/test_assets.py | 3 +- tests/api_fastapi/views/ui/test_dashboard.py | 3 +- .../endpoints/test_rpc_api_endpoint.py | 5 +- tests/api_internal/test_internal_api_call.py | 3 +- tests/assets/test_manager.py | 3 +- tests/assets/tests_asset.py | 2 +- tests/auth/managers/simple/views/test_auth.py | 3 +- tests/cli/commands/test_backfill_command.py | 3 +- tests/cli/commands/test_celery_command.py | 3 +- tests/cli/commands/test_config_command.py | 3 +- tests/cli/commands/test_connection_command.py | 3 +- tests/cli/commands/test_dag_command.py | 6 +-- .../commands/test_dag_processor_command.py | 3 +- .../cli/commands/test_fastapi_api_command.py | 1 + tests/cli/commands/test_info_command.py | 3 +- .../cli/commands/test_internal_api_command.py | 4 +- tests/cli/commands/test_jobs_command.py | 3 +- tests/cli/commands/test_kerberos_command.py | 3 +- tests/cli/commands/test_kubernetes_command.py | 3 +- tests/cli/commands/test_plugins_command.py | 16 +++--- .../test_rotate_fernet_key_command.py | 5 +- tests/cli/commands/test_scheduler_command.py | 3 +- tests/cli/commands/test_task_command.py | 7 ++- tests/cli/commands/test_variable_command.py | 3 +- tests/cli/commands/test_webserver_command.py | 4 +- tests/cli/conftest.py | 3 +- tests/cli/test_cli_parser.py | 3 +- tests/conftest.py | 4 +- tests/core/test_configuration.py | 6 +-- tests/core/test_core.py | 3 +- tests/core/test_example_dags_system.py | 5 +- tests/core/test_impersonation_tests.py | 3 +- tests/core/test_logging_config.py | 3 +- tests/core/test_otel_tracer.py | 3 +- tests/core/test_sentry.py | 3 +- tests/core/test_settings.py | 3 +- tests/core/test_sqlalchemy_config.py | 3 +- tests/core/test_stats.py | 3 +- tests/dag_processing/test_job_runner.py | 12 ++--- tests/dag_processing/test_processor.py | 11 ++-- ...st_external_task_sensor_check_existense.py | 1 + tests/dags/test_miscellaneous.py | 3 +- tests/dags/test_sensor.py | 3 +- tests/decorators/test_bash.py | 3 +- tests/decorators/test_mapped.py | 1 + tests/decorators/test_python.py | 4 +- tests/executors/test_executor_loader.py | 3 +- .../cli/commands/test_celery_command.py | 3 +- .../executors/test_celery_executor.py | 3 +- tests/integration/security/test_kerberos.py | 3 +- tests/jobs/test_base_job.py | 4 +- tests/jobs/test_local_task_job.py | 11 ++-- tests/jobs/test_scheduler_job.py | 22 ++++---- tests/jobs/test_triggerer_job.py | 4 +- tests/jobs/test_triggerer_job_logging.py | 3 +- tests/lineage/test_hook.py | 3 +- tests/lineage/test_lineage.py | 3 +- tests/listeners/class_listener.py | 3 +- tests/listeners/test_asset_listener.py | 1 + .../test_dag_import_error_listener.py | 10 ++-- tests/listeners/test_listeners.py | 1 + tests/models/test_backfill.py | 3 +- tests/models/test_base.py | 3 +- tests/models/test_baseoperator.py | 4 +- tests/models/test_cleartasks.py | 6 +-- tests/models/test_dag.py | 26 +++++----- tests/models/test_dagbag.py | 8 +-- tests/models/test_dagcode.py | 3 +- tests/models/test_dagrun.py | 10 ++-- tests/models/test_dagwarning.py | 3 +- tests/models/test_mappedoperator.py | 8 +-- tests/models/test_param.py | 3 +- tests/models/test_pool.py | 3 +- tests/models/test_renderedtifields.py | 5 +- tests/models/test_serialized_dag.py | 5 +- tests/models/test_skipmixin.py | 3 +- tests/models/test_taskinstance.py | 12 ++--- tests/models/test_timestamp.py | 3 +- tests/models/test_trigger.py | 3 +- tests/models/test_variable.py | 5 +- tests/models/test_xcom.py | 5 +- tests/models/test_xcom_arg.py | 3 +- tests/operators/test_branch_operator.py | 3 +- tests/operators/test_email.py | 3 +- tests/operators/test_latest_only_operator.py | 5 +- tests/operators/test_python.py | 7 ++- tests/plugins/test_plugin.py | 8 +-- tests/plugins/test_plugins_manager.py | 5 +- tests/secrets/test_cache.py | 3 +- tests/security/test_kerberos.py | 3 +- tests/sensors/test_base.py | 3 +- tests/sensors/test_external_task_sensor.py | 8 +-- tests/sensors/test_python.py | 1 + tests/serialization/test_dag_serialization.py | 37 +++++++------- tests/serialization/test_pydantic_models.py | 4 +- tests/serialization/test_serde.py | 3 +- .../serialization/test_serialized_objects.py | 4 +- .../example_external_task_child_deferrable.py | 2 +- ...example_external_task_parent_deferrable.py | 4 +- tests/system/example_empty.py | 4 +- .../task_runner/test_standard_task_runner.py | 6 +-- .../deps/test_pool_slots_available_dep.py | 3 +- tests/ti_deps/deps/test_prev_dagrun_dep.py | 5 +- .../deps/test_ready_to_reschedule_dep.py | 3 +- tests/utils/log/test_log_reader.py | 5 +- tests/utils/log/test_secrets_masker.py | 3 +- tests/utils/log/test_task_context_logger.py | 5 +- tests/utils/test_dag_cycle.py | 1 + tests/utils/test_db.py | 3 +- tests/utils/test_db_cleanup.py | 3 +- tests/utils/test_db_manager.py | 3 +- tests/utils/test_dot_renderer.py | 5 +- tests/utils/test_email.py | 3 +- tests/utils/test_file.py | 4 +- tests/utils/test_helpers.py | 5 +- tests/utils/test_log_handlers.py | 5 +- tests/utils/test_net.py | 3 +- tests/utils/test_serve_logs.py | 3 +- tests/utils/test_sqlalchemy.py | 3 +- tests/utils/test_state.py | 4 +- tests/utils/test_task_group.py | 6 +-- ...test_task_handler_with_custom_formatter.py | 7 ++- tests/utils/test_types.py | 4 +- tests/www/test_app.py | 5 +- tests/www/test_utils.py | 3 +- tests/www/views/conftest.py | 9 ++-- .../www/views/test_anonymous_as_admin_role.py | 3 +- tests/www/views/test_session.py | 5 +- tests/www/views/test_views.py | 15 +++--- tests/www/views/test_views_acl.py | 10 ++-- tests/www/views/test_views_base.py | 7 ++- .../www/views/test_views_cluster_activity.py | 3 +- tests/www/views/test_views_configuration.py | 5 +- tests/www/views/test_views_connection.py | 3 +- .../www/views/test_views_custom_user_views.py | 4 +- tests/www/views/test_views_dagrun.py | 8 +-- tests/www/views/test_views_dataset.py | 5 +- tests/www/views/test_views_decorators.py | 7 ++- tests/www/views/test_views_extra_links.py | 15 +++--- tests/www/views/test_views_grid.py | 7 ++- tests/www/views/test_views_home.py | 8 +-- tests/www/views/test_views_log.py | 11 ++-- tests/www/views/test_views_mount.py | 3 +- tests/www/views/test_views_paused.py | 3 +- tests/www/views/test_views_pool.py | 3 +- tests/www/views/test_views_rate_limit.py | 7 ++- tests/www/views/test_views_rendered.py | 7 ++- tests/www/views/test_views_robots.py | 2 +- tests/www/views/test_views_task_norun.py | 2 +- tests/www/views/test_views_tasks.py | 10 ++-- tests/www/views/test_views_trigger_dag.py | 4 +- tests/www/views/test_views_variable.py | 4 +- .../tests_common => tests_common}/__init__.py | 0 .../_internals/__init__.py | 0 .../_internals/capture_warnings.py | 4 +- .../_internals/forbidden_warnings.py | 2 +- .../pyproject.toml | 2 +- .../pytest_plugin.py | 51 ++++++++++--------- .../test_utils/README.md | 0 .../test_utils/__init__.py | 2 +- .../test_utils/api_connexion_utils.py | 12 ++--- .../test_utils/asserts.py | 9 +++- .../test_utils/azure_system_helpers.py | 16 +++--- .../test_utils/compat.py | 2 +- .../test_utils/config.py | 0 .../test_utils/db.py | 8 ++- .../test_utils/decorators.py | 0 .../test_utils/fake_datetime.py | 4 +- .../test_utils/gcp_system_helpers.py | 17 +++---- .../test_utils/get_all_tests.py | 8 ++- .../test_utils/hdfs_utils.py | 2 + .../test_utils/logging_command_executor.py | 6 ++- .../test_utils/mapping.py | 0 .../test_utils/mock_executor.py | 6 +-- .../test_utils/mock_operators.py | 51 ++++++++++--------- .../test_utils/mock_plugins.py | 0 .../test_utils/mock_security_manager.py | 4 +- .../test_utils/operators/__init__.py | 0 .../operators/postgres_local_executor.cfg | 0 .../test_utils/perf/__init__.py | 0 .../test_utils/perf/perf_kit/__init__.py | 5 +- .../test_utils/perf/perf_kit/memory.py | 4 +- .../test_utils/perf/perf_kit/python.py | 15 +++--- .../perf/perf_kit/repeat_and_time.py | 12 ++--- .../test_utils/perf/perf_kit/sqlalchemy.py | 15 +++--- .../test_utils/permissions.py | 2 + .../test_utils/providers.py | 4 +- .../remote_user_api_auth_backend.py | 2 +- .../test_utils/reset_warning_registry.py | 4 +- .../test_utils/salesforce_system_helpers.py | 0 .../test_utils/sftp_system_helpers.py | 2 +- .../test_utils/system_tests.py | 0 .../test_utils/system_tests_class.py | 13 ++--- .../test_utils/terraform.py | 4 +- .../test_utils/timetables.py | 2 + .../test_utils/watcher.py | 8 ++- .../test_utils/www.py | 0 768 files changed, 1560 insertions(+), 1752 deletions(-) rename {dev/tests_common => tests_common}/__init__.py (100%) rename {dev/tests_common => tests_common}/_internals/__init__.py (100%) rename {dev/tests_common => tests_common}/_internals/capture_warnings.py (99%) rename {dev/tests_common => tests_common}/_internals/forbidden_warnings.py (99%) rename {dev/tests_common => tests_common}/pyproject.toml (98%) rename {dev/tests_common => tests_common}/pytest_plugin.py (97%) rename {dev/tests_common => tests_common}/test_utils/README.md (100%) rename {dev/tests_common => tests_common}/test_utils/__init__.py (94%) rename {dev/tests_common => tests_common}/test_utils/api_connexion_utils.py (91%) rename {dev/tests_common => tests_common}/test_utils/asserts.py (95%) rename {dev/tests_common => tests_common}/test_utils/azure_system_helpers.py (93%) rename {dev/tests_common => tests_common}/test_utils/compat.py (99%) rename {dev/tests_common => tests_common}/test_utils/config.py (100%) rename {dev/tests_common => tests_common}/test_utils/db.py (97%) rename {dev/tests_common => tests_common}/test_utils/decorators.py (100%) rename {dev/tests_common => tests_common}/test_utils/fake_datetime.py (92%) rename {dev/tests_common => tests_common}/test_utils/gcp_system_helpers.py (95%) rename {dev/tests_common => tests_common}/test_utils/get_all_tests.py (92%) rename {dev/tests_common => tests_common}/test_utils/hdfs_utils.py (95%) rename {dev/tests_common => tests_common}/test_utils/logging_command_executor.py (97%) rename {dev/tests_common => tests_common}/test_utils/mapping.py (100%) rename {dev/tests_common => tests_common}/test_utils/mock_executor.py (97%) rename {dev/tests_common => tests_common}/test_utils/mock_operators.py (84%) rename {dev/tests_common => tests_common}/test_utils/mock_plugins.py (100%) rename {dev/tests_common => tests_common}/test_utils/mock_security_manager.py (89%) rename {dev/tests_common => tests_common}/test_utils/operators/__init__.py (100%) rename {dev/tests_common => tests_common}/test_utils/operators/postgres_local_executor.cfg (100%) rename {dev/tests_common => tests_common}/test_utils/perf/__init__.py (100%) rename {dev/tests_common => tests_common}/test_utils/perf/perf_kit/__init__.py (99%) rename {dev/tests_common => tests_common}/test_utils/perf/perf_kit/memory.py (94%) rename {dev/tests_common => tests_common}/test_utils/perf/perf_kit/python.py (90%) rename {dev/tests_common => tests_common}/test_utils/perf/perf_kit/repeat_and_time.py (92%) rename {dev/tests_common => tests_common}/test_utils/perf/perf_kit/sqlalchemy.py (97%) rename {dev/tests_common => tests_common}/test_utils/permissions.py (93%) rename {dev/tests_common => tests_common}/test_utils/providers.py (94%) rename {dev/tests_common => tests_common}/test_utils/remote_user_api_auth_backend.py (97%) rename {dev/tests_common => tests_common}/test_utils/reset_warning_registry.py (96%) rename {dev/tests_common => tests_common}/test_utils/salesforce_system_helpers.py (100%) rename {dev/tests_common => tests_common}/test_utils/sftp_system_helpers.py (99%) rename {dev/tests_common => tests_common}/test_utils/system_tests.py (100%) rename {dev/tests_common => tests_common}/test_utils/system_tests_class.py (95%) rename {dev/tests_common => tests_common}/test_utils/terraform.py (93%) rename {dev/tests_common => tests_common}/test_utils/timetables.py (97%) rename {dev/tests_common => tests_common}/test_utils/watcher.py (86%) rename {dev/tests_common => tests_common}/test_utils/www.py (100%) diff --git a/.dockerignore b/.dockerignore index 2d70030fd886b..e913ed4f43c89 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,6 +42,7 @@ # Add tests and kubernetes_tests to context. !tests +!tests_common !kubernetes_tests !helm_tests !docker_tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2f42e65192bf..fe4315fafc212 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -364,7 +364,7 @@ repos: args: [] require_serial: true additional_dependencies: ["ruff==0.5.5"] - exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py|^airflow/contrib/ + exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py$ - id: replace-bad-characters name: Replace bad characters entry: ./scripts/ci/pre_commit/replace_bad_characters.py @@ -736,7 +736,7 @@ repos: name: Verify usage of Airflow deprecation classes in core entry: category=DeprecationWarning|category=PendingDeprecationWarning files: \.py$ - exclude: ^airflow/configuration\.py$|^providers/src/airflow/providers/|^scripts/in_container/verify_providers\.py$|^(providers/)?tests/.*$|^dev/tests_common/ + exclude: ^airflow/configuration\.py$|^providers/src/airflow/providers/|^scripts/in_container/verify_providers\.py$|^(providers/)?tests/.*$|^tests_common/ pass_filenames: true - id: check-provide-create-sessions-imports language: pygrep @@ -1184,7 +1184,9 @@ repos: ^(providers/)?tests/ | ^dev/.*\.py$ | ^scripts/.*\.py$ | - ^\w+_tests/ | + ^docker_tests/.*$ | + ^helm_tests/.*$ | + ^tests_common/.*$ | ^docs/.*\.py$ | ^hatch_build.py$ - id: check-provider-docs-valid diff --git a/Dockerfile.ci b/Dockerfile.ci index df1cdf8c27385..0cdf9899799f3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1369,6 +1369,7 @@ COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml COPY task_sdk/pyproject.toml ${AIRFLOW_SOURCES}/task_sdk/pyproject.toml COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/ +COPY tests_common/ ${AIRFLOW_SOURCES}/tests_common/ COPY generated/* ${AIRFLOW_SOURCES}/generated/ COPY constraints/* ${AIRFLOW_SOURCES}/constraints/ COPY LICENSE ${AIRFLOW_SOURCES}/LICENSE diff --git a/contributing-docs/testing/system_tests.rst b/contributing-docs/testing/system_tests.rst index cc64d25e90cbc..fe010f76bd5d5 100644 --- a/contributing-docs/testing/system_tests.rst +++ b/contributing-docs/testing/system_tests.rst @@ -35,7 +35,7 @@ Airflow system tests are pretty special because they serve three purposes: Old System Tests ---------------- -The system tests derive from the ``dev.tests_common.test_utils.system_test_class.SystemTests`` class. +The system tests derive from the ``tests_common.test_utils.system_test_class.SystemTests`` class. Old versions of System tests should also be marked with ``@pytest.marker.system(SYSTEM)`` where ``system`` designates the system to be tested (for example, ``google.cloud``). These tests are skipped by default. diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index 468f10b68d4c5..dc4e27f10d9fe 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -1171,7 +1171,7 @@ are not part of the public API. We deal with it in one of the following ways: 1) If the whole provider is supposed to only work for later airflow version, we remove the whole provider by excluding it from compatibility test configuration (see below) -2) Some compatibility shims are defined in ``dev.tests_common.test_utils/compat.py`` - and they can be used to make the +2) Some compatibility shims are defined in ``tests_common.test_utils/compat.py`` - and they can be used to make the tests compatible - for example importing ``ParseImportError`` after the exception has been renamed from ``ImportError`` and it would fail in Airflow 2.9, but we have a fallback import in ``compat.py`` that falls back to old import automatically, so all tests testing / expecting ``ParseImportError`` should import @@ -1184,7 +1184,7 @@ are not part of the public API. We deal with it in one of the following ways: .. code-block:: python - from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS @pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="The tests should be skipped for Airflow < 2.8") @@ -1197,7 +1197,7 @@ are not part of the public API. We deal with it in one of the following ways: .. code-block:: python - from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES + from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES @pytest.mark.skipif( diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 164a1583e1fea..2352bcda533a2 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -97,6 +97,7 @@ ("pyproject.toml", "/opt/airflow/pyproject.toml"), ("scripts", "/opt/airflow/scripts"), ("scripts/docker/entrypoint_ci.sh", "/entrypoint"), + ("tests_common", "/opt/airflow/tests_common"), ("tests", "/opt/airflow/tests"), ("helm_tests", "/opt/airflow/helm_tests"), ("kubernetes_tests", "/opt/airflow/kubernetes_tests"), diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index cf4e124ff94e3..74a856f3693c6 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -245,7 +245,7 @@ def __hash__(self): ], FileGroupForCi.TESTS_UTILS_FILES: [ r"^tests/utils/", - r"^dev/tests_common/.*\.py$", + r"^tests_common/.*\.py$", ], FileGroupForCi.TASK_SDK_FILES: [ r"^task_sdk/src/airflow/sdk/.*\.py$", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 619dc8eec2f99..ec0769c0a87fe 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -778,7 +778,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ), ( pytest.param( - ("dev/tests_common/__init__.py",), + ("tests_common/__init__.py",), { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.9']", @@ -799,7 +799,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "needs-mypy": "true", "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", }, - id="All tests should be run when dev/tests_common/ change", + id="All tests should be run when tests_common/ change", ) ), ], diff --git a/dev/perf/scheduler_dag_execution_timing.py b/dev/perf/scheduler_dag_execution_timing.py index fc5c21bda7aa7..1956c8023b5f0 100755 --- a/dev/perf/scheduler_dag_execution_timing.py +++ b/dev/perf/scheduler_dag_execution_timing.py @@ -107,7 +107,7 @@ def get_executor_under_test(dotted_path): from airflow.executors.executor_loader import ExecutorLoader if dotted_path == "MockExecutor": - from dev.tests_common.test_utils.mock_executor import MockExecutor as executor + from tests_common.test_utils.mock_executor import MockExecutor as executor else: executor = ExecutorLoader.load_executor(dotted_path) diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 8c6c8c21f9805..5a507b7ee6aae 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -24,6 +24,7 @@ import pytest from helm_tests.airflow_aux.test_container_lifecycle import CONTAINER_LIFECYCLE_PARAMETERS + from tests.charts.helm_template_generator import render_chart diff --git a/providers/tests/alibaba/cloud/log/test_oss_task_handler.py b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py index 18abe57aa09b7..ff65dd89ce4ef 100644 --- a/providers/tests/alibaba/cloud/log/test_oss_task_handler.py +++ b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py @@ -22,14 +22,13 @@ from unittest.mock import PropertyMock import pytest +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.providers.alibaba.cloud.log.oss_task_handler import OSSTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs - pytestmark = pytest.mark.db_test OSS_TASK_HANDLER_STRING = "airflow.providers.alibaba.cloud.log.oss_task_handler.{}" diff --git a/providers/tests/amazon/aws/auth_manager/avp/test_facade.py b/providers/tests/amazon/aws/auth_manager/avp/test_facade.py index 3d2a0195039df..af1149aec45fd 100644 --- a/providers/tests/amazon/aws/auth_manager/avp/test_facade.py +++ b/providers/tests/amazon/aws/auth_manager/avp/test_facade.py @@ -21,6 +21,7 @@ from unittest.mock import Mock import pytest +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities, get_action_id, get_entity_type @@ -28,8 +29,6 @@ from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser from airflow.utils.helpers import prune_dict -from dev.tests_common.test_utils.config import conf_vars - if TYPE_CHECKING: from airflow.auth.managers.base_auth_manager import ResourceMethod diff --git a/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py index 6122079fee4cb..c523beec564c7 100644 --- a/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py +++ b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py @@ -20,13 +20,12 @@ from unittest.mock import ANY, Mock, patch import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS +from tests_common.test_utils.config import conf_vars from airflow.cli import cli_parser from airflow.providers.amazon.aws.auth_manager.cli.avp_commands import init_avp, update_schema -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS -from dev.tests_common.test_utils.config import conf_vars - mock_boto3 = Mock() pytestmark = [ diff --git a/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py b/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py index b6071aac955c6..1c57bc14d83ed 100644 --- a/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py +++ b/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py @@ -20,8 +20,7 @@ import pytest from flask import Flask - -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error python3_saml = pytest.importorskip("python3-saml") diff --git a/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py index 47e8a4cbcb26d..ba93b2105a81e 100644 --- a/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py +++ b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py @@ -22,6 +22,9 @@ import pytest from flask import Flask, session from flask_appbuilder.menu import MenuItem +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities from airflow.providers.amazon.aws.auth_manager.avp.facade import AwsAuthManagerAmazonVerifiedPermissionsFacade @@ -39,10 +42,6 @@ from airflow.www import app as application from airflow.www.extensions.init_appbuilder import init_appbuilder -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.www import check_content_in_response - try: from airflow.auth.managers.models.resource_details import ( AccessView, diff --git a/providers/tests/amazon/aws/auth_manager/views/test_auth.py b/providers/tests/amazon/aws/auth_manager/views/test_auth.py index 9b2eec69188be..919eb59ead059 100644 --- a/providers/tests/amazon/aws/auth_manager/views/test_auth.py +++ b/providers/tests/amazon/aws/auth_manager/views/test_auth.py @@ -20,13 +20,12 @@ import pytest from flask import session, url_for +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.config import conf_vars - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Test requires Airflow 2.9+"), pytest.mark.skip_if_database_isolation_mode, diff --git a/providers/tests/amazon/aws/executors/batch/test_batch_executor.py b/providers/tests/amazon/aws/executors/batch/test_batch_executor.py index e7037bd16c85e..85272186b3324 100644 --- a/providers/tests/amazon/aws/executors/batch/test_batch_executor.py +++ b/providers/tests/amazon/aws/executors/batch/test_batch_executor.py @@ -26,6 +26,8 @@ import yaml from botocore.exceptions import ClientError, NoCredentialsError from semver import VersionInfo +from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.executors.base_executor import BaseExecutor @@ -46,9 +48,6 @@ from airflow.utils.state import State from airflow.version import version as airflow_version_str -from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from dev.tests_common.test_utils.config import conf_vars - airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3])) ARN1 = "arn1" diff --git a/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py b/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py index 50cdb580382f3..80b7d85a9eded 100644 --- a/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py +++ b/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py @@ -32,6 +32,9 @@ from botocore.exceptions import ClientError from inflection import camelize from semver import VersionInfo +from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.executors.base_executor import BaseExecutor @@ -57,10 +60,6 @@ from airflow.utils.timezone import utcnow from airflow.version import version as airflow_version_str -from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3])) diff --git a/providers/tests/amazon/aws/hooks/test_base_aws.py b/providers/tests/amazon/aws/hooks/test_base_aws.py index c58993d748f2d..8919203da0f01 100644 --- a/providers/tests/amazon/aws/hooks/test_base_aws.py +++ b/providers/tests/amazon/aws/hooks/test_base_aws.py @@ -38,6 +38,7 @@ from botocore.utils import FileWebIdentityTokenLoader from moto import mock_aws from moto.core import DEFAULT_ACCOUNT_ID +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.models.connection import Connection @@ -50,8 +51,6 @@ ) from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper -from dev.tests_common.test_utils.config import conf_vars - pytest.importorskip("aiobotocore") MOCK_AWS_CONN_ID = "mock-conn-id" diff --git a/providers/tests/amazon/aws/hooks/test_s3.py b/providers/tests/amazon/aws/hooks/test_s3.py index 2e9a010006bab..89d29dae02dde 100644 --- a/providers/tests/amazon/aws/hooks/test_s3.py +++ b/providers/tests/amazon/aws/hooks/test_s3.py @@ -30,6 +30,7 @@ import pytest from botocore.exceptions import ClientError from moto import mock_aws +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS from airflow.exceptions import AirflowException from airflow.models import Connection @@ -43,8 +44,6 @@ ) from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS - @pytest.fixture def mocked_s3_res(): diff --git a/providers/tests/amazon/aws/links/test_base_aws.py b/providers/tests/amazon/aws/links/test_base_aws.py index 78622870806fb..738fb6acc482c 100644 --- a/providers/tests/amazon/aws/links/test_base_aws.py +++ b/providers/tests/amazon/aws/links/test_base_aws.py @@ -21,14 +21,13 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.mock_operators import MockOperator from airflow.models.xcom import XCom from airflow.providers.amazon.aws.links.base_aws import BaseAwsLink from airflow.serialization.serialized_objects import SerializedDAG -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.mock_operators import MockOperator - if TYPE_CHECKING: from airflow.models.taskinstance import TaskInstance diff --git a/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py index c78fab89e197a..d07699d9a7049 100644 --- a/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py +++ b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py @@ -26,6 +26,7 @@ import boto3 import pytest from moto import mock_aws +from tests_common.test_utils.config import conf_vars from watchtower import CloudWatchLogHandler from airflow.models import DAG, DagRun, TaskInstance @@ -36,8 +37,6 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars - def get_time_str(time_in_milliseconds): dt_time = dt.fromtimestamp(time_in_milliseconds / 1000.0, tz=timezone.utc) diff --git a/providers/tests/amazon/aws/log/test_s3_task_handler.py b/providers/tests/amazon/aws/log/test_s3_task_handler.py index 9819cf95e9cc1..a471bf0075479 100644 --- a/providers/tests/amazon/aws/log/test_s3_task_handler.py +++ b/providers/tests/amazon/aws/log/test_s3_task_handler.py @@ -26,6 +26,7 @@ import pytest from botocore.exceptions import ClientError from moto import mock_aws +from tests_common.test_utils.config import conf_vars from airflow.models import DAG, DagRun, TaskInstance from airflow.operators.empty import EmptyOperator @@ -34,8 +35,6 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars - @pytest.fixture(autouse=True) def s3mock(): diff --git a/providers/tests/amazon/aws/secrets/test_systems_manager.py b/providers/tests/amazon/aws/secrets/test_systems_manager.py index c1b35a799d3c9..e1c035ab42047 100644 --- a/providers/tests/amazon/aws/secrets/test_systems_manager.py +++ b/providers/tests/amazon/aws/secrets/test_systems_manager.py @@ -21,12 +21,11 @@ import pytest from moto import mock_aws +from tests_common.test_utils.config import conf_vars from airflow.configuration import initialize_secrets_backends from airflow.providers.amazon.aws.secrets.systems_manager import SystemsManagerParameterStoreBackend -from dev.tests_common.test_utils.config import conf_vars - URI_CONNECTION = pytest.param( "postgres://my-login:my-pass@my-host:5432/my-schema?param1=val1¶m2=val2", id="uri-connection" ) diff --git a/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py b/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py index e27ff2d0aa56d..605705a2b663f 100644 --- a/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py @@ -22,14 +22,13 @@ import pytest from boto3.session import Session +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces from airflow.exceptions import AirflowException from airflow.models.connection import Connection from airflow.providers.amazon.aws.transfers.redshift_to_s3 import RedshiftToS3Operator from airflow.providers.amazon.aws.utils.redshift import build_credentials_block -from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces - class TestRedshiftToS3Transfer: @pytest.mark.parametrize("table_as_file_name, expected_s3_key", [[True, "key/table_"], [False, "key"]]) diff --git a/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py b/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py index 6e300791e0935..c9d9ca337d111 100644 --- a/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py +++ b/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py @@ -22,6 +22,7 @@ import pytest from boto3.session import Session +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces from airflow.exceptions import AirflowException from airflow.models.connection import Connection @@ -34,8 +35,6 @@ SchemaDatasetFacetFields, ) -from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces - class TestS3ToRedshiftTransfer: @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_connection") diff --git a/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py b/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py index 545398d9666f8..bcab8b2909af3 100644 --- a/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py +++ b/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py @@ -20,6 +20,7 @@ import boto3 import pytest from moto import mock_aws +from tests_common.test_utils.config import conf_vars from airflow.models import DAG from airflow.providers.amazon.aws.hooks.s3 import S3Hook @@ -28,8 +29,6 @@ from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py b/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py index e4afe5c8c1efb..980c8a19daa55 100644 --- a/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py @@ -20,6 +20,7 @@ import boto3 import pytest from moto import mock_aws +from tests_common.test_utils.config import conf_vars from airflow.models import DAG from airflow.providers.amazon.aws.hooks.s3 import S3Hook @@ -28,8 +29,6 @@ from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test BUCKET = "test-bucket" diff --git a/providers/tests/apache/hive/hooks/test_hive.py b/providers/tests/apache/hive/hooks/test_hive.py index aee09db28088e..d975ce50fb692 100644 --- a/providers/tests/apache/hive/hooks/test_hive.py +++ b/providers/tests/apache/hive/hooks/test_hive.py @@ -25,6 +25,7 @@ import pandas as pd import pytest from hmsclient import HMSClient +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces from airflow.exceptions import AirflowException from airflow.models.connection import Connection @@ -34,7 +35,6 @@ from airflow.utils import timezone from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING -from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces from providers.tests.apache.hive import ( BaseMockConnectionCursor, InvalidHiveCliHook, diff --git a/providers/tests/apache/livy/hooks/test_livy.py b/providers/tests/apache/livy/hooks/test_livy.py index 0fca347b06a99..d2968591f90f4 100644 --- a/providers/tests/apache/livy/hooks/test_livy.py +++ b/providers/tests/apache/livy/hooks/test_livy.py @@ -25,14 +25,13 @@ import requests from aiohttp import ClientResponseError, RequestInfo from requests.exceptions import RequestException +from tests_common.test_utils.db import clear_db_connections from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.apache.livy.hooks.livy import BatchState, LivyAsyncHook, LivyHook from airflow.utils import db -from dev.tests_common.test_utils.db import clear_db_connections - pytestmark = pytest.mark.skip_if_database_isolation_mode LIVY_CONN_ID = LivyHook.default_conn_name diff --git a/providers/tests/apache/spark/hooks/test_spark_sql.py b/providers/tests/apache/spark/hooks/test_spark_sql.py index 10f63cce99326..179e75ffc7470 100644 --- a/providers/tests/apache/spark/hooks/test_spark_sql.py +++ b/providers/tests/apache/spark/hooks/test_spark_sql.py @@ -22,14 +22,13 @@ from unittest.mock import call, patch import pytest +from tests_common.test_utils.db import clear_db_connections from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.apache.spark.hooks.spark_sql import SparkSqlHook from airflow.utils import db -from dev.tests_common.test_utils.db import clear_db_connections - pytestmark = pytest.mark.db_test diff --git a/providers/tests/atlassian/jira/hooks/test_jira.py b/providers/tests/atlassian/jira/hooks/test_jira.py index e2cf9389471e7..fbb7913d128f0 100644 --- a/providers/tests/atlassian/jira/hooks/test_jira.py +++ b/providers/tests/atlassian/jira/hooks/test_jira.py @@ -20,13 +20,12 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import connection_as_json from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.atlassian.jira.hooks.jira import JiraHook -from dev.tests_common.test_utils.compat import connection_as_json - @pytest.fixture def mocked_jira_client(): diff --git a/providers/tests/atlassian/jira/operators/test_jira.py b/providers/tests/atlassian/jira/operators/test_jira.py index 614c1a4939a90..67c63a60d61d4 100644 --- a/providers/tests/atlassian/jira/operators/test_jira.py +++ b/providers/tests/atlassian/jira/operators/test_jira.py @@ -20,13 +20,12 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import connection_as_json from airflow.models import Connection from airflow.providers.atlassian.jira.operators.jira import JiraOperator from airflow.utils import timezone -from dev.tests_common.test_utils.compat import connection_as_json - DEFAULT_DATE = timezone.datetime(2017, 1, 1) MINIMAL_TEST_TICKET = { "id": "911539", diff --git a/providers/tests/atlassian/jira/sensors/test_jira.py b/providers/tests/atlassian/jira/sensors/test_jira.py index 9cb2572969ad4..ca027b7b5989d 100644 --- a/providers/tests/atlassian/jira/sensors/test_jira.py +++ b/providers/tests/atlassian/jira/sensors/test_jira.py @@ -20,13 +20,12 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import connection_as_json from airflow.models import Connection from airflow.providers.atlassian.jira.sensors.jira import JiraTicketSensor from airflow.utils import timezone -from dev.tests_common.test_utils.compat import connection_as_json - DEFAULT_DATE = timezone.datetime(2017, 1, 1) MINIMAL_TEST_TICKET = { "id": "911539", diff --git a/providers/tests/celery/cli/test_celery_command.py b/providers/tests/celery/cli/test_celery_command.py index 03f5d60dabe40..0d0310f7452ea 100644 --- a/providers/tests/celery/cli/test_celery_command.py +++ b/providers/tests/celery/cli/test_celery_command.py @@ -24,6 +24,8 @@ import pytest import sqlalchemy +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars import airflow from airflow.cli import cli_parser @@ -31,9 +33,6 @@ from airflow.executors import executor_loader from airflow.providers.celery.cli import celery_command -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/celery/executors/test_celery_executor.py b/providers/tests/celery/executors/test_celery_executor.py index 261a11f837e42..f72ee35b0f50f 100644 --- a/providers/tests/celery/executors/test_celery_executor.py +++ b/providers/tests/celery/executors/test_celery_executor.py @@ -32,6 +32,9 @@ from celery import Celery from celery.result import AsyncResult from kombu.asynchronous import set_event_loop +from tests_common.test_utils import db +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars from airflow.configuration import conf from airflow.models.baseoperator import BaseOperator @@ -42,10 +45,6 @@ from airflow.utils import timezone from airflow.utils.state import State -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/celery/log_handlers/test_log_handlers.py b/providers/tests/celery/log_handlers/test_log_handlers.py index 9eb9e33e2ae0d..5d7d4d473d2fc 100644 --- a/providers/tests/celery/log_handlers/test_log_handlers.py +++ b/providers/tests/celery/log_handlers/test_log_handlers.py @@ -23,6 +23,8 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.executors import executor_loader @@ -36,9 +38,6 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars - if AIRFLOW_V_3_0_PLUS: pass diff --git a/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py b/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py index 62019a8d2fe90..4d277c3b51e2d 100644 --- a/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py +++ b/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py @@ -24,13 +24,12 @@ import kubernetes import pytest from dateutil.parser import parse +from tests_common.test_utils.config import conf_vars from airflow.cli import cli_parser from airflow.executors import executor_loader from airflow.providers.cncf.kubernetes.cli import kubernetes_command -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py index 07b42ee3fc22a..85077ba1776bd 100644 --- a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -26,6 +26,8 @@ import yaml from kubernetes.client import models as k8s from kubernetes.client.rest import ApiException +from tests_common.test_utils.compat import BashOperator +from tests_common.test_utils.config import conf_vars from urllib3 import HTTPResponse from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning @@ -55,9 +57,6 @@ from airflow.utils import timezone from airflow.utils.state import State, TaskInstanceState -from dev.tests_common.test_utils.compat import BashOperator -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py b/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py index a6847e9c2fe56..d90ed0504401a 100644 --- a/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py +++ b/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py @@ -29,6 +29,8 @@ from kubernetes.client.rest import ApiException from kubernetes.config import ConfigException from sqlalchemy.orm import make_transient +from tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.providers import get_provider_min_airflow_version from airflow.exceptions import AirflowException, AirflowNotFoundException from airflow.hooks.base import BaseHook @@ -37,9 +39,6 @@ from airflow.utils import db from airflow.utils.db import merge_conn -from dev.tests_common.test_utils.db import clear_db_connections -from dev.tests_common.test_utils.providers import get_provider_min_airflow_version - pytestmark = pytest.mark.db_test KUBE_CONFIG_PATH = os.getenv("KUBECONFIG", "~/.kube/config") diff --git a/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py index b02f16ac92786..7ef025d7f29d4 100644 --- a/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py +++ b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py @@ -26,6 +26,8 @@ import pytest from kubernetes.client import models as k8s +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.executors import executor_loader @@ -42,9 +44,6 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/cncf/kubernetes/operators/test_pod.py b/providers/tests/cncf/kubernetes/operators/test_pod.py index ac50641370fed..a272a5c4603f6 100644 --- a/providers/tests/cncf/kubernetes/operators/test_pod.py +++ b/providers/tests/cncf/kubernetes/operators/test_pod.py @@ -27,6 +27,7 @@ import pytest from kubernetes.client import ApiClient, V1Pod, V1PodSecurityContext, V1PodStatus, models as k8s from kubernetes.client.exceptions import ApiException +from tests_common.test_utils import db from urllib3 import HTTPResponse from airflow.exceptions import ( @@ -51,8 +52,6 @@ from airflow.utils.session import create_session from airflow.utils.types import DagRunType -from dev.tests_common.test_utils import db - pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py index 18bce53688d4f..ea15480ba1212 100644 --- a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py +++ b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py @@ -27,6 +27,7 @@ import pytest import yaml from kubernetes.client import models as k8s +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow import DAG from airflow.models import Connection, DagRun, TaskInstance @@ -34,8 +35,6 @@ from airflow.utils import db, timezone from airflow.utils.types import DagRunType -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - @patch("airflow.providers.cncf.kubernetes.operators.spark_kubernetes.KubernetesHook") def test_spark_kubernetes_operator(mock_kubernetes_hook, data_file): diff --git a/providers/tests/cncf/kubernetes/test_client.py b/providers/tests/cncf/kubernetes/test_client.py index 269717ffaa0f0..ce3a961eef44a 100644 --- a/providers/tests/cncf/kubernetes/test_client.py +++ b/providers/tests/cncf/kubernetes/test_client.py @@ -21,6 +21,7 @@ import pytest from kubernetes.client import Configuration +from tests_common.test_utils.config import conf_vars from urllib3.connection import HTTPConnection, HTTPSConnection from airflow.providers.cncf.kubernetes.kube_client import ( @@ -29,8 +30,6 @@ get_kube_client, ) -from dev.tests_common.test_utils.config import conf_vars - class TestClient: @mock.patch("airflow.providers.cncf.kubernetes.kube_client.config") diff --git a/providers/tests/cncf/kubernetes/test_template_rendering.py b/providers/tests/cncf/kubernetes/test_template_rendering.py index 180b7a1e2d2ef..bf67059ed11ff 100644 --- a/providers/tests/cncf/kubernetes/test_template_rendering.py +++ b/providers/tests/cncf/kubernetes/test_template_rendering.py @@ -21,6 +21,7 @@ import pytest from sqlalchemy.orm import make_transient +from tests_common.test_utils.compat import BashOperator from airflow.models.renderedtifields import RenderedTaskInstanceFields, RenderedTaskInstanceFields as RTIF from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec, render_k8s_pod_yaml @@ -28,8 +29,6 @@ from airflow.utils.session import create_session from airflow.version import version -from dev.tests_common.test_utils.compat import BashOperator - pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] DEFAULT_DATE = timezone.datetime(2021, 9, 9) diff --git a/providers/tests/common/io/operators/test_file_transfer.py b/providers/tests/common/io/operators/test_file_transfer.py index 55a196fa8918c..4ce23de04534c 100644 --- a/providers/tests/common/io/operators/test_file_transfer.py +++ b/providers/tests/common/io/operators/test_file_transfer.py @@ -19,9 +19,9 @@ from unittest import mock -from airflow.providers.common.compat.openlineage.facet import Dataset +from tests_common.test_utils.compat import ignore_provider_compatibility_error -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from airflow.providers.common.compat.openlineage.facet import Dataset with ignore_provider_compatibility_error("2.8.0", __file__): from airflow.providers.common.io.operators.file_transfer import FileTransferOperator diff --git a/providers/tests/common/io/xcom/test_backend.py b/providers/tests/common/io/xcom/test_backend.py index ae45a2f863b78..a60fd872bc851 100644 --- a/providers/tests/common/io/xcom/test_backend.py +++ b/providers/tests/common/io/xcom/test_backend.py @@ -18,9 +18,8 @@ from __future__ import annotations import pytest - -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS, ignore_provider_compatibility_error -from dev.tests_common.test_utils.db import is_db_isolation_mode +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS, ignore_provider_compatibility_error +from tests_common.test_utils.db import is_db_isolation_mode pytestmark = [ pytest.mark.db_test, @@ -35,12 +34,12 @@ with ignore_provider_compatibility_error("2.8.0", __file__): from airflow.providers.common.io.xcom.backend import XComObjectStorageBackend +from tests_common.test_utils import db +from tests_common.test_utils.config import conf_vars + from airflow.utils import timezone from airflow.utils.xcom import XCOM_RETURN_KEY -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.config import conf_vars - @pytest.fixture(autouse=True) def reset_db(): diff --git a/providers/tests/common/sql/hooks/test_dbapi.py b/providers/tests/common/sql/hooks/test_dbapi.py index 5017a5f347c82..0b4d7dc72e4b1 100644 --- a/providers/tests/common/sql/hooks/test_dbapi.py +++ b/providers/tests/common/sql/hooks/test_dbapi.py @@ -24,14 +24,13 @@ import pytest from pyodbc import Cursor +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.hooks.base import BaseHook from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler, fetch_one_handler -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/common/sql/hooks/test_sql.py b/providers/tests/common/sql/hooks/test_sql.py index 87673f6de62e0..9908a512f8599 100644 --- a/providers/tests/common/sql/hooks/test_sql.py +++ b/providers/tests/common/sql/hooks/test_sql.py @@ -24,6 +24,7 @@ from unittest.mock import MagicMock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.exceptions import AirflowProviderDeprecationWarning @@ -31,7 +32,6 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler from airflow.utils.session import provide_session -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from providers.tests.common.sql.test_utils import mock_hook pytestmark = [ diff --git a/providers/tests/common/sql/hooks/test_sqlparse.py b/providers/tests/common/sql/hooks/test_sqlparse.py index b3622bd8eff21..a32f5ee20880b 100644 --- a/providers/tests/common/sql/hooks/test_sqlparse.py +++ b/providers/tests/common/sql/hooks/test_sqlparse.py @@ -17,11 +17,10 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.providers.common.sql.hooks.sql import DbApiHook -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/common/sql/operators/test_sql.py b/providers/tests/common/sql/operators/test_sql.py index c2e4bc6f2e86f..d2a371311c1d0 100644 --- a/providers/tests/common/sql/operators/test_sql.py +++ b/providers/tests/common/sql/operators/test_sql.py @@ -22,6 +22,7 @@ from unittest.mock import MagicMock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_3_0_PLUS from airflow import DAG from airflow.exceptions import AirflowException @@ -44,8 +45,6 @@ from airflow.utils.session import create_session from airflow.utils.state import State -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_3_0_PLUS - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/common/sql/operators/test_sql_execute.py b/providers/tests/common/sql/operators/test_sql_execute.py index bb42cbaba2026..f03c80866ca39 100644 --- a/providers/tests/common/sql/operators/test_sql_execute.py +++ b/providers/tests/common/sql/operators/test_sql_execute.py @@ -22,6 +22,7 @@ from unittest.mock import MagicMock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.models import Connection from airflow.providers.common.compat.openlineage.facet import ( @@ -34,8 +35,6 @@ from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.openlineage.extractors.base import OperatorLineage -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/common/sql/sensors/test_sql.py b/providers/tests/common/sql/sensors/test_sql.py index 33e8fe6399738..c44bbf1d9d1ab 100644 --- a/providers/tests/common/sql/sensors/test_sql.py +++ b/providers/tests/common/sql/sensors/test_sql.py @@ -20,6 +20,7 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from airflow.exceptions import AirflowException from airflow.models.dag import DAG @@ -27,8 +28,6 @@ from airflow.providers.common.sql.sensors.sql import SqlSensor from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.8.0+ only"), pytest.mark.skip_if_database_isolation_mode, diff --git a/providers/tests/common/sql/test_utils.py b/providers/tests/common/sql/test_utils.py index 19b7bcc339411..50a217c87a2cc 100644 --- a/providers/tests/common/sql/test_utils.py +++ b/providers/tests/common/sql/test_utils.py @@ -21,11 +21,10 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.models import Connection -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/conftest.py b/providers/tests/conftest.py index 6ca7990e6054f..85ffd8f1d9078 100644 --- a/providers/tests/conftest.py +++ b/providers/tests/conftest.py @@ -22,7 +22,7 @@ import pytest -pytest_plugins = "dev.tests_common.pytest_plugin" +pytest_plugins = "tests_common.pytest_plugin" @pytest.hookimpl(tryfirst=True) diff --git a/providers/tests/databricks/plugins/test_databricks_workflow.py b/providers/tests/databricks/plugins/test_databricks_workflow.py index 695466c62d266..72d6b3f321f39 100644 --- a/providers/tests/databricks/plugins/test_databricks_workflow.py +++ b/providers/tests/databricks/plugins/test_databricks_workflow.py @@ -21,6 +21,7 @@ import pytest from flask import url_for +from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES from airflow.exceptions import AirflowException from airflow.models.dagrun import DagRun @@ -41,8 +42,6 @@ ) from airflow.www.app import create_app -from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES - DAG_ID = "test_dag" TASK_ID = "test_task" RUN_ID = "test_run_1" diff --git a/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py b/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py index 9a9b4a7dd885f..4a185971e67a7 100644 --- a/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py +++ b/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py @@ -21,6 +21,8 @@ from unittest import mock import pytest +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.mock_plugins import mock_plugin_manager from airflow.api_connexion.exceptions import PermissionDenied from airflow.configuration import conf @@ -41,9 +43,6 @@ from airflow.utils.state import State from airflow.www import app -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager - # Note: Sounds a bit strange to disable internal API tests in isolation mode but... # As long as the test is modelled to run its own internal API endpoints, it is conflicting # to the test setup with a dedicated internal API server. diff --git a/providers/tests/edge/cli/test_edge_command.py b/providers/tests/edge/cli/test_edge_command.py index af3e2c00e2998..e6e07d032baa6 100644 --- a/providers/tests/edge/cli/test_edge_command.py +++ b/providers/tests/edge/cli/test_edge_command.py @@ -23,6 +23,7 @@ import pytest import time_machine +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.providers.edge.cli.edge_command import ( @@ -34,8 +35,6 @@ from airflow.providers.edge.models.edge_worker import EdgeWorker, EdgeWorkerState from airflow.utils.state import TaskInstanceState -from dev.tests_common.test_utils.config import conf_vars - pytest.importorskip("pydantic", minversion="2.0.0") # Ignore the following error for mocking diff --git a/providers/tests/edge/plugins/test_edge_executor_plugin.py b/providers/tests/edge/plugins/test_edge_executor_plugin.py index d0c5a40770b31..bdd4847865539 100644 --- a/providers/tests/edge/plugins/test_edge_executor_plugin.py +++ b/providers/tests/edge/plugins/test_edge_executor_plugin.py @@ -19,12 +19,11 @@ import importlib import pytest +from tests_common.test_utils.config import conf_vars from airflow.plugins_manager import AirflowPlugin from airflow.providers.edge.plugins import edge_executor_plugin -from dev.tests_common.test_utils.config import conf_vars - def test_plugin_inactive(): with conf_vars({("edge", "api_enabled"): "false"}): diff --git a/providers/tests/elasticsearch/log/test_es_task_handler.py b/providers/tests/elasticsearch/log/test_es_task_handler.py index abde5daf8bf18..adb10090355b1 100644 --- a/providers/tests/elasticsearch/log/test_es_task_handler.py +++ b/providers/tests/elasticsearch/log/test_es_task_handler.py @@ -31,6 +31,8 @@ import elasticsearch import pendulum import pytest +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.configuration import conf from airflow.providers.elasticsearch.log.es_response import ElasticSearchResponse @@ -44,8 +46,6 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs from providers.tests.elasticsearch.log.elasticmock import elasticmock from providers.tests.elasticsearch.log.elasticmock.utilities import SearchFailedException diff --git a/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py b/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py index 2c783e2046f88..65db6a654ae23 100644 --- a/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py @@ -21,12 +21,11 @@ import pytest from flask import Response from flask_appbuilder.const import AUTH_LDAP +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from airflow.providers.fab.auth_manager.api.auth.backend.basic_auth import requires_authentication from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), ] diff --git a/providers/tests/fab/auth_manager/api/auth/backend/test_kerberos_auth.py b/providers/tests/fab/auth_manager/api/auth/backend/test_kerberos_auth.py index 7d13a89e8db80..e57f34ce4b033 100644 --- a/providers/tests/fab/auth_manager/api/auth/backend/test_kerberos_auth.py +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_kerberos_auth.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth import init_app diff --git a/providers/tests/fab/auth_manager/api/auth/backend/test_session.py b/providers/tests/fab/auth_manager/api/auth/backend/test_session.py index 513e8ec2aa293..405eafe11dfc4 100644 --- a/providers/tests/fab/auth_manager/api/auth/backend/test_session.py +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_session.py @@ -20,12 +20,11 @@ import pytest from flask import Response +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from airflow.providers.fab.auth_manager.api.auth.backend.session import requires_authentication from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS - pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), ] diff --git a/providers/tests/fab/auth_manager/api_endpoints/api_connexion_utils.py b/providers/tests/fab/auth_manager/api_endpoints/api_connexion_utils.py index e4cbe93c9d62a..b208b845096b9 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/api_connexion_utils.py +++ b/providers/tests/fab/auth_manager/api_endpoints/api_connexion_utils.py @@ -18,7 +18,7 @@ from contextlib import contextmanager -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py index 5c22e5ba7ff92..104e7943f4cd1 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py @@ -20,14 +20,14 @@ import pytest import time_machine +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_assets, clear_db_runs +from tests_common.test_utils.www import _check_last_log from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.security import permissions from airflow.utils import timezone -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_assets, clear_db_runs -from dev.tests_common.test_utils.www import _check_last_log from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user try: diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_auth.py b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py index 630ce7050bed5..fd29120070268 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_auth.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py @@ -20,12 +20,11 @@ import pytest from flask_login import current_user - -from dev.tests_common.test_utils.api_connexion_utils import assert_401 -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_pools -from dev.tests_common.test_utils.www import client_with_login +from tests_common.test_utils.api_connexion_utils import assert_401 +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_pools +from tests_common.test_utils.www import client_with_login pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py index f8015ff907ba6..4d7a5fc08776a 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py @@ -22,6 +22,13 @@ import pendulum import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import ( + clear_db_backfills, + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, +) from airflow.models import DagBag, DagModel from airflow.models.dag import DAG @@ -31,13 +38,6 @@ from airflow.utils import timezone from airflow.utils.session import provide_session -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import ( - clear_db_backfills, - clear_db_dags, - clear_db_runs, - clear_db_serialized_dags, -) from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user try: diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_cors.py b/providers/tests/fab/auth_manager/api_endpoints/test_cors.py index 8dbc4f964e61a..3741d71fb8b96 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_cors.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_cors.py @@ -19,10 +19,9 @@ from base64 import b64encode import pytest - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_pools +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_pools pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py index e93425591f708..f2bd83efa331c 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py @@ -21,6 +21,9 @@ import pendulum import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.www import _check_last_log from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import DagBag, DagModel @@ -29,9 +32,6 @@ from airflow.security import permissions from airflow.utils.session import provide_session -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from dev.tests_common.test_utils.www import _check_last_log from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py index fa09df1be74e0..f822cddf49ee0 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py @@ -19,6 +19,8 @@ from datetime import timedelta import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags from airflow.models.dag import DAG, DagModel from airflow.models.dagrun import DagRun @@ -28,8 +30,6 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py index 27e41136555ea..8cfb6c58f5320 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py @@ -21,12 +21,12 @@ from typing import TYPE_CHECKING import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags from airflow.models import DagBag from airflow.security import permissions -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py index 01a2c68091ce1..ec8a72d74ee48 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py @@ -17,14 +17,14 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags from airflow.models.dag import DagModel from airflow.models.dagwarning import DagWarning from airflow.security import permissions from airflow.utils.session import create_session -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py index 4794893241f6f..7d7868582934e 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py @@ -17,13 +17,13 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_logs from airflow.models import Log from airflow.security import permissions from airflow.utils import timezone -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_logs from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py index 110e8e6302553..b9b8d4ff9a421 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py @@ -17,14 +17,14 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError +from tests_common.test_utils.db import clear_db_dags, clear_db_import_errors +from tests_common.test_utils.permissions import _resource_name from airflow.models.dag import DagModel from airflow.security import permissions from airflow.utils import timezone -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_import_errors -from dev.tests_common.test_utils.permissions import _resource_name from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py index b72c6fe6612f3..986b55e5ff42e 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py @@ -17,12 +17,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.api_connexion_utils import assert_401 +from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.security import permissions -from dev.tests_common.test_utils.api_connexion_utils import assert_401 -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_role, create_user, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py index 7d379b6f8c007..6aea6b86f7b50 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py @@ -20,6 +20,8 @@ import urllib import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import DagRun, TaskInstance @@ -29,8 +31,6 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py index c7a20cb59c320..29f84850d9ba6 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py @@ -20,15 +20,15 @@ import pytest from sqlalchemy.sql.functions import count +from tests_common.test_utils.api_connexion_utils import assert_401 +from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.config import conf_vars from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.security import permissions from airflow.utils import timezone from airflow.utils.session import create_session -from dev.tests_common.test_utils.api_connexion_utils import assert_401 -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error -from dev.tests_common.test_utils.config import conf_vars from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_role, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py index 802eb4824087e..038dea9e72df9 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py @@ -17,12 +17,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_variables from airflow.models import Variable from airflow.security import permissions -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_variables from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py index 06e8ee4847762..f45cf58de6a95 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py @@ -19,6 +19,8 @@ from datetime import timedelta import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom from airflow.models.dag import DagModel from airflow.models.dagrun import DagRun @@ -30,8 +32,6 @@ from airflow.utils.session import create_session from airflow.utils.types import DagRunType -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/cli_commands/test_definition.py b/providers/tests/fab/auth_manager/cli_commands/test_definition.py index de906bef1ba30..572cbee05e3db 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_definition.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_definition.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands.definition import ( diff --git a/providers/tests/fab/auth_manager/cli_commands/test_role_command.py b/providers/tests/fab/auth_manager/cli_commands/test_role_command.py index fd176702c7219..d07cfc61242fe 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_role_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_role_command.py @@ -23,12 +23,11 @@ from typing import TYPE_CHECKING import pytest +from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.config import conf_vars from airflow.cli import cli_parser -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error -from dev.tests_common.test_utils.config import conf_vars - with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import role_command from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder diff --git a/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py b/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py index e0e4a70f4ac88..a0345909dc2df 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py @@ -20,11 +20,10 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.cli import cli_parser -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error - with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import sync_perm_command diff --git a/providers/tests/fab/auth_manager/cli_commands/test_user_command.py b/providers/tests/fab/auth_manager/cli_commands/test_user_command.py index 5f2f66b0866a2..6ccd4c99716ab 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_user_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_user_command.py @@ -24,11 +24,10 @@ from io import StringIO import pytest +from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.cli import cli_parser -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error - with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import user_command from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder diff --git a/providers/tests/fab/auth_manager/cli_commands/test_utils.py b/providers/tests/fab/auth_manager/cli_commands/test_utils.py index f52defae6bb8f..193317249ed11 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_utils.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_utils.py @@ -17,8 +17,7 @@ from __future__ import annotations import pytest - -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder diff --git a/providers/tests/fab/auth_manager/conftest.py b/providers/tests/fab/auth_manager/conftest.py index 9102f5d0f65d4..5818ecf435db6 100644 --- a/providers/tests/fab/auth_manager/conftest.py +++ b/providers/tests/fab/auth_manager/conftest.py @@ -17,12 +17,11 @@ from __future__ import annotations import pytest +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules from airflow.www import app -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules - @pytest.fixture(scope="session") def minimal_app_for_auth_api(): diff --git a/providers/tests/fab/auth_manager/decorators/test_auth.py b/providers/tests/fab/auth_manager/decorators/test_auth.py index 202f0d6227073..d6f60bf42f1fb 100644 --- a/providers/tests/fab/auth_manager/decorators/test_auth.py +++ b/providers/tests/fab/auth_manager/decorators/test_auth.py @@ -19,11 +19,10 @@ from unittest.mock import Mock, patch import pytest +from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.security.permissions import ACTION_CAN_READ, RESOURCE_DAG -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error - permissions = [(ACTION_CAN_READ, RESOURCE_DAG)] with ignore_provider_compatibility_error("2.9.0+", __file__): diff --git a/providers/tests/fab/auth_manager/models/test_anonymous_user.py b/providers/tests/fab/auth_manager/models/test_anonymous_user.py index 419d17aa9f3f9..eaf6b357f9264 100644 --- a/providers/tests/fab/auth_manager/models/test_anonymous_user.py +++ b/providers/tests/fab/auth_manager/models/test_anonymous_user.py @@ -17,7 +17,7 @@ # under the License. from __future__ import annotations -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser diff --git a/providers/tests/fab/auth_manager/schemas/test_user_schema.py b/providers/tests/fab/auth_manager/schemas/test_user_schema.py index 648372d3baebd..9f8dbd3742fa8 100644 --- a/providers/tests/fab/auth_manager/schemas/test_user_schema.py +++ b/providers/tests/fab/auth_manager/schemas/test_user_schema.py @@ -17,10 +17,10 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.utils import timezone -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role with ignore_provider_compatibility_error("2.9.0+", __file__): diff --git a/providers/tests/fab/auth_manager/security_manager/test_constants.py b/providers/tests/fab/auth_manager/security_manager/test_constants.py index a6566fd987109..dbe592c59d747 100644 --- a/providers/tests/fab/auth_manager/security_manager/test_constants.py +++ b/providers/tests/fab/auth_manager/security_manager/test_constants.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.constants import EXISTING_ROLES diff --git a/providers/tests/fab/auth_manager/security_manager/test_override.py b/providers/tests/fab/auth_manager/security_manager/test_override.py index 2733b688d1ae5..6ba1ccda292cd 100644 --- a/providers/tests/fab/auth_manager/security_manager/test_override.py +++ b/providers/tests/fab/auth_manager/security_manager/test_override.py @@ -19,7 +19,7 @@ from unittest import mock from unittest.mock import Mock -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride diff --git a/providers/tests/fab/auth_manager/test_fab_auth_manager.py b/providers/tests/fab/auth_manager/test_fab_auth_manager.py index 064c8e0dfd4c7..0d111f0453ad4 100644 --- a/providers/tests/fab/auth_manager/test_fab_auth_manager.py +++ b/providers/tests/fab/auth_manager/test_fab_auth_manager.py @@ -31,7 +31,7 @@ except ImportError: pass -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager diff --git a/providers/tests/fab/auth_manager/test_models.py b/providers/tests/fab/auth_manager/test_models.py index 6f03be373187b..76b69c3dea734 100644 --- a/providers/tests/fab/auth_manager/test_models.py +++ b/providers/tests/fab/auth_manager/test_models.py @@ -19,8 +19,7 @@ from unittest import mock from sqlalchemy import Column, MetaData, String, Table - -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import ( diff --git a/providers/tests/fab/auth_manager/test_security.py b/providers/tests/fab/auth_manager/test_security.py index 8c9e221b71df0..8156999f01615 100644 --- a/providers/tests/fab/auth_manager/test_security.py +++ b/providers/tests/fab/auth_manager/test_security.py @@ -31,19 +31,23 @@ from flask_appbuilder import SQLA, Model, expose, has_access from flask_appbuilder.views import BaseView, ModelView from sqlalchemy import Column, Date, Float, Integer, String +from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.configuration import initialize_config from airflow.exceptions import AirflowException from airflow.models import DagModel from airflow.models.dag import DAG -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error - with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager from airflow.providers.fab.auth_manager.models import assoc_permission_role from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests_common.test_utils.mock_security_manager import MockSecurityManager +from tests_common.test_utils.permissions import _resource_name + from airflow.security import permissions from airflow.security.permissions import ACTION_CAN_READ from airflow.www import app as application @@ -51,10 +55,6 @@ from airflow.www.extensions.init_auth_manager import get_auth_manager from airflow.www.utils import CustomSQLAInterface -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs -from dev.tests_common.test_utils.mock_security_manager import MockSecurityManager -from dev.tests_common.test_utils.permissions import _resource_name from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, create_user_scope, diff --git a/providers/tests/fab/auth_manager/views/test_permissions.py b/providers/tests/fab/auth_manager/views/test_permissions.py index 2ac26ffe45e87..eeb005b47e69a 100644 --- a/providers/tests/fab/auth_manager/views/test_permissions.py +++ b/providers/tests/fab/auth_manager/views/test_permissions.py @@ -18,12 +18,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.www import client_with_login from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/views/test_roles_list.py b/providers/tests/fab/auth_manager/views/test_roles_list.py index 3c1509d75350a..f94907c3e0504 100644 --- a/providers/tests/fab/auth_manager/views/test_roles_list.py +++ b/providers/tests/fab/auth_manager/views/test_roles_list.py @@ -18,12 +18,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.www import client_with_login from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/views/test_user.py b/providers/tests/fab/auth_manager/views/test_user.py index ee068f7f319fe..140d7cd331255 100644 --- a/providers/tests/fab/auth_manager/views/test_user.py +++ b/providers/tests/fab/auth_manager/views/test_user.py @@ -18,12 +18,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.www import client_with_login from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/views/test_user_edit.py b/providers/tests/fab/auth_manager/views/test_user_edit.py index 7cdc1a493b007..fb994dbd66ec7 100644 --- a/providers/tests/fab/auth_manager/views/test_user_edit.py +++ b/providers/tests/fab/auth_manager/views/test_user_edit.py @@ -18,12 +18,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.www import client_with_login from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/fab/auth_manager/views/test_user_stats.py b/providers/tests/fab/auth_manager/views/test_user_stats.py index e50bc87535a48..bb7d627fdaa64 100644 --- a/providers/tests/fab/auth_manager/views/test_user_stats.py +++ b/providers/tests/fab/auth_manager/views/test_user_stats.py @@ -18,12 +18,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.www import client_with_login from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user pytestmark = [ diff --git a/providers/tests/google/cloud/hooks/test_bigquery_system.py b/providers/tests/google/cloud/hooks/test_bigquery_system.py index 676ab35035839..454ab63079794 100644 --- a/providers/tests/google/cloud/hooks/test_bigquery_system.py +++ b/providers/tests/google/cloud/hooks/test_bigquery_system.py @@ -20,10 +20,10 @@ from __future__ import annotations import pytest +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest from airflow.providers.google.cloud.hooks import bigquery as hook -from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY diff --git a/providers/tests/google/cloud/hooks/test_kms_system.py b/providers/tests/google/cloud/hooks/test_kms_system.py index 374fabb5246dd..36c5bfa207f3a 100644 --- a/providers/tests/google/cloud/hooks/test_kms_system.py +++ b/providers/tests/google/cloud/hooks/test_kms_system.py @@ -21,10 +21,10 @@ from tempfile import TemporaryDirectory import pytest +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from airflow.providers.google.cloud.hooks.kms import CloudKMSHook -from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from providers.tests.google.cloud.utils.gcp_authenticator import GCP_KMS_KEY # To prevent resource name collisions, key ring and key resources CANNOT be deleted, so diff --git a/providers/tests/google/cloud/hooks/test_secret_manager_system.py b/providers/tests/google/cloud/hooks/test_secret_manager_system.py index d13305f52b42e..9f512f5f23c8f 100644 --- a/providers/tests/google/cloud/hooks/test_secret_manager_system.py +++ b/providers/tests/google/cloud/hooks/test_secret_manager_system.py @@ -19,10 +19,10 @@ import os import pytest +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from airflow.providers.google.cloud.hooks.secret_manager import SecretsManagerHook -from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from providers.tests.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY TEST_SECRET_ID = os.environ.get("GCP_SECRET_MANAGER_SECRET_ID", "test-secret") diff --git a/providers/tests/google/cloud/log/test_gcs_task_handler.py b/providers/tests/google/cloud/log/test_gcs_task_handler.py index 9f27c618de29d..369f4b888058b 100644 --- a/providers/tests/google/cloud/log/test_gcs_task_handler.py +++ b/providers/tests/google/cloud/log/test_gcs_task_handler.py @@ -23,14 +23,13 @@ from unittest.mock import MagicMock import pytest +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.providers.google.cloud.log.gcs_task_handler import GCSTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs - @pytest.mark.db_test class TestGCSTaskHandler: diff --git a/providers/tests/google/cloud/log/test_gcs_task_handler_system.py b/providers/tests/google/cloud/log/test_gcs_task_handler_system.py index eb1ce08d14f5c..e3e2bd2cafccc 100644 --- a/providers/tests/google/cloud/log/test_gcs_task_handler_system.py +++ b/providers/tests/google/cloud/log/test_gcs_task_handler_system.py @@ -23,6 +23,13 @@ from unittest import mock import pytest +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_connections, clear_db_runs +from tests_common.test_utils.gcp_system_helpers import ( + GoogleSystemTest, + provide_gcp_context, + resolve_full_gcp_key_path, +) from airflow import settings from airflow.example_dags import example_complex @@ -30,13 +37,6 @@ from airflow.utils.log.log_reader import TaskLogReader from airflow.utils.session import provide_session -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_connections, clear_db_runs -from dev.tests_common.test_utils.gcp_system_helpers import ( - GoogleSystemTest, - provide_gcp_context, - resolve_full_gcp_key_path, -) from providers.tests.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY diff --git a/providers/tests/google/cloud/log/test_stackdriver_task_handler.py b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py index 783f1a34b762c..60752f3b0d3f3 100644 --- a/providers/tests/google/cloud/log/test_stackdriver_task_handler.py +++ b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py @@ -24,16 +24,15 @@ import pytest from google.cloud.logging import Resource from google.cloud.logging_v2.types import ListLogEntriesRequest, ListLogEntriesResponse, LogEntry +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.exceptions import RemovedInAirflow3Warning from airflow.providers.google.cloud.log.stackdriver_task_handler import StackdriverTaskHandler from airflow.utils import timezone from airflow.utils.state import TaskInstanceState -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs - def _create_list_log_entries_response_mock(messages, token): return ListLogEntriesResponse( diff --git a/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py b/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py index a53dd43d08feb..da01e9c770570 100644 --- a/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py +++ b/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py @@ -23,6 +23,13 @@ from unittest import mock import pytest +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.gcp_system_helpers import ( + GoogleSystemTest, + provide_gcp_context, + resolve_full_gcp_key_path, +) from airflow import settings from airflow.example_dags import example_complex @@ -30,13 +37,6 @@ from airflow.utils.log.log_reader import TaskLogReader from airflow.utils.session import provide_session -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_runs -from dev.tests_common.test_utils.gcp_system_helpers import ( - GoogleSystemTest, - provide_gcp_context, - resolve_full_gcp_key_path, -) from providers.tests.google.cloud.utils.gcp_authenticator import GCP_STACKDRIVER diff --git a/providers/tests/google/cloud/operators/test_bigquery.py b/providers/tests/google/cloud/operators/test_bigquery.py index 3836d737662fc..1edc67c45a761 100644 --- a/providers/tests/google/cloud/operators/test_bigquery.py +++ b/providers/tests/google/cloud/operators/test_bigquery.py @@ -27,6 +27,12 @@ import pytest from google.cloud.bigquery import DEFAULT_RETRY, ScalarQueryParameter from google.cloud.exceptions import Conflict +from tests_common.test_utils.db import ( + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, + clear_db_xcom, +) from airflow.exceptions import ( AirflowException, @@ -75,13 +81,6 @@ from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.db import ( - clear_db_dags, - clear_db_runs, - clear_db_serialized_dags, - clear_db_xcom, -) - pytestmark = pytest.mark.db_test diff --git a/providers/tests/google/cloud/operators/test_bigquery_dts.py b/providers/tests/google/cloud/operators/test_bigquery_dts.py index f50c42805b49d..b196237b5bf84 100644 --- a/providers/tests/google/cloud/operators/test_bigquery_dts.py +++ b/providers/tests/google/cloud/operators/test_bigquery_dts.py @@ -21,6 +21,7 @@ from google.api_core.gapic_v1.method import DEFAULT from google.cloud.bigquery_datatransfer_v1 import StartManualTransferRunsResponse, TransferConfig, TransferRun +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.providers.google.cloud.operators.bigquery_dts import ( BigQueryCreateDataTransferOperator, @@ -28,8 +29,6 @@ BigQueryDeleteDataTransferConfigOperator, ) -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS - PROJECT_ID = "id" diff --git a/providers/tests/google/cloud/operators/test_dataprep_system.py b/providers/tests/google/cloud/operators/test_dataprep_system.py index dad77ac4ff806..9706859f02577 100644 --- a/providers/tests/google/cloud/operators/test_dataprep_system.py +++ b/providers/tests/google/cloud/operators/test_dataprep_system.py @@ -21,14 +21,13 @@ import os import pytest +from tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest +from tests_common.test_utils.system_tests import get_test_run from airflow.models import Connection from airflow.utils.session import create_session -from dev.tests_common.test_utils.db import clear_db_connections -from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest -from dev.tests_common.test_utils.system_tests import get_test_run - TOKEN = os.environ.get("DATAPREP_TOKEN") EXTRA = {"token": TOKEN} diff --git a/providers/tests/google/cloud/operators/test_dataproc.py b/providers/tests/google/cloud/operators/test_dataproc.py index cf4bffa3a0929..a8b747d564a6f 100644 --- a/providers/tests/google/cloud/operators/test_dataproc.py +++ b/providers/tests/google/cloud/operators/test_dataproc.py @@ -27,6 +27,8 @@ from google.api_core.retry_async import AsyncRetry from google.cloud import dataproc from google.cloud.dataproc_v1 import Batch, Cluster, JobStatus +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_VERSION +from tests_common.test_utils.db import clear_db_runs, clear_db_xcom from airflow.exceptions import ( AirflowException, @@ -80,9 +82,6 @@ from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_VERSION -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom - AIRFLOW_VERSION_LABEL = "v" + str(AIRFLOW_VERSION).replace(".", "-").replace("+", "-") cluster_params = inspect.signature(ClusterGenerator.__init__).parameters diff --git a/providers/tests/google/cloud/operators/test_datastore_system.py b/providers/tests/google/cloud/operators/test_datastore_system.py index 8807288358bae..3a9e82241146c 100644 --- a/providers/tests/google/cloud/operators/test_datastore_system.py +++ b/providers/tests/google/cloud/operators/test_datastore_system.py @@ -20,12 +20,12 @@ import os import pytest - -from dev.tests_common.test_utils.gcp_system_helpers import ( +from tests_common.test_utils.gcp_system_helpers import ( CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context, ) + from providers.tests.google.cloud.utils.gcp_authenticator import GCP_DATASTORE_KEY BUCKET = os.environ.get("GCP_DATASTORE_BUCKET", "datastore-system-test") diff --git a/providers/tests/google/cloud/operators/test_looker.py b/providers/tests/google/cloud/operators/test_looker.py index b368259c503e0..faefc0eca5a08 100644 --- a/providers/tests/google/cloud/operators/test_looker.py +++ b/providers/tests/google/cloud/operators/test_looker.py @@ -20,14 +20,13 @@ from unittest.mock import MagicMock import pytest +from tests_common.test_utils.db import clear_db_runs, clear_db_xcom from airflow.exceptions import AirflowException from airflow.models import DAG, DagBag from airflow.providers.google.cloud.operators.looker import LookerStartPdtBuildOperator from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom - OPERATOR_PATH = "airflow.providers.google.cloud.operators.looker.{}" TASK_ID = "task-id" diff --git a/providers/tests/google/cloud/secrets/test_secret_manager_system.py b/providers/tests/google/cloud/secrets/test_secret_manager_system.py index b9b8b6d0cac82..04b7a9fe1f56f 100644 --- a/providers/tests/google/cloud/secrets/test_secret_manager_system.py +++ b/providers/tests/google/cloud/secrets/test_secret_manager_system.py @@ -22,8 +22,8 @@ from unittest import mock import pytest +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context -from dev.tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from providers.tests.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY BACKEND_IMPORT_PATH = "airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend" diff --git a/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py index 9cb0a9be5264e..4942a36648c21 100644 --- a/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py @@ -22,17 +22,17 @@ from contextlib import contextmanager import pytest +from tests_common.test_utils.gcp_system_helpers import ( + GoogleSystemTest, + provide_gcp_context, +) +from tests_common.test_utils.system_tests import get_test_run from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.google.cloud.example_dags import example_facebook_ads_to_gcs from airflow.utils.process_utils import patch_environ -from dev.tests_common.test_utils.gcp_system_helpers import ( - GoogleSystemTest, - provide_gcp_context, -) -from dev.tests_common.test_utils.system_tests import get_test_run from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") diff --git a/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py index d556f2d86e2f2..e525ca1adde21 100644 --- a/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py @@ -19,15 +19,15 @@ import os import pytest - -from airflow.providers.google.cloud.example_dags import example_salesforce_to_gcs - -from dev.tests_common.test_utils.gcp_system_helpers import ( +from tests_common.test_utils.gcp_system_helpers import ( GoogleSystemTest, provide_gcp_context, ) -from dev.tests_common.test_utils.salesforce_system_helpers import provide_salesforce_connection -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.salesforce_system_helpers import provide_salesforce_connection +from tests_common.test_utils.system_tests import get_test_run + +from airflow.providers.google.cloud.example_dags import example_salesforce_to_gcs + from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") diff --git a/providers/tests/google/cloud/utils/gcp_authenticator.py b/providers/tests/google/cloud/utils/gcp_authenticator.py index 6bb11b260644c..36bca98e9a6a7 100644 --- a/providers/tests/google/cloud/utils/gcp_authenticator.py +++ b/providers/tests/google/cloud/utils/gcp_authenticator.py @@ -21,13 +21,13 @@ import os import subprocess +from tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from tests_common.test_utils.logging_command_executor import CommandExecutor + from airflow import settings from airflow.exceptions import AirflowException from airflow.models import Connection -from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER -from dev.tests_common.test_utils.logging_command_executor import CommandExecutor - # Please keep these variables in alphabetical order. GCP_AI_KEY = "gcp_ai.json" diff --git a/providers/tests/google/common/auth_backend/test_google_openid.py b/providers/tests/google/common/auth_backend/test_google_openid.py index 67b0ff2003d22..02f08e6107070 100644 --- a/providers/tests/google/common/auth_backend/test_google_openid.py +++ b/providers/tests/google/common/auth_backend/test_google_openid.py @@ -20,14 +20,13 @@ import pytest from google.auth.exceptions import GoogleAuthError +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_pools +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules from airflow.www.app import create_app -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_pools -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules - @pytest.fixture(scope="module") def google_openid_app(): diff --git a/providers/tests/google/marketing_platform/operators/test_display_video_system.py b/providers/tests/google/marketing_platform/operators/test_display_video_system.py index 49f44948abfdf..57e93a0544a15 100644 --- a/providers/tests/google/marketing_platform/operators/test_display_video_system.py +++ b/providers/tests/google/marketing_platform/operators/test_display_video_system.py @@ -17,6 +17,12 @@ from __future__ import annotations import pytest +from tests_common.test_utils.gcp_system_helpers import ( + MARKETING_DAG_FOLDER, + GoogleSystemTest, + provide_gcp_context, +) +from tests_common.test_utils.system_tests import get_test_run from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.providers.google.marketing_platform.example_dags.example_display_video import ( @@ -25,12 +31,6 @@ dag_example_display_video_sdf, ) -from dev.tests_common.test_utils.gcp_system_helpers import ( - MARKETING_DAG_FOLDER, - GoogleSystemTest, - provide_gcp_context, -) -from dev.tests_common.test_utils.system_tests import get_test_run from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GMP_KEY # Requires the following scope: diff --git a/providers/tests/hashicorp/hooks/test_vault.py b/providers/tests/hashicorp/hooks/test_vault.py index 442a656d20064..8aa555567bb4c 100644 --- a/providers/tests/hashicorp/hooks/test_vault.py +++ b/providers/tests/hashicorp/hooks/test_vault.py @@ -22,13 +22,12 @@ import pytest from hvac.exceptions import VaultError +from tests_common.test_utils.config import conf_vars from airflow.configuration import AirflowConfigParser from airflow.exceptions import AirflowConfigException from airflow.providers.hashicorp.hooks.vault import VaultHook -from dev.tests_common.test_utils.config import conf_vars - class TestVaultHook: @staticmethod diff --git a/providers/tests/imap/hooks/test_imap.py b/providers/tests/imap/hooks/test_imap.py index 2971cfd1924c6..9d2fcad5c74b3 100644 --- a/providers/tests/imap/hooks/test_imap.py +++ b/providers/tests/imap/hooks/test_imap.py @@ -22,14 +22,13 @@ from unittest.mock import Mock, mock_open, patch import pytest +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.imap.hooks.imap import ImapHook from airflow.utils import db -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/microsoft/azure/log/test_wasb_task_handler.py b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py index 224961efe4336..2200a65f4d0d7 100644 --- a/providers/tests/microsoft/azure/log/test_wasb_task_handler.py +++ b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py @@ -24,15 +24,14 @@ import pytest from azure.common import AzureHttpError +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.providers.microsoft.azure.hooks.wasb import WasbHook from airflow.providers.microsoft.azure.log.wasb_task_handler import WasbTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs - pytestmark = pytest.mark.db_test diff --git a/providers/tests/mongo/hooks/test_mongo.py b/providers/tests/mongo/hooks/test_mongo.py index 78756b0cc4890..9cd5b4f0677d5 100644 --- a/providers/tests/mongo/hooks/test_mongo.py +++ b/providers/tests/mongo/hooks/test_mongo.py @@ -23,13 +23,12 @@ import pymongo import pytest +from tests_common.test_utils.compat import connection_as_json from airflow.exceptions import AirflowConfigException, AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.mongo.hooks.mongo import MongoHook -from dev.tests_common.test_utils.compat import connection_as_json - pytestmark = pytest.mark.db_test if TYPE_CHECKING: diff --git a/providers/tests/mysql/hooks/test_mysql.py b/providers/tests/mysql/hooks/test_mysql.py index 23748ae9cac19..49358ac30dcff 100644 --- a/providers/tests/mysql/hooks/test_mysql.py +++ b/providers/tests/mysql/hooks/test_mysql.py @@ -36,9 +36,9 @@ pytest.skip("MySQL not available", allow_module_level=True) -from airflow.utils import timezone +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces -from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces +from airflow.utils import timezone SSL_DICT = {"cert": "/tmp/client-cert.pem", "ca": "/tmp/server-ca.pem", "key": "/tmp/client-key.pem"} diff --git a/providers/tests/openlineage/extractors/test_bash.py b/providers/tests/openlineage/extractors/test_bash.py index d4fcdf7af2b7a..8782ce161c637 100644 --- a/providers/tests/openlineage/extractors/test_bash.py +++ b/providers/tests/openlineage/extractors/test_bash.py @@ -23,13 +23,12 @@ import pytest from openlineage.client.facet_v2 import source_code_job +from tests_common.test_utils.compat import BashOperator from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.openlineage.extractors.bash import BashExtractor -from dev.tests_common.test_utils.compat import BashOperator - pytestmark = pytest.mark.db_test with DAG( diff --git a/providers/tests/openlineage/extractors/test_manager.py b/providers/tests/openlineage/extractors/test_manager.py index 6bbf303e32857..dbd4a8e40368c 100644 --- a/providers/tests/openlineage/extractors/test_manager.py +++ b/providers/tests/openlineage/extractors/test_manager.py @@ -24,6 +24,7 @@ import pytest from openlineage.client.event_v2 import Dataset as OpenLineageDataset from openlineage.client.facet_v2 import documentation_dataset, ownership_dataset, schema_dataset +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS from airflow.io.path import ObjectStoragePath from airflow.lineage.entities import Column, File, Table, User @@ -35,8 +36,6 @@ from airflow.providers.openlineage.utils.utils import Asset from airflow.utils.state import State -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS - if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/providers/tests/openlineage/extractors/test_python.py b/providers/tests/openlineage/extractors/test_python.py index ef4fc0b861e7c..92a9d073a1109 100644 --- a/providers/tests/openlineage/extractors/test_python.py +++ b/providers/tests/openlineage/extractors/test_python.py @@ -25,14 +25,13 @@ import pytest from openlineage.client.facet_v2 import source_code_job +from tests_common.test_utils.compat import BashOperator from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.operators.python import PythonOperator from airflow.providers.openlineage.extractors.python import PythonExtractor -from dev.tests_common.test_utils.compat import BashOperator - pytestmark = pytest.mark.db_test dag = DAG( diff --git a/providers/tests/openlineage/plugins/test_adapter.py b/providers/tests/openlineage/plugins/test_adapter.py index 88f2250c638ef..8d57fad296279 100644 --- a/providers/tests/openlineage/plugins/test_adapter.py +++ b/providers/tests/openlineage/plugins/test_adapter.py @@ -36,6 +36,8 @@ processing_engine_run, sql_job, ) +from tests_common.test_utils.compat import BashOperator +from tests_common.test_utils.config import conf_vars from airflow import DAG from airflow.models.dagrun import DagRun, DagRunState @@ -52,9 +54,6 @@ from airflow.providers.openlineage.utils.utils import get_airflow_job_facet from airflow.utils.task_group import TaskGroup -from dev.tests_common.test_utils.compat import BashOperator -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/openlineage/plugins/test_execution.py b/providers/tests/openlineage/plugins/test_execution.py index c308047b1bc0d..09262adc4a6f9 100644 --- a/providers/tests/openlineage/plugins/test_execution.py +++ b/providers/tests/openlineage/plugins/test_execution.py @@ -26,6 +26,8 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars from airflow.jobs.job import Job from airflow.jobs.local_task_job_runner import LocalTaskJobRunner @@ -37,9 +39,6 @@ from airflow.utils import timezone from airflow.utils.state import State -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/openlineage/plugins/test_listener.py b/providers/tests/openlineage/plugins/test_listener.py index 0c1651ccf0277..b859160cf8424 100644 --- a/providers/tests/openlineage/plugins/test_listener.py +++ b/providers/tests/openlineage/plugins/test_listener.py @@ -29,6 +29,8 @@ from openlineage.client import OpenLineageClient from openlineage.client.transport import ConsoleTransport from openlineage.client.transport.console import ConsoleConfig +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars from airflow.models import DAG, DagRun, TaskInstance from airflow.models.baseoperator import BaseOperator @@ -39,9 +41,6 @@ from airflow.providers.openlineage.utils.selective_enable import disable_lineage, enable_lineage from airflow.utils.state import DagRunState, State -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/openlineage/plugins/test_openlineage.py b/providers/tests/openlineage/plugins/test_openlineage.py index 8685326a29432..e1c36001ebfeb 100644 --- a/providers/tests/openlineage/plugins/test_openlineage.py +++ b/providers/tests/openlineage/plugins/test_openlineage.py @@ -22,9 +22,8 @@ from unittest.mock import patch import pytest - -from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from dev.tests_common.test_utils.config import conf_vars +from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from tests_common.test_utils.config import conf_vars @pytest.mark.skipif( diff --git a/providers/tests/openlineage/plugins/test_utils.py b/providers/tests/openlineage/plugins/test_utils.py index 444839ccaef30..f0685c1351685 100644 --- a/providers/tests/openlineage/plugins/test_utils.py +++ b/providers/tests/openlineage/plugins/test_utils.py @@ -27,6 +27,7 @@ from attrs import define from openlineage.client.utils import RedactMixin from pkg_resources import parse_version +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator from airflow.models import DAG as AIRFLOW_DAG, DagModel from airflow.providers.openlineage.plugins.facets import AirflowDebugRunFacet @@ -44,8 +45,6 @@ from airflow.utils.log.secrets_masker import _secrets_masker from airflow.utils.state import State -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/openlineage/test_conf.py b/providers/tests/openlineage/test_conf.py index 7f78a6a4c2eea..b1dd57d4d8415 100644 --- a/providers/tests/openlineage/test_conf.py +++ b/providers/tests/openlineage/test_conf.py @@ -20,6 +20,7 @@ from unittest import mock import pytest +from tests_common.test_utils.config import conf_vars, env_vars from airflow.exceptions import AirflowConfigException from airflow.providers.openlineage.conf import ( @@ -39,8 +40,6 @@ transport, ) -from dev.tests_common.test_utils.config import conf_vars, env_vars - _CONFIG_SECTION = "openlineage" _VAR_CONFIG_PATH = "OPENLINEAGE_CONFIG" _CONFIG_OPTION_CONFIG_PATH = "config_path" diff --git a/providers/tests/openlineage/utils/test_utils.py b/providers/tests/openlineage/utils/test_utils.py index 6cf7904546aba..5adeceb440dc1 100644 --- a/providers/tests/openlineage/utils/test_utils.py +++ b/providers/tests/openlineage/utils/test_utils.py @@ -21,6 +21,9 @@ import pathlib from unittest.mock import MagicMock, patch +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator +from tests_common.test_utils.mock_operators import MockOperator + from airflow import DAG from airflow.decorators import task from airflow.models.baseoperator import BaseOperator @@ -44,9 +47,6 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator -from dev.tests_common.test_utils.mock_operators import MockOperator - BASH_OPERATOR_PATH = "airflow.providers.standard.operators.bash" if not AIRFLOW_V_2_10_PLUS: BASH_OPERATOR_PATH = "airflow.operators.bash" @@ -191,7 +191,7 @@ def test_get_fully_qualified_class_name_mapped_operator(): mapped = MockOperator.partial(task_id="task_2").expand(arg2=["a", "b", "c"]) assert isinstance(mapped, MappedOperator) mapped_op_path = get_fully_qualified_class_name(mapped) - assert mapped_op_path == "dev.tests_common.test_utils.mock_operators.MockOperator" + assert mapped_op_path == "tests_common.test_utils.mock_operators.MockOperator" def test_get_fully_qualified_class_name_bash_operator(): diff --git a/providers/tests/opensearch/log/test_os_task_handler.py b/providers/tests/opensearch/log/test_os_task_handler.py index 1ffa0f5251191..81ac75c855aa9 100644 --- a/providers/tests/opensearch/log/test_os_task_handler.py +++ b/providers/tests/opensearch/log/test_os_task_handler.py @@ -32,6 +32,8 @@ opensearchpy = pytest.importorskip("opensearchpy") from opensearchpy.exceptions import NotFoundError +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.configuration import conf from airflow.providers.opensearch.log.os_response import OpensearchResponse @@ -44,8 +46,6 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs from providers.tests.opensearch.conftest import MockClient pytestmark = pytest.mark.db_test diff --git a/providers/tests/redis/log/test_redis_task_handler.py b/providers/tests/redis/log/test_redis_task_handler.py index bc7345df9a53b..0d83f3a303136 100644 --- a/providers/tests/redis/log/test_redis_task_handler.py +++ b/providers/tests/redis/log/test_redis_task_handler.py @@ -21,6 +21,7 @@ from unittest.mock import patch import pytest +from tests_common.test_utils.config import conf_vars from airflow.models import DAG, DagRun, TaskInstance from airflow.operators.empty import EmptyOperator @@ -29,8 +30,6 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/sftp/operators/test_sftp.py b/providers/tests/sftp/operators/test_sftp.py index e6c90c2c74f5d..9d5689c881485 100644 --- a/providers/tests/sftp/operators/test_sftp.py +++ b/providers/tests/sftp/operators/test_sftp.py @@ -25,6 +25,7 @@ import paramiko import pytest +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import DAG, Connection @@ -36,8 +37,6 @@ from airflow.utils import timezone from airflow.utils.timezone import datetime -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/smtp/hooks/test_smtp.py b/providers/tests/smtp/hooks/test_smtp.py index 04a20e0ca6f0a..fe9471c424933 100644 --- a/providers/tests/smtp/hooks/test_smtp.py +++ b/providers/tests/smtp/hooks/test_smtp.py @@ -25,14 +25,13 @@ from unittest.mock import Mock, patch import pytest +from tests_common.test_utils.config import conf_vars from airflow.models import Connection from airflow.providers.smtp.hooks.smtp import SmtpHook from airflow.utils import db from airflow.utils.session import create_session -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/smtp/notifications/test_smtp.py b/providers/tests/smtp/notifications/test_smtp.py index 75e1ebf530fae..594fa19dfee34 100644 --- a/providers/tests/smtp/notifications/test_smtp.py +++ b/providers/tests/smtp/notifications/test_smtp.py @@ -21,6 +21,8 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars from airflow.configuration import conf from airflow.models import SlaMiss @@ -32,9 +34,6 @@ ) from airflow.utils import timezone -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test SMTP_API_DEFAULT_CONN_ID = SmtpHook.default_conn_name diff --git a/providers/tests/ssh/operators/test_ssh.py b/providers/tests/ssh/operators/test_ssh.py index b304a24882bb9..55431bd3b629e 100644 --- a/providers/tests/ssh/operators/test_ssh.py +++ b/providers/tests/ssh/operators/test_ssh.py @@ -24,6 +24,7 @@ import pytest from paramiko.client import SSHClient +from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException, AirflowSkipException, AirflowTaskTimeout from airflow.models import TaskInstance @@ -32,8 +33,6 @@ from airflow.utils.timezone import datetime from airflow.utils.types import NOTSET -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test diff --git a/providers/tests/standard/operators/test_bash.py b/providers/tests/standard/operators/test_bash.py index 305651ee596ae..f040442ecccd3 100644 --- a/providers/tests/standard/operators/test_bash.py +++ b/providers/tests/standard/operators/test_bash.py @@ -26,6 +26,7 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.exceptions import AirflowException, AirflowSkipException, AirflowTaskTimeout from airflow.providers.standard.operators.bash import BashOperator @@ -33,8 +34,6 @@ from airflow.utils.state import State from airflow.utils.types import DagRunType -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/standard/operators/test_datetime.py b/providers/tests/standard/operators/test_datetime.py index 4fbb0863d35ae..da7f98f0e3543 100644 --- a/providers/tests/standard/operators/test_datetime.py +++ b/providers/tests/standard/operators/test_datetime.py @@ -21,6 +21,7 @@ import pytest import time_machine +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.exceptions import AirflowException from airflow.models.dagrun import DagRun @@ -31,8 +32,6 @@ from airflow.utils.session import create_session from airflow.utils.state import State -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/standard/operators/test_weekday.py b/providers/tests/standard/operators/test_weekday.py index 6c20299d15e79..576fd10b54542 100644 --- a/providers/tests/standard/operators/test_weekday.py +++ b/providers/tests/standard/operators/test_weekday.py @@ -21,6 +21,7 @@ import pytest import time_machine +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.exceptions import AirflowException from airflow.models.dagrun import DagRun @@ -33,8 +34,6 @@ from airflow.utils.state import State from airflow.utils.weekday import WeekDay -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/standard/sensors/test_weekday.py b/providers/tests/standard/sensors/test_weekday.py index ef0091a9fdd3a..5bdbf4bb79c81 100644 --- a/providers/tests/standard/sensors/test_weekday.py +++ b/providers/tests/standard/sensors/test_weekday.py @@ -20,6 +20,7 @@ from datetime import timedelta import pytest +from tests_common.test_utils import db from airflow.exceptions import AirflowSensorTimeout from airflow.models import DagBag @@ -28,8 +29,6 @@ from airflow.utils.timezone import datetime from airflow.utils.weekday import WeekDay -from dev.tests_common.test_utils import db - pytestmark = pytest.mark.db_test diff --git a/providers/tests/system/airbyte/example_airbyte_trigger_job.py b/providers/tests/system/airbyte/example_airbyte_trigger_job.py index 1dd3f01d72618..240a293bbd71b 100644 --- a/providers/tests/system/airbyte/example_airbyte_trigger_job.py +++ b/providers/tests/system/airbyte/example_airbyte_trigger_job.py @@ -61,7 +61,7 @@ # Task dependency created via `XComArgs`: # async_source_destination >> airbyte_sensor -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/alibaba/example_adb_spark_batch.py b/providers/tests/system/alibaba/example_adb_spark_batch.py index 35e03a088871d..e8447cefc4b58 100644 --- a/providers/tests/system/alibaba/example_adb_spark_batch.py +++ b/providers/tests/system/alibaba/example_adb_spark_batch.py @@ -50,14 +50,14 @@ spark_pi >> spark_lr - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() # [END howto_operator_adb_spark_batch] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/alibaba/example_adb_spark_sql.py b/providers/tests/system/alibaba/example_adb_spark_sql.py index ac29330d451db..cc587e424b2b4 100644 --- a/providers/tests/system/alibaba/example_adb_spark_sql.py +++ b/providers/tests/system/alibaba/example_adb_spark_sql.py @@ -42,14 +42,14 @@ show_databases >> show_tables - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() # [END howto_operator_adb_spark_sql] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/alibaba/example_oss_bucket.py b/providers/tests/system/alibaba/example_oss_bucket.py index 4870a3b577977..cfe7a42efc55d 100644 --- a/providers/tests/system/alibaba/example_oss_bucket.py +++ b/providers/tests/system/alibaba/example_oss_bucket.py @@ -41,14 +41,14 @@ create_bucket >> delete_bucket - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() # [END howto_operator_oss_bucket] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/alibaba/example_oss_object.py b/providers/tests/system/alibaba/example_oss_object.py index 7305c05bf8384..dfa59917fd6b0 100644 --- a/providers/tests/system/alibaba/example_oss_object.py +++ b/providers/tests/system/alibaba/example_oss_object.py @@ -65,14 +65,14 @@ create_object >> download_object >> delete_object >> delete_batch_object - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/CONTRIBUTING.md b/providers/tests/system/amazon/CONTRIBUTING.md index f12062aa7fb59..4abdf410dd63d 100644 --- a/providers/tests/system/amazon/CONTRIBUTING.md +++ b/providers/tests/system/amazon/CONTRIBUTING.md @@ -212,7 +212,7 @@ chain( task2, # task2 has trigger rule "all done" defined ) -from dev.tests_common.test_utils.watcher import watcher +from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG diff --git a/providers/tests/system/amazon/aws/example_appflow.py b/providers/tests/system/amazon/aws/example_appflow.py index db9dd9a428947..262b6cb803d44 100644 --- a/providers/tests/system/amazon/aws/example_appflow.py +++ b/providers/tests/system/amazon/aws/example_appflow.py @@ -111,13 +111,13 @@ should_be_skipped, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_appflow_run.py b/providers/tests/system/amazon/aws/example_appflow_run.py index d1151d1dd5c8b..106fda1bb7a42 100644 --- a/providers/tests/system/amazon/aws/example_appflow_run.py +++ b/providers/tests/system/amazon/aws/example_appflow_run.py @@ -193,13 +193,13 @@ def delete_flow(flow_name: str): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_athena.py b/providers/tests/system/amazon/aws/example_athena.py index 2ee1c11ab7881..201e017d12a64 100644 --- a/providers/tests/system/amazon/aws/example_athena.py +++ b/providers/tests/system/amazon/aws/example_athena.py @@ -174,14 +174,14 @@ def read_results_from_s3(bucket_name, query_execution_id): delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_azure_blob_to_s3.py b/providers/tests/system/amazon/aws/example_azure_blob_to_s3.py index 78a55a6a7f7dc..3834365622d30 100644 --- a/providers/tests/system/amazon/aws/example_azure_blob_to_s3.py +++ b/providers/tests/system/amazon/aws/example_azure_blob_to_s3.py @@ -72,13 +72,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_batch.py b/providers/tests/system/amazon/aws/example_batch.py index da035050a77b4..828815a2e1c86 100644 --- a/providers/tests/system/amazon/aws/example_batch.py +++ b/providers/tests/system/amazon/aws/example_batch.py @@ -283,13 +283,13 @@ def delete_job_queue(job_queue_name): log_cleanup, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_bedrock.py b/providers/tests/system/amazon/aws/example_bedrock.py index 8e67ee4d3880b..3caa9f3a35398 100644 --- a/providers/tests/system/amazon/aws/example_bedrock.py +++ b/providers/tests/system/amazon/aws/example_bedrock.py @@ -218,14 +218,14 @@ def run_or_skip(): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py b/providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py index 0d9eb278460f8..094cb854c3a4f 100644 --- a/providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py +++ b/providers/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py @@ -594,14 +594,14 @@ def delete_opensearch_policies(collection_name: str): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_cloudformation.py b/providers/tests/system/amazon/aws/example_cloudformation.py index 38827eebbdb5f..7553f5d9bd0a3 100644 --- a/providers/tests/system/amazon/aws/example_cloudformation.py +++ b/providers/tests/system/amazon/aws/example_cloudformation.py @@ -108,14 +108,14 @@ wait_for_stack_delete, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_comprehend.py b/providers/tests/system/amazon/aws/example_comprehend.py index d8a1f9af88b6c..a4e3bdd4852ef 100644 --- a/providers/tests/system/amazon/aws/example_comprehend.py +++ b/providers/tests/system/amazon/aws/example_comprehend.py @@ -126,13 +126,13 @@ def pii_entities_detection_job_workflow(): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_comprehend_document_classifier.py b/providers/tests/system/amazon/aws/example_comprehend_document_classifier.py index f5308265a83ca..160e3f3cadf84 100644 --- a/providers/tests/system/amazon/aws/example_comprehend_document_classifier.py +++ b/providers/tests/system/amazon/aws/example_comprehend_document_classifier.py @@ -231,13 +231,13 @@ def delete_connection(conn_id): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_datasync.py b/providers/tests/system/amazon/aws/example_datasync.py index a8363fe81b36e..8d25fcf9c3989 100644 --- a/providers/tests/system/amazon/aws/example_datasync.py +++ b/providers/tests/system/amazon/aws/example_datasync.py @@ -235,13 +235,13 @@ def delete_locations(locations): delete_s3_bucket_destination, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_dms.py b/providers/tests/system/amazon/aws/example_dms.py index b2e99daf6290f..5d15aa0c4ecb0 100644 --- a/providers/tests/system/amazon/aws/example_dms.py +++ b/providers/tests/system/amazon/aws/example_dms.py @@ -429,13 +429,13 @@ def delete_security_group(security_group_id: str, security_group_name: str): delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_dynamodb.py b/providers/tests/system/amazon/aws/example_dynamodb.py index 3d709c36b02ad..3cfa15ecaa36e 100644 --- a/providers/tests/system/amazon/aws/example_dynamodb.py +++ b/providers/tests/system/amazon/aws/example_dynamodb.py @@ -117,14 +117,14 @@ def delete_table(table_name: str): delete_table, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_dynamodb_to_s3.py b/providers/tests/system/amazon/aws/example_dynamodb_to_s3.py index 86a5d76c9587f..989fe79b2d9f6 100644 --- a/providers/tests/system/amazon/aws/example_dynamodb_to_s3.py +++ b/providers/tests/system/amazon/aws/example_dynamodb_to_s3.py @@ -256,13 +256,13 @@ def skip_incremental_export(start_time: datetime, end_time: datetime): delete_table, delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_ec2.py b/providers/tests/system/amazon/aws/example_ec2.py index e55adab775ffd..750ec19c27779 100644 --- a/providers/tests/system/amazon/aws/example_ec2.py +++ b/providers/tests/system/amazon/aws/example_ec2.py @@ -202,13 +202,13 @@ def parse_response(instance_ids: list): delete_key_pair(key_name), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_ecs.py b/providers/tests/system/amazon/aws/example_ecs.py index e1aad99b5e7cb..0f9118ae34af3 100644 --- a/providers/tests/system/amazon/aws/example_ecs.py +++ b/providers/tests/system/amazon/aws/example_ecs.py @@ -212,14 +212,14 @@ def clean_logs(group_name: str): clean_logs(log_group_name), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_ecs_fargate.py b/providers/tests/system/amazon/aws/example_ecs_fargate.py index ca592234aa77f..df3afbce7df71 100644 --- a/providers/tests/system/amazon/aws/example_ecs_fargate.py +++ b/providers/tests/system/amazon/aws/example_ecs_fargate.py @@ -156,13 +156,13 @@ def delete_cluster(cluster_name: str) -> None: delete_cluster(cluster_name), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_eks_templated.py b/providers/tests/system/amazon/aws/example_eks_templated.py index 3b1ba5059725a..de4d5e621ef0a 100644 --- a/providers/tests/system/amazon/aws/example_eks_templated.py +++ b/providers/tests/system/amazon/aws/example_eks_templated.py @@ -148,13 +148,13 @@ await_delete_cluster, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py b/providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py index 27ee2be0d728a..a267463c11f3d 100644 --- a/providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py +++ b/providers/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py @@ -141,14 +141,14 @@ await_delete_cluster, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py b/providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py index 34cbed1645fa5..1348b352cc51f 100644 --- a/providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py +++ b/providers/tests/system/amazon/aws/example_eks_with_fargate_profile.py @@ -175,13 +175,13 @@ await_delete_cluster, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py b/providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py index 11f843688524c..07538d94d17b3 100644 --- a/providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py +++ b/providers/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py @@ -155,14 +155,14 @@ def delete_launch_template(template_name: str): delete_launch_template(launch_template_name), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_eks_with_nodegroups.py b/providers/tests/system/amazon/aws/example_eks_with_nodegroups.py index 8356a8ee4375c..98448767d9745 100644 --- a/providers/tests/system/amazon/aws/example_eks_with_nodegroups.py +++ b/providers/tests/system/amazon/aws/example_eks_with_nodegroups.py @@ -198,14 +198,14 @@ def delete_launch_template(template_name: str): delete_launch_template(launch_template_name), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_emr.py b/providers/tests/system/amazon/aws/example_emr.py index 44c28630eb69e..9a29667b5d098 100644 --- a/providers/tests/system/amazon/aws/example_emr.py +++ b/providers/tests/system/amazon/aws/example_emr.py @@ -227,14 +227,14 @@ def get_step_id(step_ids: list): ] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_emr_eks.py b/providers/tests/system/amazon/aws/example_emr_eks.py index d95645fd306a6..cafaf09ed7473 100644 --- a/providers/tests/system/amazon/aws/example_emr_eks.py +++ b/providers/tests/system/amazon/aws/example_emr_eks.py @@ -320,13 +320,13 @@ def delete_virtual_cluster(virtual_cluster_id): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_emr_notebook_execution.py b/providers/tests/system/amazon/aws/example_emr_notebook_execution.py index 48d2b14c96573..681e97048aa58 100644 --- a/providers/tests/system/amazon/aws/example_emr_notebook_execution.py +++ b/providers/tests/system/amazon/aws/example_emr_notebook_execution.py @@ -112,13 +112,13 @@ wait_for_execution_finish, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_emr_serverless.py b/providers/tests/system/amazon/aws/example_emr_serverless.py index 4901fa6736059..9059f0e1b1e02 100644 --- a/providers/tests/system/amazon/aws/example_emr_serverless.py +++ b/providers/tests/system/amazon/aws/example_emr_serverless.py @@ -155,13 +155,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_eventbridge.py b/providers/tests/system/amazon/aws/example_eventbridge.py index 5470bd0ca70cc..6f31fc9ce37d8 100644 --- a/providers/tests/system/amazon/aws/example_eventbridge.py +++ b/providers/tests/system/amazon/aws/example_eventbridge.py @@ -81,7 +81,7 @@ chain(test_context, put_events, put_rule, enable_rule, disable_rule) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_ftp_to_s3.py b/providers/tests/system/amazon/aws/example_ftp_to_s3.py index 98a37e197f8fe..b3e0d2ccf43f5 100644 --- a/providers/tests/system/amazon/aws/example_ftp_to_s3.py +++ b/providers/tests/system/amazon/aws/example_ftp_to_s3.py @@ -72,13 +72,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_gcs_to_s3.py b/providers/tests/system/amazon/aws/example_gcs_to_s3.py index 97ed8dfda3a98..5257b440808e9 100644 --- a/providers/tests/system/amazon/aws/example_gcs_to_s3.py +++ b/providers/tests/system/amazon/aws/example_gcs_to_s3.py @@ -115,13 +115,13 @@ def upload_gcs_file(bucket_name: str, object_name: str, user_project: str): delete_gcs_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_glacier_to_gcs.py b/providers/tests/system/amazon/aws/example_glacier_to_gcs.py index 28c061ae057c9..9281d3c92e1a0 100644 --- a/providers/tests/system/amazon/aws/example_glacier_to_gcs.py +++ b/providers/tests/system/amazon/aws/example_glacier_to_gcs.py @@ -108,13 +108,13 @@ def delete_vault(vault_name): delete_vault(vault_name), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_glue.py b/providers/tests/system/amazon/aws/example_glue.py index 5314d26eff80c..3d042c79d44cb 100644 --- a/providers/tests/system/amazon/aws/example_glue.py +++ b/providers/tests/system/amazon/aws/example_glue.py @@ -217,13 +217,13 @@ def glue_cleanup(crawler_name: str, job_name: str, db_name: str) -> None: log_cleanup, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_glue_data_quality.py b/providers/tests/system/amazon/aws/example_glue_data_quality.py index 4d05d6e22b436..3c1bdb378b5dc 100644 --- a/providers/tests/system/amazon/aws/example_glue_data_quality.py +++ b/providers/tests/system/amazon/aws/example_glue_data_quality.py @@ -198,13 +198,13 @@ def delete_ruleset(ruleset_name): delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py b/providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py index fdb3cb8e63ad8..a0dd4ebba7b56 100644 --- a/providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py +++ b/providers/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py @@ -197,13 +197,13 @@ def delete_ruleset(ruleset_name): delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_glue_databrew.py b/providers/tests/system/amazon/aws/example_glue_databrew.py index 3218b465c290e..95799666f46e4 100644 --- a/providers/tests/system/amazon/aws/example_glue_databrew.py +++ b/providers/tests/system/amazon/aws/example_glue_databrew.py @@ -155,14 +155,14 @@ def delete_job(job_name: str): delete_output_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py b/providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py index 7ea7ae9d072f7..24a4eba54d59b 100644 --- a/providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py +++ b/providers/tests/system/amazon/aws/example_google_api_sheets_to_s3.py @@ -83,13 +83,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py b/providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py index ae4f721567cec..3d5ff917bfc0a 100644 --- a/providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py +++ b/providers/tests/system/amazon/aws/example_google_api_youtube_to_s3.py @@ -197,13 +197,13 @@ def transform_video_ids(**kwargs): delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_hive_to_dynamodb.py b/providers/tests/system/amazon/aws/example_hive_to_dynamodb.py index 5582c13b855f5..0c0978361bab7 100644 --- a/providers/tests/system/amazon/aws/example_hive_to_dynamodb.py +++ b/providers/tests/system/amazon/aws/example_hive_to_dynamodb.py @@ -150,13 +150,13 @@ def configure_hive_connection(connection_id, hostname): delete_dynamodb_table(dynamodb_table_name), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_http_to_s3.py b/providers/tests/system/amazon/aws/example_http_to_s3.py index cfbc131ee5689..612d68ef36e0c 100644 --- a/providers/tests/system/amazon/aws/example_http_to_s3.py +++ b/providers/tests/system/amazon/aws/example_http_to_s3.py @@ -119,13 +119,13 @@ def create_connection(conn_id_name: str): delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py b/providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py index b43023115cf4d..c376778b0df3a 100644 --- a/providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py +++ b/providers/tests/system/amazon/aws/example_imap_attachment_to_s3.py @@ -90,13 +90,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_kinesis_analytics.py b/providers/tests/system/amazon/aws/example_kinesis_analytics.py index 600710cc43b4b..b4d4566fc7600 100644 --- a/providers/tests/system/amazon/aws/example_kinesis_analytics.py +++ b/providers/tests/system/amazon/aws/example_kinesis_analytics.py @@ -263,13 +263,13 @@ def delete_kinesis_stream(stream: str, region: str): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_lambda.py b/providers/tests/system/amazon/aws/example_lambda.py index fd346a34589f0..767fae86387fd 100644 --- a/providers/tests/system/amazon/aws/example_lambda.py +++ b/providers/tests/system/amazon/aws/example_lambda.py @@ -129,13 +129,13 @@ def delete_lambda(function_name: str): log_cleanup, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_local_to_s3.py b/providers/tests/system/amazon/aws/example_local_to_s3.py index 47344c429b958..cdd9748974433 100644 --- a/providers/tests/system/amazon/aws/example_local_to_s3.py +++ b/providers/tests/system/amazon/aws/example_local_to_s3.py @@ -91,14 +91,14 @@ def delete_temp_file(): delete_temp_file(), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_mongo_to_s3.py b/providers/tests/system/amazon/aws/example_mongo_to_s3.py index 3e5bc318e5c17..6fa5f612b0681 100644 --- a/providers/tests/system/amazon/aws/example_mongo_to_s3.py +++ b/providers/tests/system/amazon/aws/example_mongo_to_s3.py @@ -83,13 +83,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_neptune.py b/providers/tests/system/amazon/aws/example_neptune.py index 4685840b42594..7682a65b65d6d 100644 --- a/providers/tests/system/amazon/aws/example_neptune.py +++ b/providers/tests/system/amazon/aws/example_neptune.py @@ -77,13 +77,13 @@ def delete_cluster(cluster_id): # TEST TEARDOWN delete_cluster(cluster_id), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_quicksight.py b/providers/tests/system/amazon/aws/example_quicksight.py index ec5cd38003148..eb53ffe60e012 100644 --- a/providers/tests/system/amazon/aws/example_quicksight.py +++ b/providers/tests/system/amazon/aws/example_quicksight.py @@ -218,13 +218,13 @@ def delete_ingestion(aws_account_id: str, dataset_name: str, ingestion_name: str delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_rds_event.py b/providers/tests/system/amazon/aws/example_rds_event.py index 79d70c9d394e2..58ab603326b96 100644 --- a/providers/tests/system/amazon/aws/example_rds_event.py +++ b/providers/tests/system/amazon/aws/example_rds_event.py @@ -119,14 +119,14 @@ def delete_sns_topic(topic_arn) -> None: delete_sns_topic(sns_topic), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_rds_export.py b/providers/tests/system/amazon/aws/example_rds_export.py index 34638f030f506..756105d802740 100644 --- a/providers/tests/system/amazon/aws/example_rds_export.py +++ b/providers/tests/system/amazon/aws/example_rds_export.py @@ -178,13 +178,13 @@ def get_snapshot_arn(snapshot_name: str) -> str: delete_db_instance, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_rds_instance.py b/providers/tests/system/amazon/aws/example_rds_instance.py index ce49f73c1f4ae..b2a9e7482e7d0 100644 --- a/providers/tests/system/amazon/aws/example_rds_instance.py +++ b/providers/tests/system/amazon/aws/example_rds_instance.py @@ -111,14 +111,14 @@ delete_db_instance, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_rds_snapshot.py b/providers/tests/system/amazon/aws/example_rds_snapshot.py index c58c1db54d539..b6b0c468e589b 100644 --- a/providers/tests/system/amazon/aws/example_rds_snapshot.py +++ b/providers/tests/system/amazon/aws/example_rds_snapshot.py @@ -138,14 +138,14 @@ delete_db_instance, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_redshift.py b/providers/tests/system/amazon/aws/example_redshift.py index 986bce5a07b52..146a20da451ba 100644 --- a/providers/tests/system/amazon/aws/example_redshift.py +++ b/providers/tests/system/amazon/aws/example_redshift.py @@ -248,13 +248,13 @@ delete_cluster_snapshot, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_redshift_s3_transfers.py b/providers/tests/system/amazon/aws/example_redshift_s3_transfers.py index 2a553eae8cb60..536f9b97157dc 100644 --- a/providers/tests/system/amazon/aws/example_redshift_s3_transfers.py +++ b/providers/tests/system/amazon/aws/example_redshift_s3_transfers.py @@ -318,13 +318,13 @@ def _insert_data(table_name: str) -> str: delete_cluster, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_s3.py b/providers/tests/system/amazon/aws/example_s3.py index d016076022874..a9c93fdd5bebe 100644 --- a/providers/tests/system/amazon/aws/example_s3.py +++ b/providers/tests/system/amazon/aws/example_s3.py @@ -323,13 +323,13 @@ def check_fn(files: list, **kwargs) -> bool: delete_bucket, delete_bucket_2, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_s3_to_dynamodb.py b/providers/tests/system/amazon/aws/example_s3_to_dynamodb.py index efd762cfd93c1..4bc4fa6b9393e 100644 --- a/providers/tests/system/amazon/aws/example_s3_to_dynamodb.py +++ b/providers/tests/system/amazon/aws/example_s3_to_dynamodb.py @@ -181,13 +181,13 @@ def delete_dynamodb_table(table_name: str): delete_new_table, delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_s3_to_ftp.py b/providers/tests/system/amazon/aws/example_s3_to_ftp.py index 0ffb4722ba867..4dd134f95c3f8 100644 --- a/providers/tests/system/amazon/aws/example_s3_to_ftp.py +++ b/providers/tests/system/amazon/aws/example_s3_to_ftp.py @@ -71,13 +71,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_s3_to_sftp.py b/providers/tests/system/amazon/aws/example_s3_to_sftp.py index 5270d3ec7d4b9..dc7d010628679 100644 --- a/providers/tests/system/amazon/aws/example_s3_to_sftp.py +++ b/providers/tests/system/amazon/aws/example_s3_to_sftp.py @@ -71,13 +71,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_s3_to_sql.py b/providers/tests/system/amazon/aws/example_s3_to_sql.py index e7e8a5e9543cf..d7f452bfeb087 100644 --- a/providers/tests/system/amazon/aws/example_s3_to_sql.py +++ b/providers/tests/system/amazon/aws/example_s3_to_sql.py @@ -18,6 +18,8 @@ from datetime import datetime +from tests_common.test_utils.watcher import watcher + from airflow import settings from airflow.decorators import task from airflow.models import Connection @@ -40,7 +42,6 @@ from airflow.providers.common.sql.operators.sql import SQLTableCheckOperator from airflow.utils.trigger_rule import TriggerRule -from dev.tests_common.test_utils.watcher import watcher from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder # Externally fetched variables: @@ -254,7 +255,7 @@ def parse_csv_to_generator(filepath): list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sagemaker.py b/providers/tests/system/amazon/aws/example_sagemaker.py index acb05400b951e..96e9756659975 100644 --- a/providers/tests/system/amazon/aws/example_sagemaker.py +++ b/providers/tests/system/amazon/aws/example_sagemaker.py @@ -646,13 +646,13 @@ def delete_docker_image(image_name): log_cleanup, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sagemaker_endpoint.py b/providers/tests/system/amazon/aws/example_sagemaker_endpoint.py index fecaa92264b6e..165e5d5edd444 100644 --- a/providers/tests/system/amazon/aws/example_sagemaker_endpoint.py +++ b/providers/tests/system/amazon/aws/example_sagemaker_endpoint.py @@ -291,13 +291,13 @@ def set_up(env_id, role_arn, ti=None): archive_logs(f"/aws/sagemaker/Endpoints/{test_setup['endpoint_name']}"), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sagemaker_notebook.py b/providers/tests/system/amazon/aws/example_sagemaker_notebook.py index 76d35dc2b1479..e989be7a92e5f 100644 --- a/providers/tests/system/amazon/aws/example_sagemaker_notebook.py +++ b/providers/tests/system/amazon/aws/example_sagemaker_notebook.py @@ -97,13 +97,13 @@ delete_instance, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sagemaker_pipeline.py b/providers/tests/system/amazon/aws/example_sagemaker_pipeline.py index 614fb5c13cc9e..9c0c9d7284837 100644 --- a/providers/tests/system/amazon/aws/example_sagemaker_pipeline.py +++ b/providers/tests/system/amazon/aws/example_sagemaker_pipeline.py @@ -117,13 +117,13 @@ def delete_pipeline(name: str): delete_experiments([pipeline_name]), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_salesforce_to_s3.py b/providers/tests/system/amazon/aws/example_salesforce_to_s3.py index 7ee15f7030beb..a533c8c3fbbf3 100644 --- a/providers/tests/system/amazon/aws/example_salesforce_to_s3.py +++ b/providers/tests/system/amazon/aws/example_salesforce_to_s3.py @@ -78,13 +78,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sftp_to_s3.py b/providers/tests/system/amazon/aws/example_sftp_to_s3.py index 85dac988cb82e..765a68c9946f5 100644 --- a/providers/tests/system/amazon/aws/example_sftp_to_s3.py +++ b/providers/tests/system/amazon/aws/example_sftp_to_s3.py @@ -71,13 +71,13 @@ delete_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sns.py b/providers/tests/system/amazon/aws/example_sns.py index d00227536da0b..06756a244ae9d 100644 --- a/providers/tests/system/amazon/aws/example_sns.py +++ b/providers/tests/system/amazon/aws/example_sns.py @@ -75,14 +75,14 @@ def delete_topic(topic_arn) -> None: delete_topic(create_sns_topic), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sql_to_s3.py b/providers/tests/system/amazon/aws/example_sql_to_s3.py index dd333073a0431..4cb8a2e7b9be2 100644 --- a/providers/tests/system/amazon/aws/example_sql_to_s3.py +++ b/providers/tests/system/amazon/aws/example_sql_to_s3.py @@ -199,14 +199,14 @@ def create_connection(conn_id_name: str, cluster_id: str): delete_cluster, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_sqs.py b/providers/tests/system/amazon/aws/example_sqs.py index 2aba6a580c4d7..f14cf3b7a713c 100644 --- a/providers/tests/system/amazon/aws/example_sqs.py +++ b/providers/tests/system/amazon/aws/example_sqs.py @@ -100,14 +100,14 @@ def delete_queue(queue_url): delete_queue(sqs_queue), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/example_step_functions.py b/providers/tests/system/amazon/aws/example_step_functions.py index beeb12528c7ac..eb3a399c69a93 100644 --- a/providers/tests/system/amazon/aws/example_step_functions.py +++ b/providers/tests/system/amazon/aws/example_step_functions.py @@ -112,13 +112,13 @@ def delete_state_machine(state_machine_arn): delete_state_machine(state_machine_arn), ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py b/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py index 774aec21b2b6b..2656474cbbf51 100644 --- a/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py +++ b/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py @@ -21,11 +21,11 @@ import boto3 import pytest +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response from airflow.www import app as application -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.www import check_content_in_response from providers.tests.system.amazon.aws.utils import set_env_id SAML_METADATA_URL = "/saml/metadata" diff --git a/providers/tests/system/apache/beam/example_beam.py b/providers/tests/system/apache/beam/example_beam.py index 4166caa3477a4..b90971149bac8 100644 --- a/providers/tests/system/apache/beam/example_beam.py +++ b/providers/tests/system/apache/beam/example_beam.py @@ -61,7 +61,7 @@ # [END howto_operator_start_java_direct_runner_pipeline] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_beam_java_flink.py b/providers/tests/system/apache/beam/example_beam_java_flink.py index a68637d8ef8d9..8f639b0829f8d 100644 --- a/providers/tests/system/apache/beam/example_beam_java_flink.py +++ b/providers/tests/system/apache/beam/example_beam_java_flink.py @@ -60,7 +60,7 @@ jar_to_local_flink_runner >> start_java_pipeline_flink_runner -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_beam_java_spark.py b/providers/tests/system/apache/beam/example_beam_java_spark.py index eb3fea6d16dab..b45ce5bbba642 100644 --- a/providers/tests/system/apache/beam/example_beam_java_spark.py +++ b/providers/tests/system/apache/beam/example_beam_java_spark.py @@ -60,7 +60,7 @@ jar_to_local_spark_runner >> start_java_pipeline_spark_runner -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_go.py b/providers/tests/system/apache/beam/example_go.py index 792eee088bf6d..ecdd8a20ed370 100644 --- a/providers/tests/system/apache/beam/example_go.py +++ b/providers/tests/system/apache/beam/example_go.py @@ -103,7 +103,7 @@ ) -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_go_dataflow.py b/providers/tests/system/apache/beam/example_go_dataflow.py index 35a63cb56e1e7..e1904ea13050b 100644 --- a/providers/tests/system/apache/beam/example_go_dataflow.py +++ b/providers/tests/system/apache/beam/example_go_dataflow.py @@ -76,7 +76,7 @@ # [END howto_operator_start_go_dataflow_runner_pipeline_async_gcs_file] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_java_dataflow.py b/providers/tests/system/apache/beam/example_java_dataflow.py index 334142dfd315c..d01d52fe658eb 100644 --- a/providers/tests/system/apache/beam/example_java_dataflow.py +++ b/providers/tests/system/apache/beam/example_java_dataflow.py @@ -66,7 +66,7 @@ # [END howto_operator_start_java_dataflow_runner_pipeline] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_python.py b/providers/tests/system/apache/beam/example_python.py index d685cb33d2dad..4b4b9c2c05b36 100644 --- a/providers/tests/system/apache/beam/example_python.py +++ b/providers/tests/system/apache/beam/example_python.py @@ -120,7 +120,7 @@ ) -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_python_async.py b/providers/tests/system/apache/beam/example_python_async.py index 8465278b6bf76..0419a13cced23 100644 --- a/providers/tests/system/apache/beam/example_python_async.py +++ b/providers/tests/system/apache/beam/example_python_async.py @@ -129,7 +129,7 @@ ) -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/beam/example_python_dataflow.py b/providers/tests/system/apache/beam/example_python_dataflow.py index 36d6b3b8562e3..8d1295ed29a0e 100644 --- a/providers/tests/system/apache/beam/example_python_dataflow.py +++ b/providers/tests/system/apache/beam/example_python_dataflow.py @@ -79,7 +79,7 @@ # [END howto_operator_start_python_dataflow_runner_pipeline_async_gcs_file] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/cassandra/example_cassandra_dag.py b/providers/tests/system/apache/cassandra/example_cassandra_dag.py index 0b25fa2975dbd..5c1fe9b5271c5 100644 --- a/providers/tests/system/apache/cassandra/example_cassandra_dag.py +++ b/providers/tests/system/apache/cassandra/example_cassandra_dag.py @@ -50,7 +50,7 @@ record_sensor = CassandraRecordSensor(task_id="cassandra_record_sensor", keys={"p1": "v1", "p2": "v2"}) # [END howto_operator_cassandra_sensors] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/drill/example_drill_dag.py b/providers/tests/system/apache/drill/example_drill_dag.py index fdaf376d5a162..7386f3b59cb1a 100644 --- a/providers/tests/system/apache/drill/example_drill_dag.py +++ b/providers/tests/system/apache/drill/example_drill_dag.py @@ -47,7 +47,7 @@ ) # [END howto_operator_drill] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/druid/example_druid_dag.py b/providers/tests/system/apache/druid/example_druid_dag.py index 5e1bfc446bb09..eac9af6d19c57 100644 --- a/providers/tests/system/apache/druid/example_druid_dag.py +++ b/providers/tests/system/apache/druid/example_druid_dag.py @@ -55,7 +55,7 @@ """ # [END howto_operator_druid_submit] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/hive/example_twitter_dag.py b/providers/tests/system/apache/hive/example_twitter_dag.py index 129a8ebf9f77f..c247bf9033020 100644 --- a/providers/tests/system/apache/hive/example_twitter_dag.py +++ b/providers/tests/system/apache/hive/example_twitter_dag.py @@ -154,13 +154,13 @@ def transfer_to_db(): analyze >> load_to_hdfs >> load_to_hive >> hive_to_mysql - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/iceberg/example_iceberg.py b/providers/tests/system/apache/iceberg/example_iceberg.py index a4708805b6788..3c9f59f1ea14d 100644 --- a/providers/tests/system/apache/iceberg/example_iceberg.py +++ b/providers/tests/system/apache/iceberg/example_iceberg.py @@ -49,7 +49,7 @@ ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/kafka/example_dag_event_listener.py b/providers/tests/system/apache/kafka/example_dag_event_listener.py index eb6866735e830..734cb9c49554c 100644 --- a/providers/tests/system/apache/kafka/example_dag_event_listener.py +++ b/providers/tests/system/apache/kafka/example_dag_event_listener.py @@ -120,7 +120,7 @@ def wait_for_event(message, **context): t0 >> t1 -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/kafka/example_dag_hello_kafka.py b/providers/tests/system/apache/kafka/example_dag_hello_kafka.py index 5e70d7324a1a0..88fef458ea261 100644 --- a/providers/tests/system/apache/kafka/example_dag_hello_kafka.py +++ b/providers/tests/system/apache/kafka/example_dag_hello_kafka.py @@ -240,7 +240,7 @@ def hello_kafka(): t0 >> t3 >> [t4, t4b] >> t5 >> t6 -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/kylin/example_kylin_dag.py b/providers/tests/system/apache/kylin/example_kylin_dag.py index 1bfec2cd1a7dd..e4de4105e05b4 100644 --- a/providers/tests/system/apache/kylin/example_kylin_dag.py +++ b/providers/tests/system/apache/kylin/example_kylin_dag.py @@ -109,13 +109,13 @@ def gen_build_time(): # gen_build_time >> refresh_task1 # gen_build_time >> merge_task # gen_build_time >> build_task3 - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/livy/example_livy.py b/providers/tests/system/apache/livy/example_livy.py index 09f8f7e244e02..8f3ea04a00867 100644 --- a/providers/tests/system/apache/livy/example_livy.py +++ b/providers/tests/system/apache/livy/example_livy.py @@ -73,13 +73,13 @@ livy_java_task_deferrable >> livy_python_task_deferrable # [END create_livy_deferrable] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/pig/example_pig.py b/providers/tests/system/apache/pig/example_pig.py index bb556a85111d1..4d47bf5e6570c 100644 --- a/providers/tests/system/apache/pig/example_pig.py +++ b/providers/tests/system/apache/pig/example_pig.py @@ -44,7 +44,7 @@ # [END create_pig] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/pinot/example_pinot_dag.py b/providers/tests/system/apache/pinot/example_pinot_dag.py index d95022f366910..825c5070e2944 100644 --- a/providers/tests/system/apache/pinot/example_pinot_dag.py +++ b/providers/tests/system/apache/pinot/example_pinot_dag.py @@ -52,7 +52,7 @@ def pinot_dbi_api(): pinot_admin() pinot_dbi_api() -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/spark/example_pyspark.py b/providers/tests/system/apache/spark/example_pyspark.py index 0ca14a76f48f9..d2d8ef6983f04 100644 --- a/providers/tests/system/apache/spark/example_pyspark.py +++ b/providers/tests/system/apache/spark/example_pyspark.py @@ -70,7 +70,7 @@ def print_df(df: pd.DataFrame): dag = example_pyspark() # type: ignore -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/apache/spark/example_spark_dag.py b/providers/tests/system/apache/spark/example_spark_dag.py index b68fc2cf761a9..6680450ae987a 100644 --- a/providers/tests/system/apache/spark/example_spark_dag.py +++ b/providers/tests/system/apache/spark/example_spark_dag.py @@ -75,7 +75,7 @@ ) # [END howto_operator_spark_sql] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/asana/example_asana.py b/providers/tests/system/asana/example_asana.py index ca68b6cc3dd2e..a7f4943beda41 100644 --- a/providers/tests/system/asana/example_asana.py +++ b/providers/tests/system/asana/example_asana.py @@ -100,13 +100,13 @@ create >> find >> update >> delete # [END asana_example_dag] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes.py b/providers/tests/system/cncf/kubernetes/example_kubernetes.py index bed3b9da746b9..dc74895a3e0fd 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes.py @@ -167,13 +167,13 @@ write_xcom >> pod_task_xcom_result # [END howto_operator_k8s_write_xcom] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py index 43eb0a1f919cf..7eb08442be3ba 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py @@ -198,13 +198,13 @@ write_xcom_async >> pod_task_xcom_result_async # [END howto_operator_k8s_write_xcom_async] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py index 088a3fdae2ede..6784b22f3a141 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py @@ -63,7 +63,7 @@ def print_pattern(): # [END howto_operator_kubernetes] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_job.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_job.py index 1a2432c135898..c81ff48e792d4 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes_job.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_job.py @@ -94,13 +94,13 @@ k8s_job >> update_job >> delete_job_task k8s_job_def >> delete_job_task_def - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py b/providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py index 1c24a533a85f0..e7f6a4d42d32e 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py +++ b/providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py @@ -72,13 +72,13 @@ t1 >> t2 - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py b/providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py index 072cffeb90f04..b244a9a800bab 100644 --- a/providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py +++ b/providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py @@ -76,13 +76,13 @@ t1 >> t2 # [END SparkKubernetesOperator_DAG] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/cohere/example_cohere_embedding_operator.py b/providers/tests/system/cohere/example_cohere_embedding_operator.py index 9686724f2cea9..d8907260d3fd7 100644 --- a/providers/tests/system/cohere/example_cohere_embedding_operator.py +++ b/providers/tests/system/cohere/example_cohere_embedding_operator.py @@ -34,7 +34,7 @@ # [END howto_operator_cohere_embedding] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/common/io/example_file_transfer_local_to_s3.py b/providers/tests/system/common/io/example_file_transfer_local_to_s3.py index 0b90a75f2e6e2..f1522d5f2611b 100644 --- a/providers/tests/system/common/io/example_file_transfer_local_to_s3.py +++ b/providers/tests/system/common/io/example_file_transfer_local_to_s3.py @@ -76,12 +76,12 @@ def remove_bucket(): temp_file >> transfer >> remove_bucket() >> delete_temp_file(temp_file_path) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/common/sql/example_sql_column_table_check.py b/providers/tests/system/common/sql/example_sql_column_table_check.py index c48dc60bfc639..4033c12faf43b 100644 --- a/providers/tests/system/common/sql/example_sql_column_table_check.py +++ b/providers/tests/system/common/sql/example_sql_column_table_check.py @@ -79,7 +79,7 @@ column_check >> row_count_check -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/common/sql/example_sql_execute_query.py b/providers/tests/system/common/sql/example_sql_execute_query.py index 8dced19cef719..535798046305b 100644 --- a/providers/tests/system/common/sql/example_sql_execute_query.py +++ b/providers/tests/system/common/sql/example_sql_execute_query.py @@ -56,7 +56,7 @@ # [END howto_operator_sql_execute_query] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/databricks/example_databricks.py b/providers/tests/system/databricks/example_databricks.py index 00d9969bd99b3..999cebb674292 100644 --- a/providers/tests/system/databricks/example_databricks.py +++ b/providers/tests/system/databricks/example_databricks.py @@ -238,13 +238,13 @@ ) # [END howto_operator_databricks_task_sql] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/databricks/example_databricks_repos.py b/providers/tests/system/databricks/example_databricks_repos.py index d07226e76ed01..33b0d1266f477 100644 --- a/providers/tests/system/databricks/example_databricks_repos.py +++ b/providers/tests/system/databricks/example_databricks_repos.py @@ -78,13 +78,13 @@ (create_repo >> update_repo >> notebook_task >> delete_repo) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/databricks/example_databricks_sensors.py b/providers/tests/system/databricks/example_databricks_sensors.py index ea233a79395bc..cf676183e6c9b 100644 --- a/providers/tests/system/databricks/example_databricks_sensors.py +++ b/providers/tests/system/databricks/example_databricks_sensors.py @@ -88,7 +88,7 @@ # runs, else all the subsequent DAG tasks and the DAG are marked as failed. (sql_sensor >> partition_sensor) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This example does not need a watcher in order to properly mark success/failure # since it is a single task, but it is given here as an example for users to @@ -96,7 +96,7 @@ # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/databricks/example_databricks_sql.py b/providers/tests/system/databricks/example_databricks_sql.py index f08ce3cd56b65..79c2bfa2fca9a 100644 --- a/providers/tests/system/databricks/example_databricks_sql.py +++ b/providers/tests/system/databricks/example_databricks_sql.py @@ -113,13 +113,13 @@ (create >> create_file >> import_csv >> select >> select_into_file) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/databricks/example_databricks_workflow.py b/providers/tests/system/databricks/example_databricks_workflow.py index 3a6c829520d89..5921f109d0843 100644 --- a/providers/tests/system/databricks/example_databricks_workflow.py +++ b/providers/tests/system/databricks/example_databricks_workflow.py @@ -143,13 +143,13 @@ notebook_1 >> notebook_2 >> task_operator_nb_1 >> sql_query # [END howto_databricks_workflow_notebook] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/dbt/cloud/example_dbt_cloud.py b/providers/tests/system/dbt/cloud/example_dbt_cloud.py index 17f8a59efea02..7a9a58a87a67b 100644 --- a/providers/tests/system/dbt/cloud/example_dbt_cloud.py +++ b/providers/tests/system/dbt/cloud/example_dbt_cloud.py @@ -18,6 +18,8 @@ from datetime import datetime +from tests_common.test_utils.system_tests import get_test_env_id + from airflow.models import DAG from airflow.operators.empty import EmptyOperator from airflow.providers.dbt.cloud.operators.dbt import ( @@ -28,8 +30,6 @@ from airflow.providers.dbt.cloud.sensors.dbt import DbtCloudJobRunSensor from airflow.utils.edgemodifier import Label -from dev.tests_common.test_utils.system_tests import get_test_env_id - ENV_ID = get_test_env_id() DAG_ID = "example_dbt_cloud" @@ -92,13 +92,13 @@ # trigger_job_run2 >> job_run_sensor # trigger_job_run2 >> job_run_sensor_deferred - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/dingding/example_dingding.py b/providers/tests/system/dingding/example_dingding.py index 4ecb41c68cc91..536f28fcc0d41 100644 --- a/providers/tests/system/dingding/example_dingding.py +++ b/providers/tests/system/dingding/example_dingding.py @@ -199,13 +199,13 @@ def failure_callback(context): >> msg_failure_callback ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/docker/example_docker.py b/providers/tests/system/docker/example_docker.py index 108813236ac62..14fa3fa3203c7 100644 --- a/providers/tests/system/docker/example_docker.py +++ b/providers/tests/system/docker/example_docker.py @@ -57,7 +57,7 @@ t1 >> [t2, t3] >> t4 ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/docker/example_docker_copy_data.py b/providers/tests/system/docker/example_docker_copy_data.py index c270408390cfa..584b16b8e096b 100644 --- a/providers/tests/system/docker/example_docker_copy_data.py +++ b/providers/tests/system/docker/example_docker_copy_data.py @@ -103,7 +103,7 @@ t_is_data_available >> t_move >> t_print ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/docker/example_docker_swarm.py b/providers/tests/system/docker/example_docker_swarm.py index 3a68dcea81673..a45d1bba351ca 100644 --- a/providers/tests/system/docker/example_docker_swarm.py +++ b/providers/tests/system/docker/example_docker_swarm.py @@ -47,7 +47,7 @@ t1 ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py b/providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py index 3c8b528606e02..b026d9e950fc7 100644 --- a/providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py +++ b/providers/tests/system/docker/example_taskflow_api_docker_virtualenv.py @@ -116,9 +116,15 @@ def load(total_order_value: float): tutorial_dag = tutorial_taskflow_api_docker_virtualenv() # [END dag_invocation] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) # [END tutorial] + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/providers/tests/system/elasticsearch/example_elasticsearch_query.py b/providers/tests/system/elasticsearch/example_elasticsearch_query.py index 93471d874c3c9..f4450b8e335cf 100644 --- a/providers/tests/system/elasticsearch/example_elasticsearch_query.py +++ b/providers/tests/system/elasticsearch/example_elasticsearch_query.py @@ -81,7 +81,7 @@ def use_elasticsearch_hook(): task_id="print_data_from_elasticsearch", python_callable=use_elasticsearch_hook ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/ftp/example_ftp.py b/providers/tests/system/ftp/example_ftp.py index 8fb2e38806ae4..5b9bfe65db4fa 100644 --- a/providers/tests/system/ftp/example_ftp.py +++ b/providers/tests/system/ftp/example_ftp.py @@ -87,13 +87,13 @@ ftp_put >> ftp_get ftps_put >> ftps_get - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/github/example_github.py b/providers/tests/system/github/example_github.py index 70eb8bf902f04..b076647177cc9 100644 --- a/providers/tests/system/github/example_github.py +++ b/providers/tests/system/github/example_github.py @@ -100,7 +100,7 @@ def tag_checker(repo: Any, tag_name: str) -> bool | None: # [END howto_operator_list_tags_github] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/ads/example_ads.py b/providers/tests/system/google/ads/example_ads.py index 162fd45334f67..f55c7306f33aa 100644 --- a/providers/tests/system/google/ads/example_ads.py +++ b/providers/tests/system/google/ads/example_ads.py @@ -117,14 +117,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/automl/example_automl_dataset.py b/providers/tests/system/google/cloud/automl/example_automl_dataset.py index 49de111ffb358..d1305da4fb081 100644 --- a/providers/tests/system/google/cloud/automl/example_automl_dataset.py +++ b/providers/tests/system/google/cloud/automl/example_automl_dataset.py @@ -162,14 +162,14 @@ def upload_updated_csv_file_to_gcs(): >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/automl/example_automl_translation.py b/providers/tests/system/google/cloud/automl/example_automl_translation.py index 60dbf782f281e..e758bdf113e48 100644 --- a/providers/tests/system/google/cloud/automl/example_automl_translation.py +++ b/providers/tests/system/google/cloud/automl/example_automl_translation.py @@ -188,14 +188,14 @@ def upload_csv_file_to_gcs(): >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/automl/example_automl_video_classification.py b/providers/tests/system/google/cloud/automl/example_automl_video_classification.py index 36853c3826037..538831d307ca5 100644 --- a/providers/tests/system/google/cloud/automl/example_automl_video_classification.py +++ b/providers/tests/system/google/cloud/automl/example_automl_video_classification.py @@ -158,13 +158,13 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/automl/example_automl_video_tracking.py b/providers/tests/system/google/cloud/automl/example_automl_video_tracking.py index 65718f10b7697..3876e9b0a39de 100644 --- a/providers/tests/system/google/cloud/automl/example_automl_video_tracking.py +++ b/providers/tests/system/google/cloud/automl/example_automl_video_tracking.py @@ -158,13 +158,13 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/automl/example_automl_vision_classification.py b/providers/tests/system/google/cloud/automl/example_automl_vision_classification.py index 40e68d3291ce8..e329febf5d9fc 100644 --- a/providers/tests/system/google/cloud/automl/example_automl_vision_classification.py +++ b/providers/tests/system/google/cloud/automl/example_automl_vision_classification.py @@ -132,13 +132,13 @@ >> delete_image_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/automl/example_automl_vision_object_detection.py b/providers/tests/system/google/cloud/automl/example_automl_vision_object_detection.py index 334e154cea4d1..b87b42008d943 100644 --- a/providers/tests/system/google/cloud/automl/example_automl_vision_object_detection.py +++ b/providers/tests/system/google/cloud/automl/example_automl_vision_object_detection.py @@ -134,13 +134,13 @@ >> delete_image_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py b/providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py index 837cb99e5ffc1..e915642b70204 100644 --- a/providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py +++ b/providers/tests/system/google/cloud/azure/example_azure_blob_to_gcs.py @@ -59,13 +59,13 @@ (wait_for_blob >> transfer_files_to_gcs) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py b/providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py index a897b6dc7d3d1..45172fa2070db 100644 --- a/providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py +++ b/providers/tests/system/google/cloud/azure/example_azure_fileshare_to_gcs.py @@ -81,14 +81,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py index 87e13a2c25165..7be37b8631f3b 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_dataset.py @@ -85,14 +85,14 @@ >> delete_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py index b387563069a94..37677a1514ac0 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_dts.py @@ -180,14 +180,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py index 5e487f5a93e2a..308c4337684da 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations.py @@ -97,14 +97,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_operations_location.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations_location.py index 2b026070a06b4..d7547e2b810de 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_operations_location.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_operations_location.py @@ -77,14 +77,14 @@ >> delete_dataset_with_location ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py index cd31ea05bcdcb..0e4d930442924 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py @@ -247,7 +247,7 @@ execute_insert_query >> [check_count, check_value, check_interval] >> delete_dataset execute_insert_query >> [column_check, table_check] >> delete_dataset - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG @@ -257,7 +257,7 @@ globals()[DAG_ID] = dag for dag in DAGS_LIST: - from dev.tests_common.test_utils.system_tests import get_test_run + from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py index 012eba080d1a4..104f09ea3335b 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_queries_async.py @@ -259,14 +259,14 @@ insert_query_job >> execute_long_running_query >> check_value >> check_interval [check_count, check_interval, bigquery_execute_multi_query, get_data_result] >> delete_dataset - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py index aa8d36f5ed893..760275f904c95 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_sensors.py @@ -165,14 +165,14 @@ >> delete_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py index 9c1432a0a8da5..588e33d368d19 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_tables.py @@ -224,14 +224,14 @@ >> delete_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py index cb1fa63498da9..879d8aa3fb284 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_bigquery.py @@ -100,14 +100,14 @@ >> delete_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py index 21acb27f161db..76684a1c05c38 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs.py @@ -98,14 +98,14 @@ >> [delete_bucket, delete_dataset] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs_async.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs_async.py index fdf084601c770..2bc4846fea343 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs_async.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_gcs_async.py @@ -97,14 +97,14 @@ >> [delete_bucket, delete_dataset] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py index 51e713560bc23..3fd5e9259548e 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mssql.py @@ -326,13 +326,13 @@ def delete_connection(connection_id: str) -> None: >> delete_persistent_disk ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py index a3b726098773d..623924c2c96b2 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_mysql.py @@ -88,13 +88,13 @@ >> delete_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py index ca4e3148d0a18..54f5efc13d93e 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_to_postgres.py @@ -362,13 +362,13 @@ def delete_connection(connection_id: str) -> None: >> delete_persistent_disk ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_transfer.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_transfer.py index 73d55710c9db9..39f1eea2bd1d5 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_transfer.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_transfer.py @@ -117,13 +117,13 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigquery/example_bigquery_value_check.py b/providers/tests/system/google/cloud/bigquery/example_bigquery_value_check.py index 3872ca092321f..5a51ad310135f 100644 --- a/providers/tests/system/google/cloud/bigquery/example_bigquery_value_check.py +++ b/providers/tests/system/google/cloud/bigquery/example_bigquery_value_check.py @@ -131,8 +131,8 @@ >> delete_dataset ) - from dev.tests_common.test_utils.system_tests import get_test_run - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.system_tests import get_test_run + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG @@ -142,7 +142,7 @@ test_run = get_test_run(dag) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/bigtable/example_bigtable.py b/providers/tests/system/google/cloud/bigtable/example_bigtable.py index 77abc49fae258..b1f3469f4fcb9 100644 --- a/providers/tests/system/google/cloud/bigtable/example_bigtable.py +++ b/providers/tests/system/google/cloud/bigtable/example_bigtable.py @@ -225,14 +225,14 @@ def update_clusters_and_instance(): >> [delete_instance_task, delete_instance_task2] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py b/providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py index 84dc031a5268f..322c6cd7eceae 100644 --- a/providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py +++ b/providers/tests/system/google/cloud/cloud_batch/example_cloud_batch.py @@ -187,13 +187,13 @@ def _create_job(): ([submit1, submit2] >> list_tasks >> assert_tasks >> list_jobs >> get_name >> [delete_job1, delete_job2]) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_build/example_cloud_build.py b/providers/tests/system/google/cloud/cloud_build/example_cloud_build.py index bf2391413a1c3..319047fbaa63a 100644 --- a/providers/tests/system/google/cloud/cloud_build/example_cloud_build.py +++ b/providers/tests/system/google/cloud/cloud_build/example_cloud_build.py @@ -267,14 +267,14 @@ def no_wait_cancel_retry_get_deferrable(): ] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py b/providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py index f6873da9c5680..e799af6650d1e 100644 --- a/providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py +++ b/providers/tests/system/google/cloud/cloud_build/example_cloud_build_trigger.py @@ -182,14 +182,14 @@ def get_project_number(): >> list_build_triggers ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_functions/example_functions.py b/providers/tests/system/google/cloud/cloud_functions/example_functions.py index fd0b8ea5f5a4c..675224980469f 100644 --- a/providers/tests/system/google/cloud/cloud_functions/example_functions.py +++ b/providers/tests/system/google/cloud/cloud_functions/example_functions.py @@ -122,14 +122,14 @@ delete_function, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py index 670a850b04ac4..5d425b8b0e3cc 100644 --- a/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +++ b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py @@ -198,14 +198,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py index 3d8ebf6f287dc..cb5551f070405 100644 --- a/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +++ b/providers/tests/system/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py @@ -266,14 +266,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_run/example_cloud_run.py b/providers/tests/system/google/cloud/cloud_run/example_cloud_run.py index 08145e0336ede..368d835ddb6ad 100644 --- a/providers/tests/system/google/cloud/cloud_run/example_cloud_run.py +++ b/providers/tests/system/google/cloud/cloud_run/example_cloud_run.py @@ -367,13 +367,13 @@ def _create_job_instance_with_label(): >> (delete_job1, delete_job2, delete_job3) ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py b/providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py index 43318c4b14306..8d87bcbc10033 100644 --- a/providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py +++ b/providers/tests/system/google/cloud/cloud_run/example_cloud_run_service.py @@ -84,14 +84,14 @@ def _create_service(): >> delete_cloud_run_service ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py index 52414b2784b27..57da5f43c1d64 100644 --- a/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py +++ b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql.py @@ -294,14 +294,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py index 6cf7e0c08751c..3d97fad5aacf9 100644 --- a/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py +++ b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query.py @@ -541,13 +541,13 @@ def delete_connection(connection_id: str) -> None: # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py index db77f18339af0..69ec1784fa47c 100644 --- a/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py +++ b/providers/tests/system/google/cloud/cloud_sql/example_cloud_sql_query_ssl.py @@ -491,13 +491,13 @@ def delete_secret(ssl_secret_id, db_type: str) -> None: # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/composer/example_cloud_composer.py b/providers/tests/system/google/cloud/composer/example_cloud_composer.py index 266a7e4a444ad..f04a2c8596a1e 100644 --- a/providers/tests/system/google/cloud/composer/example_cloud_composer.py +++ b/providers/tests/system/google/cloud/composer/example_cloud_composer.py @@ -214,14 +214,14 @@ [delete_env, defer_delete_env], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/compute/example_compute.py b/providers/tests/system/google/cloud/compute/example_compute.py index 8343920890708..82f2b008c14dc 100644 --- a/providers/tests/system/google/cloud/compute/example_compute.py +++ b/providers/tests/system/google/cloud/compute/example_compute.py @@ -267,14 +267,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/compute/example_compute_igm.py b/providers/tests/system/google/cloud/compute/example_compute_igm.py index 11357452dfc69..5f292958c2308 100644 --- a/providers/tests/system/google/cloud/compute/example_compute_igm.py +++ b/providers/tests/system/google/cloud/compute/example_compute_igm.py @@ -236,14 +236,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/compute/example_compute_ssh.py b/providers/tests/system/google/cloud/compute/example_compute_ssh.py index 5b7d1523017ff..ffba97e91db15 100644 --- a/providers/tests/system/google/cloud/compute/example_compute_ssh.py +++ b/providers/tests/system/google/cloud/compute/example_compute_ssh.py @@ -138,14 +138,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/compute/example_compute_ssh_os_login.py b/providers/tests/system/google/cloud/compute/example_compute_ssh_os_login.py index 63ed278a08c5a..89a9223e7069c 100644 --- a/providers/tests/system/google/cloud/compute/example_compute_ssh_os_login.py +++ b/providers/tests/system/google/cloud/compute/example_compute_ssh_os_login.py @@ -146,14 +146,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/compute/example_compute_ssh_parallel.py b/providers/tests/system/google/cloud/compute/example_compute_ssh_parallel.py index 3ab61afc0c259..e8b8767fffb79 100644 --- a/providers/tests/system/google/cloud/compute/example_compute_ssh_parallel.py +++ b/providers/tests/system/google/cloud/compute/example_compute_ssh_parallel.py @@ -139,14 +139,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py index 75b1ad93dd479..78549f3735073 100644 --- a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_deidentify_content.py @@ -157,14 +157,14 @@ >> delete_template ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py index fb2198b16793f..72e35d3234491 100644 --- a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_info_types.py @@ -154,14 +154,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py index 22b259582f4ac..0d48b2fc19545 100644 --- a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_inspect_template.py @@ -115,14 +115,14 @@ >> delete_template ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job.py index b7dc2ac1a4053..27db1730ad0bc 100644 --- a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job.py @@ -90,14 +90,14 @@ (create_job >> list_jobs >> get_job >> cancel_job >> delete_job) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py index 902875f7b5921..11fd48e1e4e05 100644 --- a/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py +++ b/providers/tests/system/google/cloud/data_loss_prevention/example_dlp_job_trigger.py @@ -95,14 +95,14 @@ (create_trigger >> list_triggers >> get_trigger >> update_trigger >> delete_trigger) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_go.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_go.py index 57e5941e3ed6b..92e779f69d66d 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_go.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_go.py @@ -149,7 +149,7 @@ def check_autoscaling_event(autoscaling_events: list[dict]) -> bool: ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py index 3629194e6ee9b..66d0ce0995c73 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_java.py @@ -148,14 +148,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py index 229373ed1bf3f..70cd5db9e9dd4 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python.py @@ -110,14 +110,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py index 31f1cd026f19a..9bca9b02871b9 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py @@ -176,14 +176,14 @@ def check_autoscaling_event(autoscaling_events: list[dict]) -> bool: >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py index cfd5e06b029b6..800540eb4d70a 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py @@ -137,14 +137,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py index 1a6d0aec3923b..c6303d74231be 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py @@ -177,14 +177,14 @@ def check_autoscaling_event(autoscaling_events: list[dict]) -> bool: >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py index e3b0ee711a921..2ba0bf0534c59 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_sql.py @@ -137,13 +137,13 @@ >> delete_bq_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py index a15dffb3a3282..4e7a7ccebdd1b 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py @@ -114,14 +114,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_template.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_template.py index 86545514607aa..4b5bccacb0fa1 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_template.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_template.py @@ -158,14 +158,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py b/providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py index 2243ad695bdd4..14db8b8936223 100644 --- a/providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py +++ b/providers/tests/system/google/cloud/dataflow/example_dataflow_yaml.py @@ -162,13 +162,13 @@ >> delete_bq_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataform/example_dataform.py b/providers/tests/system/google/cloud/dataform/example_dataform.py index b61247a877008..e88a37caef2e6 100644 --- a/providers/tests/system/google/cloud/dataform/example_dataform.py +++ b/providers/tests/system/google/cloud/dataform/example_dataform.py @@ -328,13 +328,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/datafusion/example_datafusion.py b/providers/tests/system/google/cloud/datafusion/example_datafusion.py index a61fbbd01da91..2181d2b2badab 100644 --- a/providers/tests/system/google/cloud/datafusion/example_datafusion.py +++ b/providers/tests/system/google/cloud/datafusion/example_datafusion.py @@ -340,13 +340,13 @@ def get_artifacts_versions(ti=None): >> [delete_bucket1, delete_bucket2] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/datapipelines/example_datapipeline.py b/providers/tests/system/google/cloud/datapipelines/example_datapipeline.py index 91fb7eed26d2a..0c0c430eae150 100644 --- a/providers/tests/system/google/cloud/datapipelines/example_datapipeline.py +++ b/providers/tests/system/google/cloud/datapipelines/example_datapipeline.py @@ -138,14 +138,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataplex/example_dataplex.py b/providers/tests/system/google/cloud/dataplex/example_dataplex.py index 34e9a0fd05c19..f5ce1bbd41780 100644 --- a/providers/tests/system/google/cloud/dataplex/example_dataplex.py +++ b/providers/tests/system/google/cloud/dataplex/example_dataplex.py @@ -206,14 +206,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py b/providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py index 44f4af2435925..da447a8ced5e4 100644 --- a/providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py +++ b/providers/tests/system/google/cloud/dataplex/example_dataplex_dp.py @@ -329,14 +329,14 @@ [delete_lake, delete_dataset], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py b/providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py index d31d574755c90..1a1910757fca3 100644 --- a/providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py +++ b/providers/tests/system/google/cloud/dataplex/example_dataplex_dq.py @@ -363,14 +363,14 @@ [delete_lake, delete_dataset], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataprep/example_dataprep.py b/providers/tests/system/google/cloud/dataprep/example_dataprep.py index cdc736a41c66b..edffb69e580b4 100644 --- a/providers/tests/system/google/cloud/dataprep/example_dataprep.py +++ b/providers/tests/system/google/cloud/dataprep/example_dataprep.py @@ -307,14 +307,14 @@ def delete_connection(connection_id: str) -> None: [delete_bucket_task, delete_connection_task], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py index 54481f43ca762..c88a1c3c6803c 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch.py @@ -176,13 +176,13 @@ >> delete_batch_4 ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py index 79d3766995e3d..17e5f99b52585 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py @@ -91,14 +91,14 @@ >> delete_batch ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py index a8ed6b13de552..5bf500aeef3c0 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py @@ -141,14 +141,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py index 0eb27762c2f98..e64a0914941c9 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py @@ -121,13 +121,13 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py index 8ae262dbf624c..2f7153d44059e 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py @@ -137,14 +137,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py index 3eacc905bc703..9e6c228a9d90d 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py @@ -115,14 +115,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py index f990363c48004..0562be439b1c6 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py @@ -135,14 +135,14 @@ >> [delete_cluster, delete_bucket] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py index 2e4b698573bf2..c68bdd013bb2d 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py @@ -111,13 +111,13 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py index 7ed0061d9947c..620019cd4856b 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py @@ -120,14 +120,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_flink.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_flink.py index ce1a6fc4451e9..34bcdb17e5479 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_flink.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_flink.py @@ -124,14 +124,14 @@ >> [delete_cluster, delete_bucket] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py index bb1044da62414..8048a23283e4c 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_gke.py @@ -144,13 +144,13 @@ >> delete_gke_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py index c1f5423041fa4..e4370b2f7955c 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py @@ -123,14 +123,14 @@ >> [delete_cluster, delete_bucket] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py index 6aadceb552f61..bcddd461560c5 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_hive.py @@ -124,14 +124,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_pig.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_pig.py index 762ca2bc73fd2..848138ae54732 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_pig.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_pig.py @@ -108,14 +108,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py index f5bdf25732a5e..ea8ccb231df9d 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_presto.py @@ -115,14 +115,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py index 71e7245b7d728..fdc22d8cae6d9 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py @@ -140,14 +140,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py index e43a56d230bf4..5ff5ce8ed4ed2 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark.py @@ -111,14 +111,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py index 475e9912e8562..df3d969079255 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py @@ -121,14 +121,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py index 5e6d0b773af6f..20b68b0574b89 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py @@ -112,14 +112,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py index aef860a0e545a..3b34b8dbb298d 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py @@ -108,14 +108,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py index 7b416ade38648..39890a501b0bc 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py @@ -136,14 +136,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py index d3f7f2a1a3a4c..3f366f6088d95 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_trino.py @@ -117,14 +117,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow.py index ab465a124d5fd..eba4ca34a7f33 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow.py @@ -104,14 +104,14 @@ >> instantiate_inline_workflow_template ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py index e2319a35a124f..26dd27efcb836 100644 --- a/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py +++ b/providers/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py @@ -108,14 +108,14 @@ >> instantiate_inline_workflow_template_async ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py index ee3b5b70ad519..cbd612ba80162 100644 --- a/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py +++ b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore.py @@ -192,14 +192,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py index 12af45d210653..dddb69bfd0827 100644 --- a/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +++ b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py @@ -126,14 +126,14 @@ ) (create_service >> backup_service >> list_backups >> restore_service >> delete_backup >> delete_service) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py index 6b06b868c73e2..f3317b9ce7f19 100644 --- a/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py +++ b/providers/tests/system/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py @@ -232,14 +232,14 @@ def get_hive_warehouse_bucket(**kwargs): >> [delete_dataproc_cluster, delete_metastore_service, delete_warehouse_bucket] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/datastore/example_datastore_commit.py b/providers/tests/system/google/cloud/datastore/example_datastore_commit.py index 430d176ed733c..60ec53cf1f8a4 100644 --- a/providers/tests/system/google/cloud/datastore/example_datastore_commit.py +++ b/providers/tests/system/google/cloud/datastore/example_datastore_commit.py @@ -160,14 +160,14 @@ [delete_bucket, delete_export_operation, delete_import_operation], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/datastore/example_datastore_query.py b/providers/tests/system/google/cloud/datastore/example_datastore_query.py index 0b282a1feb34e..f6908823a8b28 100644 --- a/providers/tests/system/google/cloud/datastore/example_datastore_query.py +++ b/providers/tests/system/google/cloud/datastore/example_datastore_query.py @@ -80,14 +80,14 @@ allocate_ids >> begin_transaction_query >> run_query - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/datastore/example_datastore_rollback.py b/providers/tests/system/google/cloud/datastore/example_datastore_rollback.py index 44d0dc1709418..0634eabeb553d 100644 --- a/providers/tests/system/google/cloud/datastore/example_datastore_rollback.py +++ b/providers/tests/system/google/cloud/datastore/example_datastore_rollback.py @@ -63,14 +63,14 @@ begin_transaction_to_rollback >> rollback_transaction - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py index f44e7315e2d18..3b3cd028897df 100644 --- a/providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_calendar_to_gcs.py @@ -115,13 +115,13 @@ def delete_connection(connection_id: str) -> None: >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_firestore.py b/providers/tests/system/google/cloud/gcs/example_firestore.py index 89e0e3fe1c6d9..1860d30c0d5aa 100644 --- a/providers/tests/system/google/cloud/gcs/example_firestore.py +++ b/providers/tests/system/google/cloud/gcs/example_firestore.py @@ -170,14 +170,14 @@ >> [delete_dataset, delete_bucket] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_acl.py b/providers/tests/system/google/cloud/gcs/example_gcs_acl.py index 7843c0877c1d9..7000ac4d45551 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_acl.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_acl.py @@ -108,14 +108,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_copy_delete.py b/providers/tests/system/google/cloud/gcs/example_gcs_copy_delete.py index 6bbec540df9f2..62c31d80b286b 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_copy_delete.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_copy_delete.py @@ -122,14 +122,14 @@ [delete_bucket_src, delete_bucket_dst], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_sensor.py b/providers/tests/system/google/cloud/gcs/example_gcs_sensor.py index 2d4da2887d4e5..a07aff849e465 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_sensor.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_sensor.py @@ -201,14 +201,14 @@ def mode_setter(self, value): delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py index 5d7e393bff29e..efea506c909d1 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery.py @@ -82,13 +82,13 @@ >> delete_test_dataset ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py index 1ca531b818008..7572f2bc559d3 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_bigquery_async.py @@ -174,13 +174,13 @@ >> delete_test_dataset_delimiter ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py index ac1d70307a535..c4c812868efa3 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_gcs.py @@ -275,14 +275,14 @@ def delete_work_dir(create_workdir_result: str) -> None: [delete_bucket_src, delete_bucket_dst, delete_work_dir(create_workdir_task)], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py index bfb200a15a63d..c15e24ff3fbac 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_gdrive.py @@ -204,13 +204,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py b/providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py index 69947698cd7bb..991f8347f5da2 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_to_sheets.py @@ -131,13 +131,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_transform.py b/providers/tests/system/google/cloud/gcs/example_gcs_transform.py index 0b59119c6bc1e..421272f8b5775 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_transform.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_transform.py @@ -95,14 +95,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py b/providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py index 3a5b03695ee40..1c7e68b9d1252 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_transform_timespan.py @@ -114,14 +114,14 @@ [delete_bucket_src, delete_bucket_dst], ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py b/providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py index 6907a29fb5389..abde62d35bfaa 100644 --- a/providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py +++ b/providers/tests/system/google/cloud/gcs/example_gcs_upload_download.py @@ -92,14 +92,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py index 6ee188bd957ec..39633a8fc2bd4 100644 --- a/providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_gdrive_to_gcs.py @@ -165,13 +165,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py index bc6ac79f1e0b1..7aece3764c7f1 100644 --- a/providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_mssql_to_gcs.py @@ -78,14 +78,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py index cc53337e56934..ae3fea2c236e5 100644 --- a/providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_mysql_to_gcs.py @@ -295,13 +295,13 @@ def delete_connection(connection_id: str) -> None: mysql_to_gcs >> [delete_gcs_bucket, delete_firewall_rule, delete_gce_instance, delete_connection_task] delete_gce_instance >> delete_persistent_disk - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py index c727eef1f66fb..ba4d7025acfab 100644 --- a/providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_oracle_to_gcs.py @@ -64,13 +64,13 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py index 487dbbce552e2..59f88f78b276a 100644 --- a/providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_s3_to_gcs.py @@ -110,14 +110,14 @@ def upload_file(): >> delete_gcs_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py index 994e7e8242e1e..b2c250426528a 100644 --- a/providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_sftp_to_gcs.py @@ -119,14 +119,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_sheets.py b/providers/tests/system/google/cloud/gcs/example_sheets.py index 7d17379819168..001ebb69d6820 100644 --- a/providers/tests/system/google/cloud/gcs/example_sheets.py +++ b/providers/tests/system/google/cloud/gcs/example_sheets.py @@ -145,13 +145,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py index 08688c332563a..7c546060333c1 100644 --- a/providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_sheets_to_gcs.py @@ -120,13 +120,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py b/providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py index af92e550481f7..e76a66ea22be3 100644 --- a/providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py +++ b/providers/tests/system/google/cloud/gcs/example_trino_to_gcs.py @@ -221,13 +221,13 @@ def safe_name(s: str) -> str: >> [delete_dataset, delete_bucket] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py index e9fe3f6836b94..36d21ff701eb3 100644 --- a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine.py @@ -113,14 +113,14 @@ create_cluster >> [pod_task, pod_task_xcom] >> delete_cluster pod_task_xcom >> pod_task_xcom_result - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py index f5cb8f570754f..9283f77dbd2a9 100644 --- a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py @@ -116,14 +116,14 @@ create_cluster >> [pod_task, pod_task_xcom_async] >> delete_cluster pod_task_xcom_async >> pod_task_xcom_result - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py index a4c56c17e5baf..9ff3ef32a7c09 100644 --- a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py @@ -179,14 +179,14 @@ delete_cluster, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py index 06c23432a923a..44dbaae5537b2 100644 --- a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py @@ -179,14 +179,14 @@ >> delete_cluster ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py index 784ba994862b7..28a6ac985b6ee 100644 --- a/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py +++ b/providers/tests/system/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py @@ -101,14 +101,14 @@ create_cluster >> create_resource_task >> delete_resource_task >> delete_cluster - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/life_sciences/example_life_sciences.py b/providers/tests/system/google/cloud/life_sciences/example_life_sciences.py index 170eeb39ad1aa..e5024562b70a8 100644 --- a/providers/tests/system/google/cloud/life_sciences/example_life_sciences.py +++ b/providers/tests/system/google/cloud/life_sciences/example_life_sciences.py @@ -131,14 +131,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/ml_engine/example_mlengine.py b/providers/tests/system/google/cloud/ml_engine/example_mlengine.py index f0eaf7cb30171..a735d3015ab55 100644 --- a/providers/tests/system/google/cloud/ml_engine/example_mlengine.py +++ b/providers/tests/system/google/cloud/ml_engine/example_mlengine.py @@ -280,13 +280,13 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/natural_language/example_natural_language.py b/providers/tests/system/google/cloud/natural_language/example_natural_language.py index cc1aba0f165c7..4eb542d9aecce 100644 --- a/providers/tests/system/google/cloud/natural_language/example_natural_language.py +++ b/providers/tests/system/google/cloud/natural_language/example_natural_language.py @@ -119,13 +119,13 @@ analyze_sentiment >> analyze_sentiment_result analyze_classify_text >> analyze_classify_text_result - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/pubsub/example_pubsub.py b/providers/tests/system/google/cloud/pubsub/example_pubsub.py index 93ab1cde6e0a8..9ff909a3b401f 100644 --- a/providers/tests/system/google/cloud/pubsub/example_pubsub.py +++ b/providers/tests/system/google/cloud/pubsub/example_pubsub.py @@ -146,14 +146,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py b/providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py index a902ea5617f6d..f889ff16a286e 100644 --- a/providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py +++ b/providers/tests/system/google/cloud/pubsub/example_pubsub_deferrable.py @@ -101,14 +101,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/spanner/example_spanner.py b/providers/tests/system/google/cloud/spanner/example_spanner.py index b11a5cd61f92c..ba9f727e1f6dc 100644 --- a/providers/tests/system/google/cloud/spanner/example_spanner.py +++ b/providers/tests/system/google/cloud/spanner/example_spanner.py @@ -161,14 +161,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py b/providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py index f2382a6053aa4..16603bf65a86d 100644 --- a/providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py +++ b/providers/tests/system/google/cloud/speech_to_text/example_speech_to_text.py @@ -89,14 +89,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py b/providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py index d34531ba52a0a..65c88b38ccf30 100644 --- a/providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py +++ b/providers/tests/system/google/cloud/sql_to_sheets/example_sql_to_sheets.py @@ -315,13 +315,13 @@ def delete_connection(connection_id: str) -> None: ] delete_gce_instance >> delete_persistent_disk - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/stackdriver/example_stackdriver.py b/providers/tests/system/google/cloud/stackdriver/example_stackdriver.py index 202c9bcfbdccf..e720ac1f16594 100644 --- a/providers/tests/system/google/cloud/stackdriver/example_stackdriver.py +++ b/providers/tests/system/google/cloud/stackdriver/example_stackdriver.py @@ -228,13 +228,13 @@ delete_alert_policy_2, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py index 4d6a010954401..cf76f6eddd58b 100644 --- a/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py +++ b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_aws.py @@ -254,14 +254,14 @@ ] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py index b2cef2f831b5f..960a9c0679a9b 100644 --- a/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py +++ b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py @@ -195,13 +195,13 @@ >> [delete_transfer, delete_bucket_src, delete_bucket_dst] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py index e670eba755428..46289ea66ca23 100644 --- a/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py +++ b/providers/tests/system/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py @@ -106,13 +106,13 @@ >> [delete_bucket_src, delete_bucket_dst] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/tasks/example_queue.py b/providers/tests/system/google/cloud/tasks/example_queue.py index 9797a29d04e22..614bdfd352073 100644 --- a/providers/tests/system/google/cloud/tasks/example_queue.py +++ b/providers/tests/system/google/cloud/tasks/example_queue.py @@ -161,14 +161,14 @@ def generate_random_string(): delete_queue, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/tasks/example_tasks.py b/providers/tests/system/google/cloud/tasks/example_tasks.py index 0eae95fd1075d..f4ab2f53b0bb4 100644 --- a/providers/tests/system/google/cloud/tasks/example_tasks.py +++ b/providers/tests/system/google/cloud/tasks/example_tasks.py @@ -152,14 +152,14 @@ def generate_random_string(): random_string, create_queue, create_task, tasks_get, list_tasks, run_task, delete_task, delete_queue ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py b/providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py index 0227049508a74..437db735c2d57 100644 --- a/providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py +++ b/providers/tests/system/google/cloud/text_to_speech/example_text_to_speech.py @@ -78,14 +78,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py b/providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py index 8d3f19416313a..dd1a5945f2cc7 100644 --- a/providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py +++ b/providers/tests/system/google/cloud/transfers/example_gcs_to_sftp.py @@ -171,14 +171,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py b/providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py index abfcbabce0852..2e5c6f8e8f7eb 100644 --- a/providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py +++ b/providers/tests/system/google/cloud/transfers/example_gdrive_to_local.py @@ -167,13 +167,13 @@ def delete_connection(connection_id: str) -> None: >> [delete_bucket, delete_connection_task] ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py b/providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py index 91bde027982ec..0f9d5be7eaeec 100644 --- a/providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py +++ b/providers/tests/system/google/cloud/transfers/example_postgres_to_gcs.py @@ -290,13 +290,13 @@ def delete_connection(connection_id: str) -> None: postgres_to_gcs >> [delete_gcs_bucket, delete_firewall_rule, delete_gce_instance, delete_connection_task] delete_gce_instance >> delete_persistent_disk - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/translate/example_translate.py b/providers/tests/system/google/cloud/translate/example_translate.py index 13a3ee061e308..44af8bf5d1d12 100644 --- a/providers/tests/system/google/cloud/translate/example_translate.py +++ b/providers/tests/system/google/cloud/translate/example_translate.py @@ -56,14 +56,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/translate_speech/example_translate_speech.py b/providers/tests/system/google/cloud/translate_speech/example_translate_speech.py index 4fe34d0b86f57..2e023d68ae537 100644 --- a/providers/tests/system/google/cloud/translate_speech/example_translate_speech.py +++ b/providers/tests/system/google/cloud/translate_speech/example_translate_speech.py @@ -110,14 +110,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py index 8dd0ecc9c9550..e22622dd58605 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_forecasting_training.py @@ -150,13 +150,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py index bf96da992aca1..77629b0bc78c1 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_image_training.py @@ -135,13 +135,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py index ca6c65e1a6ddd..cd75035793af0 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_list_training.py @@ -52,13 +52,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py index fb32c7475c7f1..106e622abad5e 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_tabular_training.py @@ -140,13 +140,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py index 6c3db89382f24..ea68e55fa2605 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py @@ -145,13 +145,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py index 78bb9ffa6bad4..06a56f379654f 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_batch_prediction_job.py @@ -231,13 +231,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py index 1295d20983eeb..316e0d3a7cbbe 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_container.py @@ -208,13 +208,13 @@ def TABULAR_DATASET(bucket_name): # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py index 3fa6169cb2759..eb2a080a92a1f 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job.py @@ -250,13 +250,13 @@ def TABULAR_DATASET(bucket_name): # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py index fbc40b888216b..f1b471d446262 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py @@ -211,13 +211,13 @@ def TABULAR_DATASET(bucket_name): # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py index 3e1b98a2232d1..af4fbb88c2b90 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_dataset.py @@ -269,13 +269,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py index 12b1181a79280..13e24b869c79e 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_endpoint.py @@ -206,13 +206,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py index 7e696184eef41..a1140afc1983f 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py @@ -155,14 +155,14 @@ ) # [END how_to_cloud_vertex_ai_run_evaluation_operator] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py index affba0b6aede8..9ce1c77350e67 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model_tuning.py @@ -56,13 +56,13 @@ ) # [END how_to_cloud_vertex_ai_supervised_fine_tuning_train_operator] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py index 55254741d9cc2..4fabdbc6ac67b 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_hyperparameter_tuning_job.py @@ -181,13 +181,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py index 75de719b6aa9e..230aeca14a67f 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_list_custom_jobs.py @@ -51,13 +51,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py index 3ad5537a10c29..272069b2f87f0 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_model_service.py @@ -345,13 +345,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py index 5ab29ee999f16..bc97ff6a58be3 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_pipeline_job.py @@ -177,13 +177,13 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py b/providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py index e630fd1471a8a..21ee27cd9be76 100644 --- a/providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py +++ b/providers/tests/system/google/cloud/video_intelligence/example_video_intelligence.py @@ -154,14 +154,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vision/example_vision_annotate_image.py b/providers/tests/system/google/cloud/vision/example_vision_annotate_image.py index 1e09fb7fe4f9d..178920e83a433 100644 --- a/providers/tests/system/google/cloud/vision/example_vision_annotate_image.py +++ b/providers/tests/system/google/cloud/vision/example_vision_annotate_image.py @@ -191,14 +191,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vision/example_vision_autogenerated.py b/providers/tests/system/google/cloud/vision/example_vision_autogenerated.py index 11fd3cdd54029..e84af0fdbac34 100644 --- a/providers/tests/system/google/cloud/vision/example_vision_autogenerated.py +++ b/providers/tests/system/google/cloud/vision/example_vision_autogenerated.py @@ -268,14 +268,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/vision/example_vision_explicit.py b/providers/tests/system/google/cloud/vision/example_vision_explicit.py index 0c71be95bda7a..2e480fec4f75e 100644 --- a/providers/tests/system/google/cloud/vision/example_vision_explicit.py +++ b/providers/tests/system/google/cloud/vision/example_vision_explicit.py @@ -279,14 +279,14 @@ delete_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/cloud/workflows/example_workflows.py b/providers/tests/system/google/cloud/workflows/example_workflows.py index 6d13484acbb42..b933d75a6f2b6 100644 --- a/providers/tests/system/google/cloud/workflows/example_workflows.py +++ b/providers/tests/system/google/cloud/workflows/example_workflows.py @@ -227,14 +227,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/datacatalog/example_datacatalog_entries.py b/providers/tests/system/google/datacatalog/example_datacatalog_entries.py index 0f84361d4fcc8..644adadeef31c 100644 --- a/providers/tests/system/google/datacatalog/example_datacatalog_entries.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_entries.py @@ -200,14 +200,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py b/providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py index a77a023f32338..0ff6cf0716a28 100644 --- a/providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_search_catalog.py @@ -223,14 +223,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py b/providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py index af49d3de08fa7..19bf62c88ccd5 100644 --- a/providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_tag_templates.py @@ -183,14 +183,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/datacatalog/example_datacatalog_tags.py b/providers/tests/system/google/datacatalog/example_datacatalog_tags.py index fdbdaa451056a..0f74b49e102ea 100644 --- a/providers/tests/system/google/datacatalog/example_datacatalog_tags.py +++ b/providers/tests/system/google/datacatalog/example_datacatalog_tags.py @@ -233,14 +233,14 @@ # ### Everything below this line is not part of example ### # ### Just for system tests purpose ### - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/leveldb/example_leveldb.py b/providers/tests/system/google/leveldb/example_leveldb.py index 8b11d9e87bebc..52a543f048e49 100644 --- a/providers/tests/system/google/leveldb/example_leveldb.py +++ b/providers/tests/system/google/leveldb/example_leveldb.py @@ -60,14 +60,14 @@ # [END howto_operator_leveldb_put_key] get_key_leveldb_task >> put_key_leveldb_task - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/marketing_platform/example_analytics_admin.py b/providers/tests/system/google/marketing_platform/example_analytics_admin.py index 16ce6f8190bdc..3536e7b945da2 100644 --- a/providers/tests/system/google/marketing_platform/example_analytics_admin.py +++ b/providers/tests/system/google/marketing_platform/example_analytics_admin.py @@ -219,13 +219,13 @@ def delete_connection(connection_id: str) -> None: # TEST TEARDOWN >> delete_connection_task ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/marketing_platform/example_campaign_manager.py b/providers/tests/system/google/marketing_platform/example_campaign_manager.py index 0c3b26e5e98b5..70e8c208d50eb 100644 --- a/providers/tests/system/google/marketing_platform/example_campaign_manager.py +++ b/providers/tests/system/google/marketing_platform/example_campaign_manager.py @@ -322,13 +322,13 @@ def delete_connection(connection_id: str) -> None: >> delete_connection(connection_id=CONNECTION_ID) ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/marketing_platform/example_search_ads.py b/providers/tests/system/google/marketing_platform/example_search_ads.py index 5d86a3ad1684d..cb733f5bcf4f3 100644 --- a/providers/tests/system/google/marketing_platform/example_search_ads.py +++ b/providers/tests/system/google/marketing_platform/example_search_ads.py @@ -102,7 +102,7 @@ (query_report >> get_field >> search_fields >> get_custom_column >> list_custom_columns) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/google/suite/example_local_to_drive.py b/providers/tests/system/google/suite/example_local_to_drive.py index ac8ebfc5aabbc..397e65fa2bb5a 100644 --- a/providers/tests/system/google/suite/example_local_to_drive.py +++ b/providers/tests/system/google/suite/example_local_to_drive.py @@ -141,13 +141,13 @@ def delete_connection(connection_id: str) -> None: >> delete_connection_task ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/http/example_http.py b/providers/tests/system/http/example_http.py index 98423943607cd..a2900ad582461 100644 --- a/providers/tests/system/http/example_http.py +++ b/providers/tests/system/http/example_http.py @@ -157,7 +157,7 @@ def get_next_page_cursor(response) -> dict | None: task_get_op_response_filter >> task_put_op >> task_del_op >> task_post_op_formenc task_post_op_formenc >> task_get_paginated -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/influxdb/example_influxdb.py b/providers/tests/system/influxdb/example_influxdb.py index 8e4d486742d5f..9b67b05d0a6fd 100644 --- a/providers/tests/system/influxdb/example_influxdb.py +++ b/providers/tests/system/influxdb/example_influxdb.py @@ -61,13 +61,13 @@ def test_influxdb_hook(): ) as dag: test_influxdb_hook() - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/influxdb/example_influxdb_query.py b/providers/tests/system/influxdb/example_influxdb_query.py index 95940e8c8f366..8c4c548b84f4c 100644 --- a/providers/tests/system/influxdb/example_influxdb_query.py +++ b/providers/tests/system/influxdb/example_influxdb_query.py @@ -43,7 +43,7 @@ # [END howto_operator_influxdb] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/jdbc/example_jdbc_queries.py b/providers/tests/system/jdbc/example_jdbc_queries.py index ce9234a6fb0b7..5e73c22b3cf15 100644 --- a/providers/tests/system/jdbc/example_jdbc_queries.py +++ b/providers/tests/system/jdbc/example_jdbc_queries.py @@ -59,13 +59,13 @@ delete_data >> insert_data >> run_this_last - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/jenkins/example_jenkins_job_trigger.py b/providers/tests/system/jenkins/example_jenkins_job_trigger.py index 6b7fab62a0d36..a24b48dda6c37 100644 --- a/providers/tests/system/jenkins/example_jenkins_job_trigger.py +++ b/providers/tests/system/jenkins/example_jenkins_job_trigger.py @@ -72,7 +72,7 @@ def grab_artifact_from_jenkins(url): # job_trigger >> grab_artifact_from_jenkins() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_adf_run_pipeline.py b/providers/tests/system/microsoft/azure/example_adf_run_pipeline.py index a6eae4d71486d..e65524b3c746b 100644 --- a/providers/tests/system/microsoft/azure/example_adf_run_pipeline.py +++ b/providers/tests/system/microsoft/azure/example_adf_run_pipeline.py @@ -108,13 +108,13 @@ # Task dependency created via `XComArgs`: # run_pipeline2 >> pipeline_run_sensor - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_adls_create.py b/providers/tests/system/microsoft/azure/example_adls_create.py index 3f76525b6b87e..2c5828da44603 100644 --- a/providers/tests/system/microsoft/azure/example_adls_create.py +++ b/providers/tests/system/microsoft/azure/example_adls_create.py @@ -46,13 +46,13 @@ upload_data >> delete_file - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_adls_delete.py b/providers/tests/system/microsoft/azure/example_adls_delete.py index 34abc6a9b2dae..fcfe1793f27eb 100644 --- a/providers/tests/system/microsoft/azure/example_adls_delete.py +++ b/providers/tests/system/microsoft/azure/example_adls_delete.py @@ -46,13 +46,13 @@ upload_file >> remove_file - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_adls_list.py b/providers/tests/system/microsoft/azure/example_adls_list.py index 594b161aaa9d6..26e65e0cf8d9c 100644 --- a/providers/tests/system/microsoft/azure/example_adls_list.py +++ b/providers/tests/system/microsoft/azure/example_adls_list.py @@ -42,13 +42,13 @@ ) # [END howto_operator_adls_list] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_azure_batch_operator.py b/providers/tests/system/microsoft/azure/example_azure_batch_operator.py index 944c737f96a0d..33d4603c16d43 100644 --- a/providers/tests/system/microsoft/azure/example_azure_batch_operator.py +++ b/providers/tests/system/microsoft/azure/example_azure_batch_operator.py @@ -57,7 +57,7 @@ ) # [END howto_azure_batch_operator] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_azure_container_instances.py b/providers/tests/system/microsoft/azure/example_azure_container_instances.py index ee9b025193e36..fe7ccfd48ecc4 100644 --- a/providers/tests/system/microsoft/azure/example_azure_container_instances.py +++ b/providers/tests/system/microsoft/azure/example_azure_container_instances.py @@ -90,7 +90,7 @@ cpu=1.0, task_id="start_container_with_azure_container_volume", ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_azure_cosmosdb.py b/providers/tests/system/microsoft/azure/example_azure_cosmosdb.py index 1d4bb60b05163..7dc9524ef46e1 100644 --- a/providers/tests/system/microsoft/azure/example_azure_cosmosdb.py +++ b/providers/tests/system/microsoft/azure/example_azure_cosmosdb.py @@ -64,13 +64,13 @@ t1 >> t2 - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_azure_service_bus.py b/providers/tests/system/microsoft/azure/example_azure_service_bus.py index bfa77db40dba0..091fdff3c2cba 100644 --- a/providers/tests/system/microsoft/azure/example_azure_service_bus.py +++ b/providers/tests/system/microsoft/azure/example_azure_service_bus.py @@ -172,13 +172,13 @@ delete_service_bus_queue, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_azure_synapse.py b/providers/tests/system/microsoft/azure/example_azure_synapse.py index c7f7800e57834..571cd5b46ff88 100644 --- a/providers/tests/system/microsoft/azure/example_azure_synapse.py +++ b/providers/tests/system/microsoft/azure/example_azure_synapse.py @@ -69,7 +69,7 @@ ) # [END howto_operator_azure_synapse] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_fileshare.py b/providers/tests/system/microsoft/azure/example_fileshare.py index bfa9819dbef68..49909ae38fe73 100644 --- a/providers/tests/system/microsoft/azure/example_fileshare.py +++ b/providers/tests/system/microsoft/azure/example_fileshare.py @@ -55,13 +55,13 @@ def delete_fileshare(): ) as dag: create_fileshare() >> delete_fileshare() - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_local_to_adls.py b/providers/tests/system/microsoft/azure/example_local_to_adls.py index d540aaf34d993..f8eace7577528 100644 --- a/providers/tests/system/microsoft/azure/example_local_to_adls.py +++ b/providers/tests/system/microsoft/azure/example_local_to_adls.py @@ -47,13 +47,13 @@ upload_file >> delete_file - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_local_to_wasb.py b/providers/tests/system/microsoft/azure/example_local_to_wasb.py index 7e368ca839928..859c33975842f 100644 --- a/providers/tests/system/microsoft/azure/example_local_to_wasb.py +++ b/providers/tests/system/microsoft/azure/example_local_to_wasb.py @@ -49,13 +49,13 @@ upload >> delete - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_msfabric.py b/providers/tests/system/microsoft/azure/example_msfabric.py index b1113025286d4..0f65df2f72f9e 100644 --- a/providers/tests/system/microsoft/azure/example_msfabric.py +++ b/providers/tests/system/microsoft/azure/example_msfabric.py @@ -51,13 +51,13 @@ ) # [END howto_operator_ms_fabric_create_item_schedule] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_msgraph.py b/providers/tests/system/microsoft/azure/example_msgraph.py index 33ee00468523b..239ebb76ab09b 100644 --- a/providers/tests/system/microsoft/azure/example_msgraph.py +++ b/providers/tests/system/microsoft/azure/example_msgraph.py @@ -49,13 +49,13 @@ site_task >> site_pages_task - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_powerbi.py b/providers/tests/system/microsoft/azure/example_powerbi.py index b5a982a1159a5..a2d9d675476bb 100644 --- a/providers/tests/system/microsoft/azure/example_powerbi.py +++ b/providers/tests/system/microsoft/azure/example_powerbi.py @@ -97,13 +97,13 @@ workspaces_task >> workspaces_info_task >> check_workspace_status_task refresh_dataset_task >> refresh_dataset_history_task - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_powerbi_dataset_refresh.py b/providers/tests/system/microsoft/azure/example_powerbi_dataset_refresh.py index c02cec3e57f5e..5453caff6463b 100644 --- a/providers/tests/system/microsoft/azure/example_powerbi_dataset_refresh.py +++ b/providers/tests/system/microsoft/azure/example_powerbi_dataset_refresh.py @@ -76,13 +76,13 @@ def create_connection(conn_id_name: str): refresh_powerbi_dataset, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_s3_to_wasb.py b/providers/tests/system/microsoft/azure/example_s3_to_wasb.py index 88ae64324d17e..699da911d0846 100644 --- a/providers/tests/system/microsoft/azure/example_s3_to_wasb.py +++ b/providers/tests/system/microsoft/azure/example_s3_to_wasb.py @@ -103,13 +103,13 @@ remove_s3_bucket, ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure when "tearDown" task with trigger # rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_sftp_to_wasb.py b/providers/tests/system/microsoft/azure/example_sftp_to_wasb.py index 42e5968304f08..0ed953a5853d8 100644 --- a/providers/tests/system/microsoft/azure/example_sftp_to_wasb.py +++ b/providers/tests/system/microsoft/azure/example_sftp_to_wasb.py @@ -75,13 +75,13 @@ def delete_sftp_file(): transfer_files_to_sftp_step >> transfer_files_to_azure >> delete_blob_file_step >> delete_sftp_file() - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py b/providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py index de87d6b4c2483..589f840e46ab9 100644 --- a/providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py +++ b/providers/tests/system/microsoft/azure/example_synapse_run_pipeline.py @@ -43,13 +43,13 @@ # [END howto_operator_azure_synapse_run_pipeline] begin >> run_pipeline1 - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/azure/example_wasb_sensors.py b/providers/tests/system/microsoft/azure/example_wasb_sensors.py index 56c1dce34d688..c4dd7c8fa0b2e 100644 --- a/providers/tests/system/microsoft/azure/example_wasb_sensors.py +++ b/providers/tests/system/microsoft/azure/example_wasb_sensors.py @@ -62,7 +62,7 @@ # [END wasb_prefix_sensor] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/mssql/example_mssql.py b/providers/tests/system/microsoft/mssql/example_mssql.py index 12e2815c59d01..8e29d805ab261 100644 --- a/providers/tests/system/microsoft/mssql/example_mssql.py +++ b/providers/tests/system/microsoft/mssql/example_mssql.py @@ -146,12 +146,12 @@ def insert_mssql_hook(): ) # [END mssql_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/microsoft/winrm/example_winrm.py b/providers/tests/system/microsoft/winrm/example_winrm.py index 1de1cb8c0fd68..1c7e31ed6752e 100644 --- a/providers/tests/system/microsoft/winrm/example_winrm.py +++ b/providers/tests/system/microsoft/winrm/example_winrm.py @@ -64,13 +64,13 @@ [t1, t2, t3] >> run_this_last - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/mysql/example_mysql.py b/providers/tests/system/mysql/example_mysql.py index 6f0f884197f3a..2f1f37f573a22 100644 --- a/providers/tests/system/mysql/example_mysql.py +++ b/providers/tests/system/mysql/example_mysql.py @@ -59,7 +59,7 @@ drop_table_mysql_task >> mysql_task -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/neo4j/example_neo4j.py b/providers/tests/system/neo4j/example_neo4j.py index 80db3fbb2ab1d..54de74d5f9fbd 100644 --- a/providers/tests/system/neo4j/example_neo4j.py +++ b/providers/tests/system/neo4j/example_neo4j.py @@ -48,7 +48,7 @@ # [END run_query_neo4j_operator] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/openai/example_openai.py b/providers/tests/system/openai/example_openai.py index d342207b5d09b..3475597abd209 100644 --- a/providers/tests/system/openai/example_openai.py +++ b/providers/tests/system/openai/example_openai.py @@ -104,7 +104,7 @@ def task_to_store_input_text_in_xcom(): example_openai_dag() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/openai/example_trigger_batch_operator.py b/providers/tests/system/openai/example_trigger_batch_operator.py index 3dae1df5d9d0b..4fd3531fab379 100644 --- a/providers/tests/system/openai/example_trigger_batch_operator.py +++ b/providers/tests/system/openai/example_trigger_batch_operator.py @@ -111,7 +111,7 @@ def cleanup_batch_output_file(batch_id, **context): openai_batch_chat_completions() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/opensearch/example_opensearch.py b/providers/tests/system/opensearch/example_opensearch.py index e339fbe788481..904f17a47be7a 100644 --- a/providers/tests/system/opensearch/example_opensearch.py +++ b/providers/tests/system/opensearch/example_opensearch.py @@ -118,14 +118,14 @@ def load_connections(): chain(create_index, add_document_by_class, add_document_by_args, search_high_level, search_low_level) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/opsgenie/example_opsgenie_alert.py b/providers/tests/system/opsgenie/example_opsgenie_alert.py index cf11f824eec8c..be4056b407e6d 100644 --- a/providers/tests/system/opsgenie/example_opsgenie_alert.py +++ b/providers/tests/system/opsgenie/example_opsgenie_alert.py @@ -51,7 +51,7 @@ ) # [END howto_opsgenie_delete_alert_operator] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/opsgenie/example_opsgenie_notifier.py b/providers/tests/system/opsgenie/example_opsgenie_notifier.py index 97f5945aa9772..79123cf83c537 100644 --- a/providers/tests/system/opsgenie/example_opsgenie_notifier.py +++ b/providers/tests/system/opsgenie/example_opsgenie_notifier.py @@ -37,7 +37,7 @@ ) # [END howto_notifier_opsgenie] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/papermill/example_papermill.py b/providers/tests/system/papermill/example_papermill.py index 529647fe4e719..33b427785cf25 100644 --- a/providers/tests/system/papermill/example_papermill.py +++ b/providers/tests/system/papermill/example_papermill.py @@ -52,7 +52,7 @@ ) # [END howto_operator_papermill] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/papermill/example_papermill_remote_verify.py b/providers/tests/system/papermill/example_papermill_remote_verify.py index f325928b8cd01..89d482858cb06 100644 --- a/providers/tests/system/papermill/example_papermill_remote_verify.py +++ b/providers/tests/system/papermill/example_papermill_remote_verify.py @@ -75,7 +75,7 @@ def check_notebook(output_notebook, execution_date): ) # [END howto_verify_operator_papermill_remote_kernel] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/papermill/example_papermill_verify.py b/providers/tests/system/papermill/example_papermill_verify.py index cfae2cc6ed3e0..632b209f21ef6 100644 --- a/providers/tests/system/papermill/example_papermill_verify.py +++ b/providers/tests/system/papermill/example_papermill_verify.py @@ -73,7 +73,7 @@ def check_notebook(inlets, execution_date): run_this >> check_notebook(inlets=AUTO, execution_date="{{ execution_date }}") # [END howto_verify_operator_papermill] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/pgvector/example_pgvector.py b/providers/tests/system/pgvector/example_pgvector.py index 5c16b9da307cb..27402ae38f11b 100644 --- a/providers/tests/system/pgvector/example_pgvector.py +++ b/providers/tests/system/pgvector/example_pgvector.py @@ -76,7 +76,7 @@ def cleanup_postgres_objects(): example_pgvector_dag() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/pgvector/example_pgvector_openai.py b/providers/tests/system/pgvector/example_pgvector_openai.py index 9c8f841e8f933..9d0a3165c22dd 100644 --- a/providers/tests/system/pgvector/example_pgvector_openai.py +++ b/providers/tests/system/pgvector/example_pgvector_openai.py @@ -90,7 +90,7 @@ def cleanup_postgres_objects(): example_pgvector_dag() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/pinecone/example_create_pod_index.py b/providers/tests/system/pinecone/example_create_pod_index.py index e0342e2d62008..9a93b1241f884 100644 --- a/providers/tests/system/pinecone/example_create_pod_index.py +++ b/providers/tests/system/pinecone/example_create_pod_index.py @@ -56,7 +56,7 @@ def delete_index(): create_index >> delete_index() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/pinecone/example_create_serverless_index.py b/providers/tests/system/pinecone/example_create_serverless_index.py index 28078b1b608cd..e276c345c4a03 100644 --- a/providers/tests/system/pinecone/example_create_serverless_index.py +++ b/providers/tests/system/pinecone/example_create_serverless_index.py @@ -55,7 +55,7 @@ def delete_index(): create_index >> delete_index() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/pinecone/example_dag_pinecone.py b/providers/tests/system/pinecone/example_dag_pinecone.py index 744f6518dceae..fd23f79bf7291 100644 --- a/providers/tests/system/pinecone/example_dag_pinecone.py +++ b/providers/tests/system/pinecone/example_dag_pinecone.py @@ -46,7 +46,7 @@ # [END howto_operator_pinecone_ingest] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/pinecone/example_pinecone_cohere.py b/providers/tests/system/pinecone/example_pinecone_cohere.py index 80e9b1efad8eb..8813479d61a70 100644 --- a/providers/tests/system/pinecone/example_pinecone_cohere.py +++ b/providers/tests/system/pinecone/example_pinecone_cohere.py @@ -76,7 +76,7 @@ def delete_index(): create_index() >> embed_task >> transformed_output >> perform_ingestion >> delete_index() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/pinecone/example_pinecone_openai.py b/providers/tests/system/pinecone/example_pinecone_openai.py index dad83f48085a5..b75a155192578 100644 --- a/providers/tests/system/pinecone/example_pinecone_openai.py +++ b/providers/tests/system/pinecone/example_pinecone_openai.py @@ -107,7 +107,7 @@ def delete_index(): create_index >> embed_task >> perform_ingestion >> delete_index() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/postgres/example_postgres.py b/providers/tests/system/postgres/example_postgres.py index 5ffaf6ea0098a..70ad784ecdf19 100644 --- a/providers/tests/system/postgres/example_postgres.py +++ b/providers/tests/system/postgres/example_postgres.py @@ -80,13 +80,13 @@ create_pet_table >> populate_pet_table >> get_all_pets >> get_birth_date # [END postgres_sql_execute_query_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/presto/example_gcs_to_presto.py b/providers/tests/system/presto/example_gcs_to_presto.py index ceccf098471dc..6c8baaf691346 100644 --- a/providers/tests/system/presto/example_gcs_to_presto.py +++ b/providers/tests/system/presto/example_gcs_to_presto.py @@ -50,7 +50,7 @@ # [END gcs_csv_to_presto_table] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/qdrant/example_dag_qdrant.py b/providers/tests/system/qdrant/example_dag_qdrant.py index 85c92d9c99312..a6dc94d443586 100644 --- a/providers/tests/system/qdrant/example_dag_qdrant.py +++ b/providers/tests/system/qdrant/example_dag_qdrant.py @@ -43,7 +43,7 @@ # [END howto_operator_qdrant_ingest] -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/redis/example_redis_publish.py b/providers/tests/system/redis/example_redis_publish.py index 335256f68e57d..671041e7d9808 100644 --- a/providers/tests/system/redis/example_redis_publish.py +++ b/providers/tests/system/redis/example_redis_publish.py @@ -73,13 +73,13 @@ publish_task >> key_sensor_task - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/salesforce/example_bulk.py b/providers/tests/system/salesforce/example_bulk.py index 7d43de648805d..71887e57f2156 100644 --- a/providers/tests/system/salesforce/example_bulk.py +++ b/providers/tests/system/salesforce/example_bulk.py @@ -90,7 +90,7 @@ # [END howto_salesforce_bulk_delete_operation] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/salesforce/example_salesforce_apex_rest.py b/providers/tests/system/salesforce/example_salesforce_apex_rest.py index 2961ffa4e4784..3d724289b338e 100644 --- a/providers/tests/system/salesforce/example_salesforce_apex_rest.py +++ b/providers/tests/system/salesforce/example_salesforce_apex_rest.py @@ -41,7 +41,7 @@ # [END howto_salesforce_apex_rest_operator] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/samba/example_gcs_to_samba.py b/providers/tests/system/samba/example_gcs_to_samba.py index d7717fcbfc727..895ddcf624e7e 100644 --- a/providers/tests/system/samba/example_gcs_to_samba.py +++ b/providers/tests/system/samba/example_gcs_to_samba.py @@ -138,14 +138,14 @@ >> delete_bucket ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/sftp/example_sftp_sensor.py b/providers/tests/system/sftp/example_sftp_sensor.py index a3bde8a03af0a..cd23e18d7d56a 100644 --- a/providers/tests/system/sftp/example_sftp_sensor.py +++ b/providers/tests/system/sftp/example_sftp_sensor.py @@ -89,14 +89,14 @@ def sftp_sensor_decorator(): >> remove_file_task_end ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/singularity/example_singularity.py b/providers/tests/system/singularity/example_singularity.py index e80a0f40eb955..7f311f39c5faa 100644 --- a/providers/tests/system/singularity/example_singularity.py +++ b/providers/tests/system/singularity/example_singularity.py @@ -50,7 +50,7 @@ t3 >> t4 -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/slack/example_slack.py b/providers/tests/system/slack/example_slack.py index f122dffca2782..1b0be71efdc9f 100644 --- a/providers/tests/system/slack/example_slack.py +++ b/providers/tests/system/slack/example_slack.py @@ -94,7 +94,7 @@ ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/slack/example_slack_webhook.py b/providers/tests/system/slack/example_slack_webhook.py index 7e45dc92d921f..6a59f6e4de70d 100644 --- a/providers/tests/system/slack/example_slack_webhook.py +++ b/providers/tests/system/slack/example_slack_webhook.py @@ -69,7 +69,7 @@ slack_webhook_operator_text >> slack_webhook_operator_blocks -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/slack/example_sql_to_slack.py b/providers/tests/system/slack/example_sql_to_slack.py index 457bcecaea671..bd42e3b47d4c5 100644 --- a/providers/tests/system/slack/example_sql_to_slack.py +++ b/providers/tests/system/slack/example_sql_to_slack.py @@ -52,7 +52,7 @@ # [END howto_operator_sql_to_slack_api_file] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/slack/example_sql_to_slack_webhook.py b/providers/tests/system/slack/example_sql_to_slack_webhook.py index 44fad156657be..75da7353daa2d 100644 --- a/providers/tests/system/slack/example_sql_to_slack_webhook.py +++ b/providers/tests/system/slack/example_sql_to_slack_webhook.py @@ -51,7 +51,7 @@ # [END howto_operator_sql_to_slack_webhook] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/snowflake/example_copy_into_snowflake.py b/providers/tests/system/snowflake/example_copy_into_snowflake.py index 32eb9699d74b6..a2f9b9277e308 100644 --- a/providers/tests/system/snowflake/example_copy_into_snowflake.py +++ b/providers/tests/system/snowflake/example_copy_into_snowflake.py @@ -60,7 +60,7 @@ # [END howto_operator_s3_copy_into_snowflake] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/snowflake/example_snowflake.py b/providers/tests/system/snowflake/example_snowflake.py index e851b13c08200..f2bf8d00679de 100644 --- a/providers/tests/system/snowflake/example_snowflake.py +++ b/providers/tests/system/snowflake/example_snowflake.py @@ -96,7 +96,7 @@ ) -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/snowflake/example_snowpark_decorator.py b/providers/tests/system/snowflake/example_snowpark_decorator.py index 4c2319a91b21e..59cf1f1fedbb6 100644 --- a/providers/tests/system/snowflake/example_snowpark_decorator.py +++ b/providers/tests/system/snowflake/example_snowpark_decorator.py @@ -80,7 +80,7 @@ def check_num_rows(table_name: str): # [END howto_decorator_snowpark] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/snowflake/example_snowpark_operator.py b/providers/tests/system/snowflake/example_snowpark_operator.py index 1d7cce34f7aa1..c9ede310074c5 100644 --- a/providers/tests/system/snowflake/example_snowpark_operator.py +++ b/providers/tests/system/snowflake/example_snowpark_operator.py @@ -89,7 +89,7 @@ def check_num_rows(table_name: str): # [END howto_operator_snowpark] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/sqlite/example_sqlite.py b/providers/tests/system/sqlite/example_sqlite.py index 1ea0270a0df27..e3eec31c1bfd6 100644 --- a/providers/tests/system/sqlite/example_sqlite.py +++ b/providers/tests/system/sqlite/example_sqlite.py @@ -91,13 +91,13 @@ def replace_sqlite_hook(): >> replace_sqlite_hook() ) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/tableau/example_tableau.py b/providers/tests/system/tableau/example_tableau.py index 1472511a66c95..41b99b0ece89e 100644 --- a/providers/tests/system/tableau/example_tableau.py +++ b/providers/tests/system/tableau/example_tableau.py @@ -71,7 +71,7 @@ # task_refresh_workbook_non_blocking >> task_check_job_status -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/telegram/example_telegram.py b/providers/tests/system/telegram/example_telegram.py index a7e9018cf85f1..23341f991447d 100644 --- a/providers/tests/system/telegram/example_telegram.py +++ b/providers/tests/system/telegram/example_telegram.py @@ -45,7 +45,7 @@ # [END howto_operator_telegram] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py b/providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py index 26fd27554d5d1..dec92a90366bb 100644 --- a/providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py +++ b/providers/tests/system/teradata/example_azure_blob_to_teradata_transfer.py @@ -205,13 +205,13 @@ ) # [END azure_blob_to_teradata_transfer_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/teradata/example_s3_to_teradata_transfer.py b/providers/tests/system/teradata/example_s3_to_teradata_transfer.py index ab361990b21e4..d7ba2a3b415f8 100644 --- a/providers/tests/system/teradata/example_s3_to_teradata_transfer.py +++ b/providers/tests/system/teradata/example_s3_to_teradata_transfer.py @@ -206,13 +206,13 @@ ) # [END s3_to_teradata_transfer_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/teradata/example_ssl_teradata.py b/providers/tests/system/teradata/example_ssl_teradata.py index 3836715a8e634..8ec5898f7437c 100644 --- a/providers/tests/system/teradata/example_ssl_teradata.py +++ b/providers/tests/system/teradata/example_ssl_teradata.py @@ -121,13 +121,13 @@ # [END teradata_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/teradata/example_teradata.py b/providers/tests/system/teradata/example_teradata.py index ae41effce7fec..a12fb3abb7e6c 100644 --- a/providers/tests/system/teradata/example_teradata.py +++ b/providers/tests/system/teradata/example_teradata.py @@ -164,13 +164,13 @@ # [END teradata_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/teradata/example_teradata_call_sp.py b/providers/tests/system/teradata/example_teradata_call_sp.py index f48583d83ec55..d41121324bcf3 100644 --- a/providers/tests/system/teradata/example_teradata_call_sp.py +++ b/providers/tests/system/teradata/example_teradata_call_sp.py @@ -162,13 +162,13 @@ # [END howto_teradata_operator_for_sp] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/teradata/example_teradata_compute_cluster.py b/providers/tests/system/teradata/example_teradata_compute_cluster.py index 210cde7816bc2..6ffb4fd99ec67 100644 --- a/providers/tests/system/teradata/example_teradata_compute_cluster.py +++ b/providers/tests/system/teradata/example_teradata_compute_cluster.py @@ -146,13 +146,13 @@ # [END teradata_vantage_lake_compute_cluster_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/teradata/example_teradata_to_teradata_transfer.py b/providers/tests/system/teradata/example_teradata_to_teradata_transfer.py index 7f315202606fc..a96d42816a271 100644 --- a/providers/tests/system/teradata/example_teradata_to_teradata_transfer.py +++ b/providers/tests/system/teradata/example_teradata_to_teradata_transfer.py @@ -148,13 +148,13 @@ ) # [END teradata_to_teradata_transfer_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/trino/example_gcs_to_trino.py b/providers/tests/system/trino/example_gcs_to_trino.py index e0249101eb1a4..4c559301e64da 100644 --- a/providers/tests/system/trino/example_gcs_to_trino.py +++ b/providers/tests/system/trino/example_gcs_to_trino.py @@ -50,7 +50,7 @@ # [END gcs_csv_to_trino_table] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/trino/example_trino.py b/providers/tests/system/trino/example_trino.py index d708d9b7b8fb7..db9fef4128b93 100644 --- a/providers/tests/system/trino/example_trino.py +++ b/providers/tests/system/trino/example_trino.py @@ -91,7 +91,7 @@ # [END howto_operator_trino] -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/weaviate/example_weaviate_cohere.py b/providers/tests/system/weaviate/example_weaviate_cohere.py index 1db5e2cccb72a..11b45c2cb45be 100644 --- a/providers/tests/system/weaviate/example_weaviate_cohere.py +++ b/providers/tests/system/weaviate/example_weaviate_cohere.py @@ -115,7 +115,7 @@ def delete_weaviate_collections(): example_weaviate_cohere() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/weaviate/example_weaviate_dynamic_mapping_dag.py b/providers/tests/system/weaviate/example_weaviate_dynamic_mapping_dag.py index 38c0168a9f615..b3dfb52a60fb9 100644 --- a/providers/tests/system/weaviate/example_weaviate_dynamic_mapping_dag.py +++ b/providers/tests/system/weaviate/example_weaviate_dynamic_mapping_dag.py @@ -92,7 +92,7 @@ def delete_weaviate_collection(collection_name): example_weaviate_dynamic_mapping_dag() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/weaviate/example_weaviate_openai.py b/providers/tests/system/weaviate/example_weaviate_openai.py index edad9058f8738..76713826035a1 100644 --- a/providers/tests/system/weaviate/example_weaviate_openai.py +++ b/providers/tests/system/weaviate/example_weaviate_openai.py @@ -123,7 +123,7 @@ def delete_weaviate_collection(): example_weaviate_openai() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/weaviate/example_weaviate_operator.py b/providers/tests/system/weaviate/example_weaviate_operator.py index 309864ad64924..734a7e5fa3945 100644 --- a/providers/tests/system/weaviate/example_weaviate_operator.py +++ b/providers/tests/system/weaviate/example_weaviate_operator.py @@ -297,7 +297,7 @@ def delete_weaviate_docs_collection_without_vector(): example_weaviate_using_operator() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/weaviate/example_weaviate_using_hook.py b/providers/tests/system/weaviate/example_weaviate_using_hook.py index 601a222f94d4e..8df913e0044fd 100644 --- a/providers/tests/system/weaviate/example_weaviate_using_hook.py +++ b/providers/tests/system/weaviate/example_weaviate_using_hook.py @@ -140,7 +140,7 @@ def delete_weaviate_collection_without_vector(): example_weaviate_dag_using_hook() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/weaviate/example_weaviate_vectorizer_dag.py b/providers/tests/system/weaviate/example_weaviate_vectorizer_dag.py index f0306177bffd4..17b7e25edb3cb 100644 --- a/providers/tests/system/weaviate/example_weaviate_vectorizer_dag.py +++ b/providers/tests/system/weaviate/example_weaviate_vectorizer_dag.py @@ -99,7 +99,7 @@ def delete_weaviate_collection(): example_weaviate_vectorizer_dag() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/weaviate/example_weaviate_without_vectorizer_dag.py b/providers/tests/system/weaviate/example_weaviate_without_vectorizer_dag.py index e2cf9e560e785..efb8d89730f6e 100644 --- a/providers/tests/system/weaviate/example_weaviate_without_vectorizer_dag.py +++ b/providers/tests/system/weaviate/example_weaviate_without_vectorizer_dag.py @@ -111,7 +111,7 @@ def delete_weaviate_collection(): example_weaviate_without_vectorizer_dag() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/yandex/example_yandexcloud.py b/providers/tests/system/yandex/example_yandexcloud.py index ab72adc14c051..85284af2642ef 100644 --- a/providers/tests/system/yandex/example_yandexcloud.py +++ b/providers/tests/system/yandex/example_yandexcloud.py @@ -27,14 +27,13 @@ import yandex.cloud.dataproc.v1.job_service_pb2_grpc as job_service_grpc_pb import yandex.cloud.dataproc.v1.subcluster_pb2 as subcluster_pb from google.protobuf.json_format import MessageToDict +from tests_common.test_utils.system_tests import get_test_env_id from yandexcloud.operations import OperationError from airflow import DAG from airflow.decorators import task from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook -from dev.tests_common.test_utils.system_tests import get_test_env_id - ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_hook" @@ -197,14 +196,14 @@ def delete_cluster( spark_job >> delete_task - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/yandex/example_yandexcloud_dataproc.py b/providers/tests/system/yandex/example_yandexcloud_dataproc.py index d97e3d592ef88..13cabfc0c48a8 100644 --- a/providers/tests/system/yandex/example_yandexcloud_dataproc.py +++ b/providers/tests/system/yandex/example_yandexcloud_dataproc.py @@ -19,6 +19,8 @@ import uuid from datetime import datetime +from tests_common.test_utils.system_tests import get_test_env_id + from airflow import DAG from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, @@ -32,8 +34,6 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule -from dev.tests_common.test_utils.system_tests import get_test_env_id - # should be filled with appropriate ids @@ -163,13 +163,13 @@ create_cluster >> create_mapreduce_job >> create_hive_query >> create_hive_query_from_file create_hive_query_from_file >> create_spark_job >> create_pyspark_job >> delete_cluster - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py b/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py index 38401a1cdcd39..fe77142f3d069 100644 --- a/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py +++ b/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py @@ -18,6 +18,8 @@ from datetime import datetime +from tests_common.test_utils.system_tests import get_test_env_id + from airflow import DAG from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, @@ -28,8 +30,6 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule -from dev.tests_common.test_utils.system_tests import get_test_env_id - # should be filled with appropriate ids @@ -71,13 +71,13 @@ ) create_spark_job >> delete_cluster - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/yandex/example_yandexcloud_yq.py b/providers/tests/system/yandex/example_yandexcloud_yq.py index 02478825ed5e0..a83e8bd789f97 100644 --- a/providers/tests/system/yandex/example_yandexcloud_yq.py +++ b/providers/tests/system/yandex/example_yandexcloud_yq.py @@ -18,12 +18,12 @@ from datetime import datetime +from tests_common.test_utils.system_tests import get_test_env_id + from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator from airflow.providers.yandex.operators.yq import YQExecuteQueryOperator -from dev.tests_common.test_utils.system_tests import get_test_env_id - ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_yq" @@ -40,13 +40,13 @@ yq_operator = YQExecuteQueryOperator(task_id="sample_query", sql="select 33 as d, 44 as t") yq_operator >> run_this_last - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/ydb/example_ydb.py b/providers/tests/system/ydb/example_ydb.py index d7293cb0d1b56..2f3f88cabe64b 100644 --- a/providers/tests/system/ydb/example_ydb.py +++ b/providers/tests/system/ydb/example_ydb.py @@ -119,13 +119,13 @@ def populate_pet_table_via_bulk_upsert(): ) # [END ydb_operator_howto_guide] - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/system/zendesk/example_zendesk_custom_get.py b/providers/tests/system/zendesk/example_zendesk_custom_get.py index c38d10a26abd3..a94f4e3ef6b5b 100644 --- a/providers/tests/system/zendesk/example_zendesk_custom_get.py +++ b/providers/tests/system/zendesk/example_zendesk_custom_get.py @@ -45,7 +45,7 @@ def fetch_organizations() -> list[dict]: fetch_organizations() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/providers/tests/yandex/hooks/test_yandex.py b/providers/tests/yandex/hooks/test_yandex.py index fbf77115053b2..eacb2708e944a 100644 --- a/providers/tests/yandex/hooks/test_yandex.py +++ b/providers/tests/yandex/hooks/test_yandex.py @@ -25,11 +25,11 @@ yandexcloud = pytest.importorskip("yandexcloud") +from tests_common.test_utils.config import conf_vars + from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook -from dev.tests_common.test_utils.config import conf_vars - class TestYandexHook: @mock.patch("airflow.hooks.base.BaseHook.get_connection") diff --git a/providers/tests/yandex/links/test_yq.py b/providers/tests/yandex/links/test_yq.py index 2ed720e7e141c..dfd255d41c0eb 100644 --- a/providers/tests/yandex/links/test_yq.py +++ b/providers/tests/yandex/links/test_yq.py @@ -19,14 +19,13 @@ from unittest import mock import pytest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.mock_operators import MockOperator from airflow.models.taskinstance import TaskInstance from airflow.models.xcom import XCom from airflow.providers.yandex.links.yq import YQLink -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.mock_operators import MockOperator - yandexcloud = pytest.importorskip("yandexcloud") diff --git a/providers/tests/yandex/operators/test_yq.py b/providers/tests/yandex/operators/test_yq.py index 7e586e929eaff..8c5204497230d 100644 --- a/providers/tests/yandex/operators/test_yq.py +++ b/providers/tests/yandex/operators/test_yq.py @@ -21,8 +21,7 @@ from unittest.mock import MagicMock, call, patch import pytest - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS yandexcloud = pytest.importorskip("yandexcloud") diff --git a/pyproject.toml b/pyproject.toml index 160e8e2ce0b0c..a5dadf0e96651 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -342,7 +342,7 @@ section-order = [ # Make sure we put the "dev" imports at the end, not as a third-party module [tool.ruff.lint.isort.sections] -testing = ["dev", "providers.tests", "task_sdk.tests"] +testing = ["dev", "providers.tests", "task_sdk.tests", "test_common", "tests"] [tool.ruff.lint.extend-per-file-ignores] "airflow/__init__.py" = ["F401", "TCH004"] @@ -377,7 +377,7 @@ testing = ["dev", "providers.tests", "task_sdk.tests"] # TRY002: Use `raise Exception` "dev/perf/*" = ["TID253"] "dev/check_files.py" = ["S101"] -"dev/tests_common/*" = ["S101", "TRY002"] +"tests_common/*" = ["S101", "TRY002"] "dev/breeze/tests/*" = ["TID253", "S101", "TRY002"] "tests/*" = ["D", "TID253", "S101", "TRY002"] "docker_tests/*" = ["D", "TID253", "S101", "TRY002"] @@ -395,7 +395,7 @@ testing = ["dev", "providers.tests", "task_sdk.tests"] "helm_tests/airflow_aux/test_basic_helm_chart.py" = ["B028"] # Test compat imports banned imports to allow testing against older airflow versions -"dev/tests_common/test_utils/compat.py" = ["TID251", "F401"] +"tests_common/test_utils/compat.py" = ["TID251", "F401"] [tool.ruff.lint.flake8-tidy-imports] # Disallow all relative imports. diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index f0bce46894164..1b68dc84aa08f 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -100,6 +100,9 @@ services: - type: bind source: ../../../scripts/docker/entrypoint_ci.sh target: /entrypoint + - type: bind + source: ../../../tests_common + target: /opt/airflow/tests_common - type: bind source: ../../../tests target: /opt/airflow/tests diff --git a/scripts/ci/pre_commit/check_system_tests.py b/scripts/ci/pre_commit/check_system_tests.py index c7c305460cb3e..1eceafaab1da4 100755 --- a/scripts/ci/pre_commit/check_system_tests.py +++ b/scripts/ci/pre_commit/check_system_tests.py @@ -37,13 +37,13 @@ WATCHER_APPEND_INSTRUCTION = "list(dag.tasks) >> watcher()" PYTEST_FUNCTION = """ -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) """ PYTEST_FUNCTION_PATTERN = re.compile( - r"from dev\.tests_common\.test_utils\.system_tests import get_test_run(?: # noqa: E402)?\s+" + r"from tests_common\.test_utils\.system_tests import get_test_run(?: # noqa: E402)?\s+" r"(?:# .+\))?\s+" r"test_run = get_test_run\(dag\)" ) @@ -51,11 +51,11 @@ def _check_file(file: Path): content = file.read_text() - if "from dev.tests_common.test_utils.watcher import watcher" in content: + if "from tests_common.test_utils.watcher import watcher" in content: index = content.find(WATCHER_APPEND_INSTRUCTION) if index == -1: errors.append( - f"[red]The example {file} imports dev.tests_common.test_utils.watcher " + f"[red]The example {file} imports tests_common.test_utils.watcher " f"but does not use it properly![/]\n\n" "[yellow]Make sure you have:[/]\n\n" f" {WATCHER_APPEND_INSTRUCTION}\n\n" diff --git a/scripts/in_container/install_devel_deps.py b/scripts/in_container/install_devel_deps.py index 84f28bb2cc48b..971e11725dd8d 100755 --- a/scripts/in_container/install_devel_deps.py +++ b/scripts/in_container/install_devel_deps.py @@ -28,7 +28,7 @@ def get_devel_test_deps() -> list[str]: # Pre-install the tests_common pytest plugin/utils, in case sources aren't mounted - devel_deps: list[str] = ["./dev/tests_common"] + devel_deps: list[str] = ["./tests_common"] hatch_build_content = (AIRFLOW_SOURCES_DIR / "hatch_build.py").read_text().splitlines() store = False for line in hatch_build_content: diff --git a/task_sdk/tests/conftest.py b/task_sdk/tests/conftest.py index 7839b299eef2c..a410a1217c2f0 100644 --- a/task_sdk/tests/conftest.py +++ b/task_sdk/tests/conftest.py @@ -18,7 +18,7 @@ import pytest -pytest_plugins = "dev.tests_common.pytest_plugin" +pytest_plugins = "tests_common.pytest_plugin" @pytest.hookimpl(tryfirst=True) diff --git a/tests/always/test_connection.py b/tests/always/test_connection.py index 0f6b27c62d4ef..e289c7d2172ed 100644 --- a/tests/always/test_connection.py +++ b/tests/always/test_connection.py @@ -33,8 +33,7 @@ from airflow.models import Connection, crypto from airflow.providers.sqlite.hooks.sqlite import SqliteHook from airflow.providers_manager import HookInfo - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars ConnectionParts = namedtuple("ConnectionParts", ["conn_type", "login", "password", "host", "port", "schema"]) diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 9def5f42206f8..edfc9b46f916e 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -29,8 +29,7 @@ from airflow.models import DagBag from airflow.utils import yaml - -from dev.tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.asserts import assert_queries_count AIRFLOW_SOURCES_ROOT = Path(__file__).resolve().parents[2] AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" diff --git a/tests/always/test_secrets.py b/tests/always/test_secrets.py index a8b2296de1ccd..1e47fc13ce7c9 100644 --- a/tests/always/test_secrets.py +++ b/tests/always/test_secrets.py @@ -24,9 +24,8 @@ from airflow.configuration import ensure_secrets_loaded, initialize_secrets_backends from airflow.models import Connection, Variable from airflow.secrets.cache import SecretCache - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_variables +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_variables class TestConnectionsFromSecrets: diff --git a/tests/always/test_secrets_backends.py b/tests/always/test_secrets_backends.py index c5093d2e562a7..a74c7ac4e95f0 100644 --- a/tests/always/test_secrets_backends.py +++ b/tests/always/test_secrets_backends.py @@ -28,8 +28,7 @@ from airflow.secrets.environment_variables import EnvironmentVariablesBackend from airflow.secrets.metastore import MetastoreBackend from airflow.utils.session import create_session - -from dev.tests_common.test_utils.db import clear_db_connections, clear_db_variables +from tests_common.test_utils.db import clear_db_connections, clear_db_variables pytestmark = pytest.mark.db_test diff --git a/tests/always/test_secrets_local_filesystem.py b/tests/always/test_secrets_local_filesystem.py index 40472ec64d5f5..0a84120d2d087 100644 --- a/tests/always/test_secrets_local_filesystem.py +++ b/tests/always/test_secrets_local_filesystem.py @@ -29,8 +29,7 @@ from airflow.models import Variable from airflow.secrets import local_filesystem from airflow.secrets.local_filesystem import LocalFilesystemBackend - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars @contextmanager diff --git a/tests/api_connexion/conftest.py b/tests/api_connexion/conftest.py index 8ab76fbb76298..0d91a8dc34a76 100644 --- a/tests/api_connexion/conftest.py +++ b/tests/api_connexion/conftest.py @@ -19,9 +19,8 @@ import pytest from airflow.www import app - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules @pytest.fixture(scope="session") @@ -39,7 +38,7 @@ def minimal_app_for_api(): def factory(): with conf_vars( { - ("api", "auth_backends"): "dev.tests_common.test_utils.remote_user_api_auth_backend", + ("api", "auth_backends"): "tests_common.test_utils.remote_user_api_auth_backend", ( "core", "auth_manager", diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index 39b0129c3317d..e02844561f8c4 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -36,12 +36,11 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_assets, clear_db_runs -from dev.tests_common.test_utils.www import _check_last_log +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_assets, clear_db_runs +from tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_backfill_endpoint.py b/tests/api_connexion/endpoints/test_backfill_endpoint.py index 5bb959cabaf56..bc5a214654368 100644 --- a/tests/api_connexion/endpoints/test_backfill_endpoint.py +++ b/tests/api_connexion/endpoints/test_backfill_endpoint.py @@ -29,9 +29,8 @@ from airflow.operators.empty import EmptyOperator from airflow.utils import timezone from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.db import ( clear_db_backfills, clear_db_dags, clear_db_runs, diff --git a/tests/api_connexion/endpoints/test_config_endpoint.py b/tests/api_connexion/endpoints/test_config_endpoint.py index c46a6571c23b9..86046ec3f37c3 100644 --- a/tests/api_connexion/endpoints/test_config_endpoint.py +++ b/tests/api_connexion/endpoints/test_config_endpoint.py @@ -21,8 +21,8 @@ import pytest -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_connection_endpoint.py b/tests/api_connexion/endpoints/test_connection_endpoint.py index fb2d53074cd87..b82fac609bd9e 100644 --- a/tests/api_connexion/endpoints/test_connection_endpoint.py +++ b/tests/api_connexion/endpoints/test_connection_endpoint.py @@ -25,11 +25,10 @@ from airflow.models import Connection from airflow.secrets.environment_variables import CONN_ENV_PREFIX from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_connections -from dev.tests_common.test_utils.www import _check_last_log +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py index 86e0a3b4b3ec6..fd3b8b7d90d17 100644 --- a/tests/api_connexion/endpoints/test_dag_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_endpoint.py @@ -30,11 +30,10 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.session import provide_session from airflow.utils.state import TaskInstanceState - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from dev.tests_common.test_utils.www import _check_last_log +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_parsing.py b/tests/api_connexion/endpoints/test_dag_parsing.py index 0fca8ee63c973..53648150c2c01 100644 --- a/tests/api_connexion/endpoints/test_dag_parsing.py +++ b/tests/api_connexion/endpoints/test_dag_parsing.py @@ -24,9 +24,8 @@ from airflow.models import DagBag from airflow.models.dagbag import DagPriorityParsingRequest - -from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_dag_parsing_requests +from tests_common.test_utils.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.db import clear_db_dag_parsing_requests pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_run_endpoint.py b/tests/api_connexion/endpoints/test_dag_run_endpoint.py index b881f5ea98638..bad9ff1481c4d 100644 --- a/tests/api_connexion/endpoints/test_dag_run_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_run_endpoint.py @@ -34,12 +34,11 @@ from airflow.utils.session import create_session, provide_session from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from dev.tests_common.test_utils.www import _check_last_log +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.www import _check_last_log if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/api_connexion/endpoints/test_dag_source_endpoint.py b/tests/api_connexion/endpoints/test_dag_source_endpoint.py index 80f32dc715a3c..8a5c91ae670a3 100644 --- a/tests/api_connexion/endpoints/test_dag_source_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_source_endpoint.py @@ -23,9 +23,8 @@ import pytest from airflow.models import DagBag - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py index fe563b9444030..179662cd8f4ed 100644 --- a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py @@ -26,9 +26,8 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_dag_warning_endpoint.py b/tests/api_connexion/endpoints/test_dag_warning_endpoint.py index b9c00991413bb..f3521ed883c73 100644 --- a/tests/api_connexion/endpoints/test_dag_warning_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_warning_endpoint.py @@ -23,9 +23,8 @@ from airflow.models.dag import DagModel from airflow.models.dagwarning import DagWarning from airflow.utils.session import create_session - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_event_log_endpoint.py b/tests/api_connexion/endpoints/test_event_log_endpoint.py index 42a610890ede2..bcd668c93cce6 100644 --- a/tests/api_connexion/endpoints/test_event_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_event_log_endpoint.py @@ -21,10 +21,9 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import Log from airflow.utils import timezone - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_logs +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_logs pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_extra_link_endpoint.py b/tests/api_connexion/endpoints/test_extra_link_endpoint.py index 373571a42ee47..30b8234cd746a 100644 --- a/tests/api_connexion/endpoints/test_extra_link_endpoint.py +++ b/tests/api_connexion/endpoints/test_extra_link_endpoint.py @@ -30,12 +30,11 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom -from dev.tests_common.test_utils.mock_operators import CustomOperator -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager +from tests_common.test_utils.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink +from tests_common.test_utils.db import clear_db_runs, clear_db_xcom +from tests_common.test_utils.mock_operators import CustomOperator +from tests_common.test_utils.mock_plugins import mock_plugin_manager if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/api_connexion/endpoints/test_import_error_endpoint.py b/tests/api_connexion/endpoints/test_import_error_endpoint.py index f4f7d03b323f2..0e2ea9f97200c 100644 --- a/tests/api_connexion/endpoints/test_import_error_endpoint.py +++ b/tests/api_connexion/endpoints/test_import_error_endpoint.py @@ -23,11 +23,10 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.utils import timezone from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.compat import ParseImportError -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_import_errors +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.compat import ParseImportError +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_import_errors pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_log_endpoint.py b/tests/api_connexion/endpoints/test_log_endpoint.py index 21d4e8feb83ad..ef542a74d7a8f 100644 --- a/tests/api_connexion/endpoints/test_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_log_endpoint.py @@ -32,9 +32,8 @@ from airflow.operators.empty import EmptyOperator from airflow.utils import timezone from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.db import clear_db_runs pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py index e41bbcc6c2e88..843de41f1b401 100644 --- a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py @@ -32,10 +32,9 @@ from airflow.utils.session import provide_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields -from dev.tests_common.test_utils.mock_operators import MockOperator +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from tests_common.test_utils.mock_operators import MockOperator pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_plugin_endpoint.py b/tests/api_connexion/endpoints/test_plugin_endpoint.py index 2831af06bf3d9..b04ec7b7d5823 100644 --- a/tests/api_connexion/endpoints/test_plugin_endpoint.py +++ b/tests/api_connexion/endpoints/test_plugin_endpoint.py @@ -28,11 +28,10 @@ from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.timetables.base import Timetable from airflow.utils.module_loading import qualname - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.compat import BaseOperatorLink -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.compat import BaseOperatorLink +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.mock_plugins import mock_plugin_manager pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_pool_endpoint.py b/tests/api_connexion/endpoints/test_pool_endpoint.py index 455c417bd2581..d19dbf836d025 100644 --- a/tests/api_connexion/endpoints/test_pool_endpoint.py +++ b/tests/api_connexion/endpoints/test_pool_endpoint.py @@ -21,11 +21,10 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models.pool import Pool from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_pools -from dev.tests_common.test_utils.www import _check_last_log +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_pools +from tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_provider_endpoint.py b/tests/api_connexion/endpoints/test_provider_endpoint.py index 92e14887ec038..c526aabe8c005 100644 --- a/tests/api_connexion/endpoints/test_provider_endpoint.py +++ b/tests/api_connexion/endpoints/test_provider_endpoint.py @@ -21,8 +21,7 @@ import pytest from airflow.providers_manager import ProviderInfo - -from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.api_connexion_utils import create_user, delete_user pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_task_endpoint.py b/tests/api_connexion/endpoints/test_task_endpoint.py index 60237946a82ca..567394208510b 100644 --- a/tests/api_connexion/endpoints/test_task_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_endpoint.py @@ -27,9 +27,8 @@ from airflow.models.expandinput import EXPAND_INPUT_EMPTY from airflow.models.serialized_dag import SerializedDagModel from airflow.operators.empty import EmptyOperator - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_task_instance_endpoint.py index 809c2fab9aa2d..bec52292fbe46 100644 --- a/tests/api_connexion/endpoints/test_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_instance_endpoint.py @@ -35,10 +35,9 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields -from dev.tests_common.test_utils.www import _check_last_log +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_variable_endpoint.py b/tests/api_connexion/endpoints/test_variable_endpoint.py index e4302636184d3..640251ebc14f3 100644 --- a/tests/api_connexion/endpoints/test_variable_endpoint.py +++ b/tests/api_connexion/endpoints/test_variable_endpoint.py @@ -22,11 +22,10 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import Variable - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_variables -from dev.tests_common.test_utils.www import _check_last_log +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_variables +from tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_xcom_endpoint.py b/tests/api_connexion/endpoints/test_xcom_endpoint.py index 0293e672c47d2..12d56ae9894ea 100644 --- a/tests/api_connexion/endpoints/test_xcom_endpoint.py +++ b/tests/api_connexion/endpoints/test_xcom_endpoint.py @@ -30,10 +30,9 @@ from airflow.utils.session import create_session from airflow.utils.timezone import utcnow from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom +from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_connection_schema.py b/tests/api_connexion/schemas/test_connection_schema.py index d7a0de4e89f05..e52b573875138 100644 --- a/tests/api_connexion/schemas/test_connection_schema.py +++ b/tests/api_connexion/schemas/test_connection_schema.py @@ -30,8 +30,7 @@ ) from airflow.models import Connection from airflow.utils.session import create_session, provide_session - -from dev.tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.db import clear_db_connections pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_dag_run_schema.py b/tests/api_connexion/schemas/test_dag_run_schema.py index b874ade4442ff..929a8af451112 100644 --- a/tests/api_connexion/schemas/test_dag_run_schema.py +++ b/tests/api_connexion/schemas/test_dag_run_schema.py @@ -29,9 +29,8 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/api_connexion/schemas/test_dataset_schema.py b/tests/api_connexion/schemas/test_dataset_schema.py index 769de6f1b10eb..669b87f6cf636 100644 --- a/tests/api_connexion/schemas/test_dataset_schema.py +++ b/tests/api_connexion/schemas/test_dataset_schema.py @@ -30,8 +30,7 @@ from airflow.assets import Asset from airflow.models.asset import AssetAliasModel, AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator - -from dev.tests_common.test_utils.db import clear_db_assets, clear_db_dags +from tests_common.test_utils.db import clear_db_assets, clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_error_schema.py b/tests/api_connexion/schemas/test_error_schema.py index 5c358f2e2c374..5f31dfe2f1460 100644 --- a/tests/api_connexion/schemas/test_error_schema.py +++ b/tests/api_connexion/schemas/test_error_schema.py @@ -25,9 +25,8 @@ ) from airflow.utils import timezone from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.compat import ParseImportError -from dev.tests_common.test_utils.db import clear_db_import_errors +from tests_common.test_utils.compat import ParseImportError +from tests_common.test_utils.db import clear_db_import_errors pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_plugin_schema.py b/tests/api_connexion/schemas/test_plugin_schema.py index ee079ce664348..df550a8301bdd 100644 --- a/tests/api_connexion/schemas/test_plugin_schema.py +++ b/tests/api_connexion/schemas/test_plugin_schema.py @@ -28,8 +28,7 @@ ) from airflow.hooks.base import BaseHook from airflow.plugins_manager import AirflowPlugin - -from dev.tests_common.test_utils.compat import BaseOperatorLink +from tests_common.test_utils.compat import BaseOperatorLink class PluginHook(BaseHook): ... diff --git a/tests/api_connexion/schemas/test_pool_schemas.py b/tests/api_connexion/schemas/test_pool_schemas.py index 23788426c5018..3e5f3d7f02904 100644 --- a/tests/api_connexion/schemas/test_pool_schemas.py +++ b/tests/api_connexion/schemas/test_pool_schemas.py @@ -21,8 +21,7 @@ from airflow.api_connexion.schemas.pool_schema import PoolCollection, pool_collection_schema, pool_schema from airflow.models.pool import Pool from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.db import clear_db_pools +from tests_common.test_utils.db import clear_db_pools pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_xcom_schema.py b/tests/api_connexion/schemas/test_xcom_schema.py index 06c36c31b7767..366d707878b9c 100644 --- a/tests/api_connexion/schemas/test_xcom_schema.py +++ b/tests/api_connexion/schemas/test_xcom_schema.py @@ -30,8 +30,7 @@ from airflow.models import DagRun, XCom from airflow.utils.dates import parse_execution_date from airflow.utils.session import create_session - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/test_auth.py b/tests/api_connexion/test_auth.py index e568d618ad662..65e98c18c974d 100644 --- a/tests/api_connexion/test_auth.py +++ b/tests/api_connexion/test_auth.py @@ -22,10 +22,9 @@ from airflow.auth.managers.simple.simple_auth_manager import SimpleAuthManager from airflow.auth.managers.simple.user import SimpleAuthManagerUser - -from dev.tests_common.test_utils.api_connexion_utils import assert_401 -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_pools +from tests_common.test_utils.api_connexion_utils import assert_401 +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_pools pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/test_parameters.py b/tests/api_connexion/test_parameters.py index 57c9c05415f65..26fdcfa7aeca0 100644 --- a/tests/api_connexion/test_parameters.py +++ b/tests/api_connexion/test_parameters.py @@ -29,8 +29,7 @@ validate_istimezone, ) from airflow.utils import timezone - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars class TestValidateIsTimezone: diff --git a/tests/api_connexion/test_security.py b/tests/api_connexion/test_security.py index 2051cf613510e..bebf2ec340a60 100644 --- a/tests/api_connexion/test_security.py +++ b/tests/api_connexion/test_security.py @@ -18,7 +18,7 @@ import pytest -from dev.tests_common.test_utils.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.api_connexion_utils import create_user, delete_user pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_fastapi/views/public/test_connections.py b/tests/api_fastapi/views/public/test_connections.py index a5cb301a43136..a0da0d2b9b28e 100644 --- a/tests/api_fastapi/views/public/test_connections.py +++ b/tests/api_fastapi/views/public/test_connections.py @@ -20,8 +20,7 @@ from airflow.models import Connection from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.db import clear_db_connections pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/public/test_dag_run.py b/tests/api_fastapi/views/public/test_dag_run.py index 377aca4553d86..3c60b307bbbd2 100644 --- a/tests/api_fastapi/views/public/test_dag_run.py +++ b/tests/api_fastapi/views/public/test_dag_run.py @@ -25,8 +25,7 @@ from airflow.utils.session import provide_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunTriggeredByType, DagRunType - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/public/test_dags.py b/tests/api_fastapi/views/public/test_dags.py index 8bc10407e6287..cd1809cb70d3d 100644 --- a/tests/api_fastapi/views/public/test_dags.py +++ b/tests/api_fastapi/views/public/test_dags.py @@ -27,8 +27,7 @@ from airflow.utils.session import provide_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/public/test_variables.py b/tests/api_fastapi/views/public/test_variables.py index 58a09538a971a..21f86c428b397 100644 --- a/tests/api_fastapi/views/public/test_variables.py +++ b/tests/api_fastapi/views/public/test_variables.py @@ -20,8 +20,7 @@ from airflow.models.variable import Variable from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.db import clear_db_variables +from tests_common.test_utils.db import clear_db_variables pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/ui/test_assets.py b/tests/api_fastapi/views/ui/test_assets.py index d0e93c709b076..d06631ec88b59 100644 --- a/tests/api_fastapi/views/ui/test_assets.py +++ b/tests/api_fastapi/views/ui/test_assets.py @@ -20,8 +20,7 @@ from airflow.assets import Asset from airflow.operators.empty import EmptyOperator - -from dev.tests_common.test_utils.db import initial_db_init +from tests_common.test_utils.db import initial_db_init pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/views/ui/test_dashboard.py b/tests/api_fastapi/views/ui/test_dashboard.py index bee683535704f..f4a061f1dd0ff 100644 --- a/tests/api_fastapi/views/ui/test_dashboard.py +++ b/tests/api_fastapi/views/ui/test_dashboard.py @@ -26,8 +26,7 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/api_internal/endpoints/test_rpc_api_endpoint.py b/tests/api_internal/endpoints/test_rpc_api_endpoint.py index 364b0f44eaf27..72ce2afe4c19f 100644 --- a/tests/api_internal/endpoints/test_rpc_api_endpoint.py +++ b/tests/api_internal/endpoints/test_rpc_api_endpoint.py @@ -34,9 +34,8 @@ from airflow.utils.jwt_signer import JWTSigner from airflow.utils.state import State from airflow.www import app - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules # Note: Sounds a bit strange to disable internal API tests in isolation mode but... # As long as the test is modelled to run its own internal API endpoints, it is conflicting diff --git a/tests/api_internal/test_internal_api_call.py b/tests/api_internal/test_internal_api_call.py index 02ae2d9f55125..880e1e89b0388 100644 --- a/tests/api_internal/test_internal_api_call.py +++ b/tests/api_internal/test_internal_api_call.py @@ -34,8 +34,7 @@ from airflow.serialization.serialized_objects import BaseSerialization from airflow.settings import _ENABLE_AIP_44 from airflow.utils.state import State - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars if TYPE_CHECKING: from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic diff --git a/tests/assets/test_manager.py b/tests/assets/test_manager.py index cd4563d9a9c4e..f2dcc88b8338b 100644 --- a/tests/assets/test_manager.py +++ b/tests/assets/test_manager.py @@ -38,6 +38,7 @@ from airflow.models.dag import DagModel from airflow.models.dagbag import DagPriorityParsingRequest from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic + from tests.listeners import asset_listener pytestmark = pytest.mark.db_test @@ -48,7 +49,7 @@ @pytest.fixture def clear_assets(): - from dev.tests_common.test_utils.db import clear_db_assets + from tests_common.test_utils.db import clear_db_assets clear_db_assets() yield diff --git a/tests/assets/tests_asset.py b/tests/assets/tests_asset.py index 10d91e7a00fb7..0bcfb83e88a79 100644 --- a/tests/assets/tests_asset.py +++ b/tests/assets/tests_asset.py @@ -43,7 +43,7 @@ @pytest.fixture def clear_assets(): - from dev.tests_common.test_utils.db import clear_db_assets + from tests_common.test_utils.db import clear_db_assets clear_db_assets() yield diff --git a/tests/auth/managers/simple/views/test_auth.py b/tests/auth/managers/simple/views/test_auth.py index f61a9278f79a5..49f6237fc7e44 100644 --- a/tests/auth/managers/simple/views/test_auth.py +++ b/tests/auth/managers/simple/views/test_auth.py @@ -23,8 +23,7 @@ from airflow.auth.managers.simple.simple_auth_manager import SimpleAuthManager from airflow.www import app as application - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars @pytest.fixture diff --git a/tests/cli/commands/test_backfill_command.py b/tests/cli/commands/test_backfill_command.py index c01e1e4f9d07b..e798436f24445 100644 --- a/tests/cli/commands/test_backfill_command.py +++ b/tests/cli/commands/test_backfill_command.py @@ -28,8 +28,7 @@ from airflow.cli import cli_parser from airflow.models import DagBag from airflow.utils import timezone - -from dev.tests_common.test_utils.db import clear_db_backfills, clear_db_dags, clear_db_runs +from tests_common.test_utils.db import clear_db_backfills, clear_db_dags, clear_db_runs DEFAULT_DATE = timezone.make_aware(datetime(2015, 1, 1), timezone=timezone.utc) if pendulum.__version__.startswith("3"): diff --git a/tests/cli/commands/test_celery_command.py b/tests/cli/commands/test_celery_command.py index c29d835d8ee07..7e20a9c933f94 100644 --- a/tests/cli/commands/test_celery_command.py +++ b/tests/cli/commands/test_celery_command.py @@ -30,8 +30,7 @@ from airflow.cli.commands import celery_command from airflow.configuration import conf from airflow.executors import executor_loader - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_config_command.py b/tests/cli/commands/test_config_command.py index 90d38fa27ad03..3202990664285 100644 --- a/tests/cli/commands/test_config_command.py +++ b/tests/cli/commands/test_config_command.py @@ -22,8 +22,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import config_command - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars STATSD_CONFIG_BEGIN_WITH = "# `StatsD `" diff --git a/tests/cli/commands/test_connection_command.py b/tests/cli/commands/test_connection_command.py index 06578eb1492c1..a04775433f468 100644 --- a/tests/cli/commands/test_connection_command.py +++ b/tests/cli/commands/test_connection_command.py @@ -33,8 +33,7 @@ from airflow.models import Connection from airflow.utils.db import merge_conn from airflow.utils.session import create_session - -from dev.tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.db import clear_db_connections pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py index e9d10a2e33c18..fe48ec5f357cb 100644 --- a/tests/cli/commands/test_dag_command.py +++ b/tests/cli/commands/test_dag_command.py @@ -46,10 +46,10 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests.models import TEST_DAGS_FOLDER +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests.models import TEST_DAGS_FOLDER DEFAULT_DATE = timezone.make_aware(datetime(2015, 1, 1), timezone=timezone.utc) if pendulum.__version__.startswith("3"): diff --git a/tests/cli/commands/test_dag_processor_command.py b/tests/cli/commands/test_dag_processor_command.py index 643710ce1e415..71a30dcd7d89b 100644 --- a/tests/cli/commands/test_dag_processor_command.py +++ b/tests/cli/commands/test_dag_processor_command.py @@ -24,8 +24,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import dag_processor_command from airflow.configuration import conf - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_fastapi_api_command.py b/tests/cli/commands/test_fastapi_api_command.py index 2de4b729bfb0e..b1a02b3b186e1 100644 --- a/tests/cli/commands/test_fastapi_api_command.py +++ b/tests/cli/commands/test_fastapi_api_command.py @@ -27,6 +27,7 @@ from airflow.cli.commands import fastapi_api_command from airflow.exceptions import AirflowConfigException + from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass console = Console(width=400, color_system="standard") diff --git a/tests/cli/commands/test_info_command.py b/tests/cli/commands/test_info_command.py index c6bea2d8a5bb8..ca5db5819c04b 100644 --- a/tests/cli/commands/test_info_command.py +++ b/tests/cli/commands/test_info_command.py @@ -32,8 +32,7 @@ from airflow.config_templates import airflow_local_settings from airflow.logging_config import configure_logging from airflow.version import version as airflow_version - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars def capture_show_output(instance): diff --git a/tests/cli/commands/test_internal_api_command.py b/tests/cli/commands/test_internal_api_command.py index 11123c7c211a1..f3dd664d1a273 100644 --- a/tests/cli/commands/test_internal_api_command.py +++ b/tests/cli/commands/test_internal_api_command.py @@ -31,9 +31,9 @@ from airflow.cli.commands import internal_api_command from airflow.cli.commands.internal_api_command import GunicornMonitor from airflow.settings import _ENABLE_AIP_44 -from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass +from tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.config import conf_vars +from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass console = Console(width=400, color_system="standard") diff --git a/tests/cli/commands/test_jobs_command.py b/tests/cli/commands/test_jobs_command.py index 61540a68c51c6..1c52aa71334e4 100644 --- a/tests/cli/commands/test_jobs_command.py +++ b/tests/cli/commands/test_jobs_command.py @@ -27,8 +27,7 @@ from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.utils.session import create_session from airflow.utils.state import JobState, State - -from dev.tests_common.test_utils.db import clear_db_jobs +from tests_common.test_utils.db import clear_db_jobs @pytest.mark.skip_if_database_isolation_mode diff --git a/tests/cli/commands/test_kerberos_command.py b/tests/cli/commands/test_kerberos_command.py index 4a5e73ece788f..5d2f2e513bcc5 100644 --- a/tests/cli/commands/test_kerberos_command.py +++ b/tests/cli/commands/test_kerberos_command.py @@ -23,8 +23,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import kerberos_command from airflow.security.kerberos import KerberosMode - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_kubernetes_command.py b/tests/cli/commands/test_kubernetes_command.py index 1e53bdc77d8d8..dbde7a89664ed 100644 --- a/tests/cli/commands/test_kubernetes_command.py +++ b/tests/cli/commands/test_kubernetes_command.py @@ -28,8 +28,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import kubernetes_command from airflow.executors import executor_loader - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_plugins_command.py b/tests/cli/commands/test_plugins_command.py index 703b6879d7a3a..00d6cee5a0b5d 100644 --- a/tests/cli/commands/test_plugins_command.py +++ b/tests/cli/commands/test_plugins_command.py @@ -28,9 +28,9 @@ from airflow.hooks.base import BaseHook from airflow.listeners.listener import get_listener_manager from airflow.plugins_manager import AirflowPlugin -from tests.plugins.test_plugin import AirflowTestPlugin as ComplexAirflowPlugin +from tests_common.test_utils.mock_plugins import mock_plugin_manager -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager +from tests.plugins.test_plugin import AirflowTestPlugin as ComplexAirflowPlugin pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] @@ -89,15 +89,15 @@ def test_should_display_one_plugin(self): } ], "global_operator_extra_links": [ - "", - "", + "", + "", ], "timetables": ["tests.plugins.test_plugin.CustomCronDataIntervalTimetable"], "operator_extra_links": [ - "", - "", - "", - "", + "", + "", + "", + "", ], "hooks": ["tests.plugins.test_plugin.PluginHook"], "listeners": [ diff --git a/tests/cli/commands/test_rotate_fernet_key_command.py b/tests/cli/commands/test_rotate_fernet_key_command.py index d638c2010e187..473c911d9a10c 100644 --- a/tests/cli/commands/test_rotate_fernet_key_command.py +++ b/tests/cli/commands/test_rotate_fernet_key_command.py @@ -26,9 +26,8 @@ from airflow.hooks.base import BaseHook from airflow.models import Connection, Variable from airflow.utils.session import provide_session - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_connections, clear_db_variables +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_connections, clear_db_variables pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_scheduler_command.py b/tests/cli/commands/test_scheduler_command.py index 8fce5d6e289b3..a6adb514f203b 100644 --- a/tests/cli/commands/test_scheduler_command.py +++ b/tests/cli/commands/test_scheduler_command.py @@ -29,8 +29,7 @@ from airflow.executors import executor_loader from airflow.utils.scheduler_health import HealthServer, serve_health_check from airflow.utils.serve_logs import serve_logs - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index fb512792eb324..823172cffc555 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -51,10 +51,9 @@ from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_pools, clear_db_runs +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_pools, clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/cli/commands/test_variable_command.py b/tests/cli/commands/test_variable_command.py index d35184c73c80a..630e7d321d3e0 100644 --- a/tests/cli/commands/test_variable_command.py +++ b/tests/cli/commands/test_variable_command.py @@ -29,8 +29,7 @@ from airflow.cli.commands import variable_command from airflow.models import Variable from airflow.utils.session import create_session - -from dev.tests_common.test_utils.db import clear_db_variables +from tests_common.test_utils.db import clear_db_variables pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_webserver_command.py b/tests/cli/commands/test_webserver_command.py index b72f4e4596035..a2cba19c9a216 100644 --- a/tests/cli/commands/test_webserver_command.py +++ b/tests/cli/commands/test_webserver_command.py @@ -30,9 +30,9 @@ from airflow.cli import cli_parser from airflow.cli.commands import webserver_command from airflow.cli.commands.webserver_command import GunicornMonitor -from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass +from tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.config import conf_vars +from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass console = Console(width=400, color_system="standard") diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 61e27b0e74de4..83aeecdbb1ffb 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -25,8 +25,7 @@ from airflow.models.dagbag import DagBag from airflow.providers.celery.executors import celery_executor, celery_kubernetes_executor from airflow.providers.cncf.kubernetes.executors import kubernetes_executor, local_kubernetes_executor - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars # Create custom executors here because conftest is imported first custom_executor_module = type(sys)("custom_executor") diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index 46ac36803face..f777908c23d19 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -43,8 +43,7 @@ from airflow.executors.local_executor import LocalExecutor from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/conftest.py b/tests/conftest.py index f956865e7a1c6..c5c6441f0b55f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,14 +26,14 @@ # unit test mode config is set as early as possible. assert "airflow" not in sys.modules, "No airflow module can be imported before these lines" -pytest_plugins = "dev.tests_common.pytest_plugin" +pytest_plugins = "tests_common.pytest_plugin" # Ignore files that are really test dags to be ignored by pytest collect_ignore = [ "tests/dags/subdir1/test_ignore_this.py", "tests/dags/test_invalid_dup_task.py", "tests/dags_corrupted/test_impersonation_custom.py", - "dev.tests_common.test_utils/perf/dags/elastic_dag.py", + "tests_common.test_utils/perf/dags/elastic_dag.py", ] diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 0690a5f0182dc..1f376c955f503 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -42,6 +42,9 @@ write_default_airflow_configuration_if_needed, ) from airflow.providers_manager import ProvidersManager +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.reset_warning_registry import reset_warning_registry + from tests.utils.test_config import ( remove_all_configurations, set_deprecated_options, @@ -49,9 +52,6 @@ use_config, ) -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.reset_warning_registry import reset_warning_registry - HOME_DIR = os.path.expanduser("~") # The conf has been updated with sql_alchemy_con and deactivate_stale_dags_interval to test the diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 13b85a9ef1700..2186d04f195b2 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -32,8 +32,7 @@ from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/core/test_example_dags_system.py b/tests/core/test_example_dags_system.py index bd34d9bb15919..a0d2a60d85299 100644 --- a/tests/core/test_example_dags_system.py +++ b/tests/core/test_example_dags_system.py @@ -28,9 +28,8 @@ from airflow.utils.module_loading import import_string from airflow.utils.state import DagRunState from airflow.utils.trigger_rule import TriggerRule - -from dev.tests_common.test_utils.system_tests import get_test_run -from dev.tests_common.test_utils.system_tests_class import SystemTest +from tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests_class import SystemTest def fail(): diff --git a/tests/core/test_impersonation_tests.py b/tests/core/test_impersonation_tests.py index 8350e95a8f4fd..0683b619d60f4 100644 --- a/tests/core/test_impersonation_tests.py +++ b/tests/core/test_impersonation_tests.py @@ -32,8 +32,7 @@ from airflow.utils.db import add_default_pool_if_not_exists from airflow.utils.state import State from airflow.utils.timezone import datetime - -from dev.tests_common.test_utils import db +from tests_common.test_utils import db # The entire module into the quarantined mark, this might have unpredictable side effects to other tests # and should be moved into the isolated environment into the future. diff --git a/tests/core/test_logging_config.py b/tests/core/test_logging_config.py index a316130ecdabe..aeb6d36ced04f 100644 --- a/tests/core/test_logging_config.py +++ b/tests/core/test_logging_config.py @@ -29,8 +29,7 @@ import pytest from airflow.configuration import conf - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars SETTINGS_FILE_VALID = """ LOGGING_CONFIG = { diff --git a/tests/core/test_otel_tracer.py b/tests/core/test_otel_tracer.py index b9612c49dae92..36ebb9f09ce07 100644 --- a/tests/core/test_otel_tracer.py +++ b/tests/core/test_otel_tracer.py @@ -26,8 +26,7 @@ from airflow.traces import TRACEPARENT, TRACESTATE, otel_tracer, utils from airflow.traces.tracer import Trace - -from dev.tests_common.test_utils.config import env_vars +from tests_common.test_utils.config import env_vars @pytest.fixture diff --git a/tests/core/test_sentry.py b/tests/core/test_sentry.py index c67b49980aae2..972e45dbba247 100644 --- a/tests/core/test_sentry.py +++ b/tests/core/test_sentry.py @@ -30,8 +30,7 @@ from airflow.utils import timezone from airflow.utils.module_loading import import_string from airflow.utils.state import State - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars EXECUTION_DATE = timezone.utcnow() SCHEDULE_INTERVAL = datetime.timedelta(days=1) diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index 619d64630029d..27b40ad7d1fae 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -33,8 +33,7 @@ from airflow.exceptions import AirflowClusterPolicyViolation, AirflowConfigException from airflow.settings import _ENABLE_AIP_44, TracebackSession, is_usage_data_collection_enabled from airflow.utils.session import create_session - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars SETTINGS_FILE_POLICY = """ def test_policy(task_instance): diff --git a/tests/core/test_sqlalchemy_config.py b/tests/core/test_sqlalchemy_config.py index 5fed8745fd085..99fb90d3ba2e3 100644 --- a/tests/core/test_sqlalchemy_config.py +++ b/tests/core/test_sqlalchemy_config.py @@ -25,8 +25,7 @@ from airflow import settings from airflow.api_internal.internal_api_call import InternalApiConfig from airflow.exceptions import AirflowConfigException - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars SQL_ALCHEMY_CONNECT_ARGS = {"test": 43503, "dict": {"is": 1, "supported": "too"}} diff --git a/tests/core/test_stats.py b/tests/core/test_stats.py index 9a218010a189e..e2969204d1d32 100644 --- a/tests/core/test_stats.py +++ b/tests/core/test_stats.py @@ -36,8 +36,7 @@ PatternAllowListValidator, PatternBlockListValidator, ) - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars class CustomStatsd(statsd.StatsClient): diff --git a/tests/dag_processing/test_job_runner.py b/tests/dag_processing/test_job_runner.py index 81b94dd18b18a..e16ec2d1dddcc 100644 --- a/tests/dag_processing/test_job_runner.py +++ b/tests/dag_processing/test_job_runner.py @@ -58,12 +58,9 @@ from airflow.utils import timezone from airflow.utils.net import get_hostname from airflow.utils.session import create_session -from tests.core.test_logging_config import SETTINGS_FILE_VALID, settings_context -from tests.models import TEST_DAGS_FOLDER - -from dev.tests_common.test_utils.compat import ParseImportError -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.compat import ParseImportError +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import ( clear_db_callbacks, clear_db_dags, clear_db_import_errors, @@ -71,6 +68,9 @@ clear_db_serialized_dags, ) +from tests.core.test_logging_config import SETTINGS_FILE_VALID, settings_context +from tests.models import TEST_DAGS_FOLDER + pytestmark = pytest.mark.db_test logger = logging.getLogger(__name__) diff --git a/tests/dag_processing/test_processor.py b/tests/dag_processing/test_processor.py index c3a0f5a26e329..40412a131b40b 100644 --- a/tests/dag_processing/test_processor.py +++ b/tests/dag_processing/test_processor.py @@ -39,11 +39,10 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError -from dev.tests_common.test_utils.config import conf_vars, env_vars -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError +from tests_common.test_utils.config import conf_vars, env_vars +from tests_common.test_utils.db import ( clear_db_dags, clear_db_import_errors, clear_db_jobs, @@ -51,7 +50,7 @@ clear_db_runs, clear_db_serialized_dags, ) -from dev.tests_common.test_utils.mock_executor import MockExecutor +from tests_common.test_utils.mock_executor import MockExecutor if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/dags/test_external_task_sensor_check_existense.py b/tests/dags/test_external_task_sensor_check_existense.py index 05bd82c509f98..9de992c073b91 100644 --- a/tests/dags/test_external_task_sensor_check_existense.py +++ b/tests/dags/test_external_task_sensor_check_existense.py @@ -20,6 +20,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator from airflow.sensors.external_task import ExternalTaskSensor + from tests.models import DEFAULT_DATE with DAG( diff --git a/tests/dags/test_miscellaneous.py b/tests/dags/test_miscellaneous.py index b08e61e3bbddc..e54ae1b458491 100644 --- a/tests/dags/test_miscellaneous.py +++ b/tests/dags/test_miscellaneous.py @@ -23,8 +23,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator - -from dev.tests_common.test_utils.compat import BashOperator +from tests_common.test_utils.compat import BashOperator args = { "owner": "airflow", diff --git a/tests/dags/test_sensor.py b/tests/dags/test_sensor.py index 7c96160402663..5dc8fbe020901 100644 --- a/tests/dags/test_sensor.py +++ b/tests/dags/test_sensor.py @@ -21,8 +21,7 @@ from airflow.decorators import task from airflow.models.dag import DAG from airflow.utils import timezone - -from dev.tests_common.test_utils.compat import DateTimeSensor +from tests_common.test_utils.compat import DateTimeSensor with DAG( dag_id="test_sensor", start_date=datetime.datetime(2022, 1, 1), catchup=False, schedule="@once" diff --git a/tests/decorators/test_bash.py b/tests/decorators/test_bash.py index da79fb4cca0fb..dd2ce6223c53c 100644 --- a/tests/decorators/test_bash.py +++ b/tests/decorators/test_bash.py @@ -29,8 +29,7 @@ from airflow.models.renderedtifields import RenderedTaskInstanceFields from airflow.utils import timezone from airflow.utils.types import NOTSET - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields DEFAULT_DATE = timezone.datetime(2023, 1, 1) diff --git a/tests/decorators/test_mapped.py b/tests/decorators/test_mapped.py index 2d3747b5f34ef..9bd59f03be7e1 100644 --- a/tests/decorators/test_mapped.py +++ b/tests/decorators/test_mapped.py @@ -22,6 +22,7 @@ from airflow.decorators import task from airflow.models.dag import DAG from airflow.utils.task_group import TaskGroup + from tests.models import DEFAULT_DATE diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py index d34fbbf552b30..4307b29afb48b 100644 --- a/tests/decorators/test_python.py +++ b/tests/decorators/test_python.py @@ -39,9 +39,9 @@ from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from airflow.utils.xcom import XCOM_RETURN_KEY -from tests.operators.test_python import BasePythonTest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.operators.test_python import BasePythonTest if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/executors/test_executor_loader.py b/tests/executors/test_executor_loader.py index d9bf81dd3116d..d6e783282d0ef 100644 --- a/tests/executors/test_executor_loader.py +++ b/tests/executors/test_executor_loader.py @@ -29,8 +29,7 @@ from airflow.executors.local_executor import LocalExecutor from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/tests/integration/cli/commands/test_celery_command.py b/tests/integration/cli/commands/test_celery_command.py index 186addedb1164..bb6c097578a6c 100644 --- a/tests/integration/cli/commands/test_celery_command.py +++ b/tests/integration/cli/commands/test_celery_command.py @@ -25,8 +25,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import celery_command from airflow.executors import executor_loader - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars @pytest.mark.integration("celery") diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 169d3e91356c9..4dd5e2bdc94d9 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -45,8 +45,7 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.state import State, TaskInstanceState - -from dev.tests_common.test_utils import db +from tests_common.test_utils import db logger = logging.getLogger(__name__) diff --git a/tests/integration/security/test_kerberos.py b/tests/integration/security/test_kerberos.py index d16b8bc332c7d..7267b69442d78 100644 --- a/tests/integration/security/test_kerberos.py +++ b/tests/integration/security/test_kerberos.py @@ -26,8 +26,7 @@ from airflow.security import kerberos from airflow.security.kerberos import renew_from_kt - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars @pytest.mark.integration("kerberos") diff --git a/tests/jobs/test_base_job.py b/tests/jobs/test_base_job.py index 20079f4f235e5..058539868f1c0 100644 --- a/tests/jobs/test_base_job.py +++ b/tests/jobs/test_base_job.py @@ -32,11 +32,11 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State +from tests_common.test_utils.config import conf_vars + from tests.listeners import lifecycle_listener from tests.utils.test_helpers import MockJobRunner, SchedulerJobRunner, TriggererJobRunner -from dev.tests_common.test_utils.config import conf_vars - if TYPE_CHECKING: from airflow.serialization.pydantic.job import JobPydantic diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py index 577fddd84d2b7..7d99c593511dd 100644 --- a/tests/jobs/test_local_task_job.py +++ b/tests/jobs/test_local_task_job.py @@ -52,12 +52,11 @@ from airflow.utils.state import State from airflow.utils.timeout import timeout from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.mock_executor import MockExecutor +from tests_common.test_utils import db +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.mock_executor import MockExecutor if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 7eae1639d0e1b..f47f4497d5aab 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -69,15 +69,10 @@ from airflow.utils.session import create_session, provide_session from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState from airflow.utils.types import DagRunType -from tests.listeners import dag_listener -from tests.listeners.test_listeners import get_listener_manager -from tests.models import TEST_DAGS_FOLDER -from tests.utils.test_timezone import UTC - -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars, env_vars -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars, env_vars +from tests_common.test_utils.db import ( clear_db_assets, clear_db_backfills, clear_db_dags, @@ -89,8 +84,13 @@ clear_db_sla_miss, set_default_pool_slots, ) -from dev.tests_common.test_utils.mock_executor import MockExecutor -from dev.tests_common.test_utils.mock_operators import CustomOperator +from tests_common.test_utils.mock_executor import MockExecutor +from tests_common.test_utils.mock_operators import CustomOperator + +from tests.listeners import dag_listener +from tests.listeners.test_listeners import get_listener_manager +from tests.models import TEST_DAGS_FOLDER +from tests.utils.test_timezone import UTC if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/jobs/test_triggerer_job.py b/tests/jobs/test_triggerer_job.py index da8405e197317..3987aa86438fa 100644 --- a/tests/jobs/test_triggerer_job.py +++ b/tests/jobs/test_triggerer_job.py @@ -46,9 +46,9 @@ from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunType -from tests.core.test_logging_config import reset_logging +from tests_common.test_utils.db import clear_db_dags, clear_db_runs -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests.core.test_logging_config import reset_logging pytestmark = pytest.mark.db_test diff --git a/tests/jobs/test_triggerer_job_logging.py b/tests/jobs/test_triggerer_job_logging.py index 4f39047f050ee..fcca727641512 100644 --- a/tests/jobs/test_triggerer_job_logging.py +++ b/tests/jobs/test_triggerer_job_logging.py @@ -30,8 +30,7 @@ from airflow.utils.log.file_task_handler import FileTaskHandler from airflow.utils.log.logging_mixin import RedirectStdHandler from airflow.utils.log.trigger_handler import DropTriggerLogsFilter, TriggererHandlerWrapper - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars def non_pytest_handlers(val): diff --git a/tests/lineage/test_hook.py b/tests/lineage/test_hook.py index 3fbbfaa021ffa..6287b39fc8fa1 100644 --- a/tests/lineage/test_hook.py +++ b/tests/lineage/test_hook.py @@ -33,8 +33,7 @@ NoOpCollector, get_hook_lineage_collector, ) - -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager +from tests_common.test_utils.mock_plugins import mock_plugin_manager class TestHookLineageCollector: diff --git a/tests/lineage/test_lineage.py b/tests/lineage/test_lineage.py index ac42b2ee7f814..bd5b2ebe0fc03 100644 --- a/tests/lineage/test_lineage.py +++ b/tests/lineage/test_lineage.py @@ -30,8 +30,7 @@ from airflow.utils import timezone from airflow.utils.context import Context from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/listeners/class_listener.py b/tests/listeners/class_listener.py index 2b9cefa178d7d..5ae71733bf3bd 100644 --- a/tests/listeners/class_listener.py +++ b/tests/listeners/class_listener.py @@ -19,8 +19,7 @@ from airflow.listeners import hookimpl from airflow.utils.state import DagRunState, TaskInstanceState - -from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS if AIRFLOW_V_2_10_PLUS: diff --git a/tests/listeners/test_asset_listener.py b/tests/listeners/test_asset_listener.py index bb93acd8a0fff..a075b87a7f3d2 100644 --- a/tests/listeners/test_asset_listener.py +++ b/tests/listeners/test_asset_listener.py @@ -23,6 +23,7 @@ from airflow.models.asset import AssetModel from airflow.operators.empty import EmptyOperator from airflow.utils.session import provide_session + from tests.listeners import asset_listener diff --git a/tests/listeners/test_dag_import_error_listener.py b/tests/listeners/test_dag_import_error_listener.py index 886044d27b44e..52a6103dc5bbd 100644 --- a/tests/listeners/test_dag_import_error_listener.py +++ b/tests/listeners/test_dag_import_error_listener.py @@ -31,10 +31,8 @@ from airflow.models import DagModel from airflow.models.errors import ParseImportError from airflow.utils import timezone -from tests.listeners import dag_import_error_listener - -from dev.tests_common.test_utils.config import conf_vars, env_vars -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.config import conf_vars, env_vars +from tests_common.test_utils.db import ( clear_db_dags, clear_db_import_errors, clear_db_jobs, @@ -43,7 +41,9 @@ clear_db_serialized_dags, clear_db_sla_miss, ) -from dev.tests_common.test_utils.mock_executor import MockExecutor +from tests_common.test_utils.mock_executor import MockExecutor + +from tests.listeners import dag_import_error_listener pytestmark = pytest.mark.db_test diff --git a/tests/listeners/test_listeners.py b/tests/listeners/test_listeners.py index 29ec25a9a8d2a..025e84f1cd696 100644 --- a/tests/listeners/test_listeners.py +++ b/tests/listeners/test_listeners.py @@ -29,6 +29,7 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import DagRunState, TaskInstanceState + from tests.listeners import ( class_listener, full_listener, diff --git a/tests/models/test_backfill.py b/tests/models/test_backfill.py index 06c41cadd8cc3..fafcca478c397 100644 --- a/tests/models/test_backfill.py +++ b/tests/models/test_backfill.py @@ -33,8 +33,7 @@ ) from airflow.operators.python import PythonOperator from airflow.utils.state import DagRunState - -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.db import ( clear_db_backfills, clear_db_dags, clear_db_runs, diff --git a/tests/models/test_base.py b/tests/models/test_base.py index 27eeba2912f52..2a58dc05009f3 100644 --- a/tests/models/test_base.py +++ b/tests/models/test_base.py @@ -19,8 +19,7 @@ import pytest from airflow.models.base import get_id_collation_args - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index 999529e14a997..b9edc2b56f5a4 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -49,9 +49,9 @@ from airflow.utils.template import literal from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType -from tests.models import DEFAULT_DATE +from tests_common.test_utils.mock_operators import DeprecatedOperator, MockOperator -from dev.tests_common.test_utils.mock_operators import DeprecatedOperator, MockOperator +from tests.models import DEFAULT_DATE if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/tests/models/test_cleartasks.py b/tests/models/test_cleartasks.py index 810453053dd0c..a58e308b01861 100644 --- a/tests/models/test_cleartasks.py +++ b/tests/models/test_cleartasks.py @@ -34,10 +34,10 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.types import DagRunType -from tests.models import DEFAULT_DATE +from tests_common.test_utils import db +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.models import DEFAULT_DATE if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 67dc699fc3c8b..0fe22ccf77fe3 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -90,6 +90,19 @@ from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from airflow.utils.weight_rule import WeightRule +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import ( + clear_db_assets, + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, +) +from tests_common.test_utils.mapping import expand_mapped_task +from tests_common.test_utils.mock_plugins import mock_plugin_manager +from tests_common.test_utils.timetables import cron_timetable, delta_timetable + from tests.models import DEFAULT_DATE from tests.plugins.priority_weight_strategy import ( FactorPriorityWeightStrategy, @@ -98,19 +111,6 @@ TestPriorityWeightStrategyPlugin, ) -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import ( - clear_db_assets, - clear_db_dags, - clear_db_runs, - clear_db_serialized_dags, -) -from dev.tests_common.test_utils.mapping import expand_mapped_task -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager -from dev.tests_common.test_utils.timetables import cron_timetable, delta_timetable - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py index 0b477cc24c636..0973da449ef38 100644 --- a/tests/models/test_dagbag.py +++ b/tests/models/test_dagbag.py @@ -44,13 +44,13 @@ from airflow.utils.dates import timezone as tz from airflow.utils.session import create_session from airflow.www.security_appless import ApplessAirflowSecurityManager +from tests_common.test_utils import db +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.config import conf_vars + from tests import cluster_policies from tests.models import TEST_DAGS_FOLDER -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.config import conf_vars - pytestmark = pytest.mark.db_test example_dags_folder = pathlib.Path(airflow.example_dags.__path__[0]) # type: ignore[attr-defined] diff --git a/tests/models/test_dagcode.py b/tests/models/test_dagcode.py index 0bc105cfaf42c..f0cec513c09d3 100644 --- a/tests/models/test_dagcode.py +++ b/tests/models/test_dagcode.py @@ -30,8 +30,7 @@ # To move it to a shared module. from airflow.utils.file import open_maybe_zipped from airflow.utils.session import create_session - -from dev.tests_common.test_utils.db import clear_db_dag_code +from tests_common.test_utils.db import clear_db_dag_code pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index 46773232f77fe..587d514cf807e 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -46,12 +46,12 @@ from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType -from tests.models import DEFAULT_DATE as _DEFAULT_DATE +from tests_common.test_utils import db +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.mock_operators import MockOperator -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.mock_operators import MockOperator +from tests.models import DEFAULT_DATE as _DEFAULT_DATE if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dagwarning.py b/tests/models/test_dagwarning.py index 9b98e0c31d74d..95d12c005e4bb 100644 --- a/tests/models/test_dagwarning.py +++ b/tests/models/test_dagwarning.py @@ -25,8 +25,7 @@ from airflow.models import DagModel from airflow.models.dagwarning import DagWarning - -from dev.tests_common.test_utils.db import clear_db_dags +from tests_common.test_utils.db import clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_mappedoperator.py b/tests/models/test_mappedoperator.py index 927dbfd73a563..7180421a90bd6 100644 --- a/tests/models/test_mappedoperator.py +++ b/tests/models/test_mappedoperator.py @@ -41,15 +41,15 @@ from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.trigger_rule import TriggerRule from airflow.utils.xcom import XCOM_RETURN_KEY -from tests.models import DEFAULT_DATE - -from dev.tests_common.test_utils.mapping import expand_mapped_task -from dev.tests_common.test_utils.mock_operators import ( +from tests_common.test_utils.mapping import expand_mapped_task +from tests_common.test_utils.mock_operators import ( MockOperator, MockOperatorWithNestedFields, NestedFields, ) +from tests.models import DEFAULT_DATE + pytestmark = pytest.mark.db_test if TYPE_CHECKING: diff --git a/tests/models/test_param.py b/tests/models/test_param.py index 2d324478deefe..7ade5bd21cebf 100644 --- a/tests/models/test_param.py +++ b/tests/models/test_param.py @@ -26,8 +26,7 @@ from airflow.serialization.serialized_objects import BaseSerialization from airflow.utils import timezone from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom class TestParam: diff --git a/tests/models/test_pool.py b/tests/models/test_pool.py index 22852261671ce..698899be002e4 100644 --- a/tests/models/test_pool.py +++ b/tests/models/test_pool.py @@ -27,8 +27,7 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State - -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.db import ( clear_db_dags, clear_db_pools, clear_db_runs, diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index 6ff87b28a89b6..6d160164ecff8 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -37,9 +37,8 @@ from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.timezone import datetime - -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields pytestmark = pytest.mark.db_test diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py index 5e6714feda373..2ed2d5090a2d6 100644 --- a/tests/models/test_serialized_dag.py +++ b/tests/models/test_serialized_dag.py @@ -36,9 +36,8 @@ from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 from airflow.utils.session import create_session - -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils import db +from tests_common.test_utils.asserts import assert_queries_count pytestmark = pytest.mark.db_test diff --git a/tests/models/test_skipmixin.py b/tests/models/test_skipmixin.py index 0f2406c5737c5..e15f5521d1cd0 100644 --- a/tests/models/test_skipmixin.py +++ b/tests/models/test_skipmixin.py @@ -31,8 +31,7 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 03566dca30bc9..5753f51fcd2f8 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -98,13 +98,13 @@ from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.types import DagRunType from airflow.utils.xcom import XCOM_RETURN_KEY -from tests.models import DEFAULT_DATE, TEST_DAGS_FOLDER +from tests_common.test_utils import db +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_connections, clear_db_runs +from tests_common.test_utils.mock_operators import MockOperator -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_connections, clear_db_runs -from dev.tests_common.test_utils.mock_operators import MockOperator +from tests.models import DEFAULT_DATE, TEST_DAGS_FOLDER if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_timestamp.py b/tests/models/test_timestamp.py index 4888c2fdac268..6bf9b9b768a60 100644 --- a/tests/models/test_timestamp.py +++ b/tests/models/test_timestamp.py @@ -25,8 +25,7 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import State - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_logs, clear_db_runs +from tests_common.test_utils.db import clear_db_dags, clear_db_logs, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/models/test_trigger.py b/tests/models/test_trigger.py index b44e4b9f6e881..8a9c21c685bbb 100644 --- a/tests/models/test_trigger.py +++ b/tests/models/test_trigger.py @@ -42,8 +42,7 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.xcom import XCOM_RETURN_KEY - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/models/test_variable.py b/tests/models/test_variable.py index dfe448092c842..29a5c94d9ae0d 100644 --- a/tests/models/test_variable.py +++ b/tests/models/test_variable.py @@ -27,9 +27,8 @@ from airflow.models import Variable, crypto, variable from airflow.secrets.cache import SecretCache from airflow.secrets.metastore import MetastoreBackend - -from dev.tests_common.test_utils import db -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils import db +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/models/test_xcom.py b/tests/models/test_xcom.py index 17ea5fa4ff114..ee47ac79fab1e 100644 --- a/tests/models/test_xcom.py +++ b/tests/models/test_xcom.py @@ -35,9 +35,8 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.xcom import XCOM_RETURN_KEY - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import is_db_isolation_mode +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import is_db_isolation_mode pytestmark = pytest.mark.db_test diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index fbdd500661d22..11613bcfec6b9 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -22,8 +22,7 @@ from airflow.operators.python import PythonOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.types import NOTSET - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/operators/test_branch_operator.py b/tests/operators/test_branch_operator.py index 8943bf580e50a..3442ff0a7e5e1 100644 --- a/tests/operators/test_branch_operator.py +++ b/tests/operators/test_branch_operator.py @@ -28,8 +28,7 @@ from airflow.utils.state import State from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/operators/test_email.py b/tests/operators/test_email.py index c86e0f94f6145..8ef2634aad4d3 100644 --- a/tests/operators/test_email.py +++ b/tests/operators/test_email.py @@ -24,8 +24,7 @@ from airflow.operators.email import EmailOperator from airflow.utils import timezone - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/operators/test_latest_only_operator.py b/tests/operators/test_latest_only_operator.py index 78fabc4ca922f..57bc2d27c1a3f 100644 --- a/tests/operators/test_latest_only_operator.py +++ b/tests/operators/test_latest_only_operator.py @@ -30,9 +30,8 @@ from airflow.utils.state import State from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs, clear_db_xcom if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index 3dd8f506ee986..5c91362723c1e 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -68,10 +68,9 @@ from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import NOTSET, DagRunType - -from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/plugins/test_plugin.py b/tests/plugins/test_plugin.py index fd474766d41ea..8751d75cd20fd 100644 --- a/tests/plugins/test_plugin.py +++ b/tests/plugins/test_plugin.py @@ -33,10 +33,7 @@ from airflow.task.priority_strategy import PriorityWeightStrategy from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.timetables.interval import CronDataIntervalTimetable -from tests.listeners import empty_listener -from tests.listeners.class_listener import ClassBasedListener - -from dev.tests_common.test_utils.mock_operators import ( +from tests_common.test_utils.mock_operators import ( AirflowLink, AirflowLink2, CustomBaseIndexOpLink, @@ -45,6 +42,9 @@ GoogleLink, ) +from tests.listeners import empty_listener +from tests.listeners.class_listener import ClassBasedListener + # Will show up under airflow.hooks.test_plugin.PluginHook class PluginHook(BaseHook): diff --git a/tests/plugins/test_plugins_manager.py b/tests/plugins/test_plugins_manager.py index 1c54c3ebb195c..0c742dd70c67d 100644 --- a/tests/plugins/test_plugins_manager.py +++ b/tests/plugins/test_plugins_manager.py @@ -33,9 +33,8 @@ from airflow.plugins_manager import AirflowPlugin from airflow.utils.module_loading import qualname from airflow.www import app as application - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.mock_plugins import mock_plugin_manager pytestmark = pytest.mark.db_test diff --git a/tests/secrets/test_cache.py b/tests/secrets/test_cache.py index 40ab4aa290df5..86ed57688f34d 100644 --- a/tests/secrets/test_cache.py +++ b/tests/secrets/test_cache.py @@ -22,8 +22,7 @@ import pytest from airflow.secrets.cache import SecretCache - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars def test_cache_disabled_by_default(): diff --git a/tests/security/test_kerberos.py b/tests/security/test_kerberos.py index 2b661fe9a1478..9f15c0296894c 100644 --- a/tests/security/test_kerberos.py +++ b/tests/security/test_kerberos.py @@ -25,8 +25,7 @@ from airflow.security import kerberos from airflow.security.kerberos import get_kerberos_principle, renew_from_kt - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/sensors/test_base.py b/tests/sensors/test_base.py index 8a3b2b5c5c0fa..f89cee879f9b8 100644 --- a/tests/sensors/test_base.py +++ b/tests/sensors/test_base.py @@ -57,8 +57,7 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.timezone import datetime - -from dev.tests_common.test_utils import db +from tests_common.test_utils import db pytestmark = pytest.mark.db_test diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index f1739e6b84418..2f17bfd38d133 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -51,11 +51,11 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests.models import TEST_DAGS_FOLDER +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.mock_operators import MockOperator -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs -from dev.tests_common.test_utils.mock_operators import MockOperator +from tests.models import TEST_DAGS_FOLDER if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/sensors/test_python.py b/tests/sensors/test_python.py index 60814620b3b99..0a898b339108c 100644 --- a/tests/sensors/test_python.py +++ b/tests/sensors/test_python.py @@ -25,6 +25,7 @@ from airflow.exceptions import AirflowSensorTimeout from airflow.sensors.base import PokeReturnValue from airflow.sensors.python import PythonSensor + from tests.operators.test_python import BasePythonTest pytestmark = pytest.mark.db_test diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 35f5e196a2222..da6723800d54a 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -80,10 +80,9 @@ from airflow.utils.operator_resources import Resources from airflow.utils.task_group import TaskGroup from airflow.utils.xcom import XCOM_RETURN_KEY - -from dev.tests_common.test_utils.compat import BaseOperatorLink -from dev.tests_common.test_utils.mock_operators import AirflowLink2, CustomOperator, GoogleLink, MockOperator -from dev.tests_common.test_utils.timetables import ( +from tests_common.test_utils.compat import BaseOperatorLink +from tests_common.test_utils.mock_operators import AirflowLink2, CustomOperator, GoogleLink, MockOperator +from tests_common.test_utils.timetables import ( CustomSerializationTimetable, cron_timetable, delta_timetable, @@ -190,9 +189,7 @@ "max_retry_delay": 600.0, "downstream_task_ids": [], "_is_empty": False, - "_operator_extra_links": [ - {"dev.tests_common.test_utils.mock_operators.CustomOpLink": {}} - ], + "_operator_extra_links": [{"tests_common.test_utils.mock_operators.CustomOpLink": {}}], "ui_color": "#fff", "ui_fgcolor": "#000", "template_ext": [], @@ -200,7 +197,7 @@ "template_fields_renderers": {}, "_task_type": "CustomOperator", "_operator_name": "@custom", - "_task_module": "dev.tests_common.test_utils.mock_operators", + "_task_module": "tests_common.test_utils.mock_operators", "pool": "default_pool", "is_setup": False, "is_teardown": False, @@ -240,7 +237,7 @@ ) CUSTOM_TIMETABLE_SERIALIZED = { - "__type": "dev.tests_common.test_utils.timetables.CustomSerializationTimetable", + "__type": "tests_common.test_utils.timetables.CustomSerializationTimetable", "__var": {"value": "foo"}, } @@ -373,7 +370,7 @@ def timetable_plugin(monkeypatch): monkeypatch.setattr( plugins_manager, "timetable_classes", - {"dev.tests_common.test_utils.timetables.CustomSerializationTimetable": CustomSerializationTimetable}, + {"tests_common.test_utils.timetables.CustomSerializationTimetable": CustomSerializationTimetable}, ) @@ -469,7 +466,7 @@ def test_dag_serialization_unregistered_custom_timetable(self): message = ( "Failed to serialize DAG 'simple_dag': Timetable class " - "'dev.tests_common.test_utils.timetables.CustomSerializationTimetable' " + "'tests_common.test_utils.timetables.CustomSerializationTimetable' " "is not registered or " "you have a top level database access that disrupted the session. " "Please check the airflow best practices documentation." @@ -834,7 +831,7 @@ def test_deserialization_timetable_unregistered(self): SerializedDAG.from_dict(serialized) message = ( "Timetable class " - "'dev.tests_common.test_utils.timetables.CustomSerializationTimetable' " + "'tests_common.test_utils.timetables.CustomSerializationTimetable' " "is not registered or " "you have a top level database access that disrupted the session. " "Please check the airflow best practices documentation." @@ -986,15 +983,15 @@ def test_task_params_roundtrip(self, val, expected_val): [ pytest.param( "true", - [{"dev.tests_common.test_utils.mock_operators.CustomOpLink": {}}], + [{"tests_common.test_utils.mock_operators.CustomOpLink": {}}], {"Google Custom": "http://google.com/custom_base_link?search=true"}, id="non-indexed-link", ), pytest.param( ["echo", "true"], [ - {"dev.tests_common.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 0}}, - {"dev.tests_common.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 1}}, + {"tests_common.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 0}}, + {"tests_common.test_utils.mock_operators.CustomBaseIndexOpLink": {"index": 1}}, ], { "BigQuery Console #1": "https://console.cloud.google.com/bigquery?j=echo", @@ -1297,7 +1294,7 @@ def test_operator_deserialize_old_names(self): "template_fields": ["bash_command"], "template_fields_renderers": {}, "_task_type": "CustomOperator", - "_task_module": "dev.tests_common.test_utils.mock_operators", + "_task_module": "tests_common.test_utils.mock_operators", "pool": "default_pool", "ui_color": "#fff", "ui_fgcolor": "#000", @@ -2354,7 +2351,7 @@ def test_operator_expand_xcomarg_serde(): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "dev.tests_common.test_utils.mock_operators", + "_task_module": "tests_common.test_utils.mock_operators", "_task_type": "MockOperator", "downstream_task_ids": [], "expand_input": { @@ -2409,7 +2406,7 @@ def test_operator_expand_kwargs_literal_serde(strict): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "dev.tests_common.test_utils.mock_operators", + "_task_module": "tests_common.test_utils.mock_operators", "_task_type": "MockOperator", "downstream_task_ids": [], "expand_input": { @@ -2464,7 +2461,7 @@ def test_operator_expand_kwargs_xcomarg_serde(strict): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "dev.tests_common.test_utils.mock_operators", + "_task_module": "tests_common.test_utils.mock_operators", "_task_type": "MockOperator", "downstream_task_ids": [], "expand_input": { @@ -2827,7 +2824,7 @@ def operator_extra_links(self): "_disallow_kwargs_override": False, "_expand_input_attr": "expand_input", "downstream_task_ids": [], - "_operator_extra_links": [{"dev.tests_common.test_utils.mock_operators.AirflowLink2": {}}], + "_operator_extra_links": [{"tests_common.test_utils.mock_operators.AirflowLink2": {}}], "ui_color": "#fff", "ui_fgcolor": "#000", "template_ext": [], diff --git a/tests/serialization/test_pydantic_models.py b/tests/serialization/test_pydantic_models.py index 423dc41a38068..6ecf9c8d5d90d 100644 --- a/tests/serialization/test_pydantic_models.py +++ b/tests/serialization/test_pydantic_models.py @@ -44,9 +44,9 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import AttributeRemoved, DagRunType -from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.models import DEFAULT_DATE if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/serialization/test_serde.py b/tests/serialization/test_serde.py index 5c915839c6422..829195476c3d6 100644 --- a/tests/serialization/test_serde.py +++ b/tests/serialization/test_serde.py @@ -43,8 +43,7 @@ serialize, ) from airflow.utils.module_loading import import_string, iter_namespace, qualname - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars @pytest.fixture diff --git a/tests/serialization/test_serialized_objects.py b/tests/serialization/test_serialized_objects.py index 67fdceeaf08d1..5d35278d89bfc 100644 --- a/tests/serialization/test_serialized_objects.py +++ b/tests/serialization/test_serialized_objects.py @@ -67,6 +67,7 @@ from airflow.utils.state import DagRunState, State from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType + from tests import REPO_ROOT @@ -471,8 +472,7 @@ def test_all_pydantic_models_round_trip(): @pytest.mark.db_test def test_serialized_mapped_operator_unmap(dag_maker): from airflow.serialization.serialized_objects import SerializedDAG - - from dev.tests_common.test_utils.mock_operators import MockOperator + from tests_common.test_utils.mock_operators import MockOperator with dag_maker(dag_id="dag") as dag: MockOperator(task_id="task1", arg1="x") diff --git a/tests/system/core/example_external_task_child_deferrable.py b/tests/system/core/example_external_task_child_deferrable.py index 4e8eca3b15eb2..1da360b57b56d 100644 --- a/tests/system/core/example_external_task_child_deferrable.py +++ b/tests/system/core/example_external_task_child_deferrable.py @@ -34,7 +34,7 @@ ) -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/core/example_external_task_parent_deferrable.py b/tests/system/core/example_external_task_parent_deferrable.py index 1a64f7e98ae8a..62bb1afc18b2e 100644 --- a/tests/system/core/example_external_task_parent_deferrable.py +++ b/tests/system/core/example_external_task_parent_deferrable.py @@ -57,14 +57,14 @@ start >> [trigger_child_task, external_task_sensor] >> end - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run +from tests_common.test_utils.system_tests import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/system/example_empty.py b/tests/system/example_empty.py index de316a11eee77..d2d362012c633 100644 --- a/tests/system/example_empty.py +++ b/tests/system/example_empty.py @@ -35,14 +35,14 @@ chain(task) - from dev.tests_common.test_utils.watcher import watcher + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "tearDown" task with trigger rule is part of the DAG list(dag.tasks) >> watcher() -from dev.tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) diff --git a/tests/task/task_runner/test_standard_task_runner.py b/tests/task/task_runner/test_standard_task_runner.py index 55e3d34192d60..7e1c80c8f8d6f 100644 --- a/tests/task/task_runner/test_standard_task_runner.py +++ b/tests/task/task_runner/test_standard_task_runner.py @@ -40,12 +40,12 @@ from airflow.utils.platform import getuser from airflow.utils.state import State from airflow.utils.timeout import timeout +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs + from tests.listeners import xcom_listener from tests.listeners.file_write_listener import FileWriteListener -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/ti_deps/deps/test_pool_slots_available_dep.py b/tests/ti_deps/deps/test_pool_slots_available_dep.py index d7fc33aa41bf8..1cf2d1d2072fe 100644 --- a/tests/ti_deps/deps/test_pool_slots_available_dep.py +++ b/tests/ti_deps/deps/test_pool_slots_available_dep.py @@ -26,8 +26,7 @@ from airflow.ti_deps.deps.pool_slots_available_dep import PoolSlotsAvailableDep from airflow.utils.session import create_session from airflow.utils.state import TaskInstanceState - -from dev.tests_common.test_utils import db +from tests_common.test_utils import db pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py b/tests/ti_deps/deps/test_prev_dagrun_dep.py index 638f4b69d7e92..bb7ea047ac651 100644 --- a/tests/ti_deps/deps/test_prev_dagrun_dep.py +++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py @@ -29,9 +29,8 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import convert_to_utc, datetime from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py index d137d43bbf38d..02b291797956f 100644 --- a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py +++ b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py @@ -30,8 +30,7 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State - -from dev.tests_common.test_utils import db +from tests_common.test_utils import db pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/log/test_log_reader.py b/tests/utils/log/test_log_reader.py index 53e702efd8a8e..95d8471a3a35b 100644 --- a/tests/utils/log/test_log_reader.py +++ b/tests/utils/log/test_log_reader.py @@ -38,9 +38,8 @@ from airflow.utils.log.logging_mixin import ExternalLoggingMixin from airflow.utils.state import TaskInstanceState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/log/test_secrets_masker.py b/tests/utils/log/test_secrets_masker.py index f3fed73675cce..ffe37fbfc4955 100644 --- a/tests/utils/log/test_secrets_masker.py +++ b/tests/utils/log/test_secrets_masker.py @@ -38,8 +38,7 @@ should_hide_value_for_key, ) from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.enable_redact p = "password" diff --git a/tests/utils/log/test_task_context_logger.py b/tests/utils/log/test_task_context_logger.py index 3a08947ede091..5d326a240f45e 100644 --- a/tests/utils/log/test_task_context_logger.py +++ b/tests/utils/log/test_task_context_logger.py @@ -24,9 +24,8 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.utils.log.task_context_logger import TaskContextLogger - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_dag_cycle.py b/tests/utils/test_dag_cycle.py index 1cf607fd8ee50..55937f1382e38 100644 --- a/tests/utils/test_dag_cycle.py +++ b/tests/utils/test_dag_cycle.py @@ -24,6 +24,7 @@ from airflow.utils.dag_cycle_tester import check_cycle from airflow.utils.edgemodifier import Label from airflow.utils.task_group import TaskGroup + from tests.models import DEFAULT_DATE diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index 0f3a56a5d1c8d..dcfcd16421527 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -50,8 +50,7 @@ upgradedb, ) from airflow.utils.db_manager import RunDBManager - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py index 3b5a3b2227231..c7533827d805d 100644 --- a/tests/utils/test_db_cleanup.py +++ b/tests/utils/test_db_cleanup.py @@ -48,8 +48,7 @@ run_cleanup, ) from airflow.utils.session import create_session - -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.db import ( clear_db_assets, clear_db_dags, clear_db_runs, diff --git a/tests/utils/test_db_manager.py b/tests/utils/test_db_manager.py index 975ea65499aae..001d89e1d4c59 100644 --- a/tests/utils/test_db_manager.py +++ b/tests/utils/test_db_manager.py @@ -25,8 +25,7 @@ from airflow.models import Base from airflow.utils.db import downgrade, initdb from airflow.utils.db_manager import BaseDBManager, RunDBManager - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test] diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index d1521ca6e0f69..718d2d03b0832 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -29,9 +29,8 @@ from airflow.utils import dot_renderer, timezone from airflow.utils.state import State from airflow.utils.task_group import TaskGroup - -from dev.tests_common.test_utils.compat import BashOperator -from dev.tests_common.test_utils.db import clear_db_dags +from tests_common.test_utils.compat import BashOperator +from tests_common.test_utils.db import clear_db_dags START_DATE = timezone.utcnow() diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index b47dcbc87585a..ac90c20ef6a75 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -29,8 +29,7 @@ from airflow.configuration import conf from airflow.utils import email - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars EMAILS = ["test1@example.com", "test2@example.com"] diff --git a/tests/utils/test_file.py b/tests/utils/test_file.py index 9424b90a92cf4..a7ca4bd5575f7 100644 --- a/tests/utils/test_file.py +++ b/tests/utils/test_file.py @@ -26,9 +26,9 @@ from airflow.utils import file as file_utils from airflow.utils.file import correct_maybe_zipped, find_path_from_directory, open_maybe_zipped -from tests.models import TEST_DAGS_FOLDER +from tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.config import conf_vars +from tests.models import TEST_DAGS_FOLDER def might_contain_dag(file_path: str, zip_file: zipfile.ZipFile | None = None): diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index 0b0046b1ba082..ded59b999f8bf 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -37,9 +37,8 @@ validate_key, ) from airflow.utils.types import NOTSET - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs if TYPE_CHECKING: from airflow.jobs.job import Job diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py index 4be3564567ebd..c1135d8fe4a18 100644 --- a/tests/utils/test_log_handlers.py +++ b/tests/utils/test_log_handlers.py @@ -54,9 +54,8 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_net.py b/tests/utils/test_net.py index 60823911b8b03..50fdd207dec3f 100644 --- a/tests/utils/test_net.py +++ b/tests/utils/test_net.py @@ -24,8 +24,7 @@ from airflow.exceptions import AirflowConfigException from airflow.utils import net - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars def get_hostname(): diff --git a/tests/utils/test_serve_logs.py b/tests/utils/test_serve_logs.py index 8d41202be2545..0c6fb8d87591e 100644 --- a/tests/utils/test_serve_logs.py +++ b/tests/utils/test_serve_logs.py @@ -28,8 +28,7 @@ from airflow.utils import timezone from airflow.utils.jwt_signer import JWTSigner from airflow.utils.serve_logs import create_app - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars if TYPE_CHECKING: from flask.testing import FlaskClient diff --git a/tests/utils/test_sqlalchemy.py b/tests/utils/test_sqlalchemy.py index 42717a4fb045a..49e4db42788a6 100644 --- a/tests/utils/test_sqlalchemy.py +++ b/tests/utils/test_sqlalchemy.py @@ -42,8 +42,7 @@ ) from airflow.utils.state import State from airflow.utils.timezone import utcnow - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_state.py b/tests/utils/test_state.py index 6447477ef589b..12d857ee1999e 100644 --- a/tests/utils/test_state.py +++ b/tests/utils/test_state.py @@ -25,9 +25,9 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.models import DEFAULT_DATE if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index 6ea0fcdc36e13..6b2185bf791fc 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -38,9 +38,9 @@ from airflow.operators.python import PythonOperator from airflow.utils.dag_edges import dag_edges from airflow.utils.task_group import TASKGROUP_ARGS_EXPECTED_TYPES, TaskGroup, task_group_to_dict -from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import BashOperator -from dev.tests_common.test_utils.compat import BashOperator +from tests.models import DEFAULT_DATE def make_task(name, type_="classic"): @@ -1415,7 +1415,7 @@ def test_task_group_edge_modifier_chain(): def test_mapped_task_group_id_prefix_task_id(): - from dev.tests_common.test_utils.mock_operators import MockOperator + from tests_common.test_utils.mock_operators import MockOperator with DAG(dag_id="d", schedule=None, start_date=DEFAULT_DATE) as dag: t1 = MockOperator.partial(task_id="t1").expand(arg1=[]) diff --git a/tests/utils/test_task_handler_with_custom_formatter.py b/tests/utils/test_task_handler_with_custom_formatter.py index aee646d858b15..785cb34eaa013 100644 --- a/tests/utils/test_task_handler_with_custom_formatter.py +++ b/tests/utils/test_task_handler_with_custom_formatter.py @@ -28,10 +28,9 @@ from airflow.utils.state import DagRunState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_types.py b/tests/utils/test_types.py index cb5849b20599a..d982730392076 100644 --- a/tests/utils/test_types.py +++ b/tests/utils/test_types.py @@ -25,9 +25,9 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.types import DagRunType -from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.models import DEFAULT_DATE if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/test_app.py b/tests/www/test_app.py index 673abd371dfb5..20dd5eb05e7e0 100644 --- a/tests/www/test_app.py +++ b/tests/www/test_app.py @@ -29,9 +29,8 @@ from airflow.exceptions import AirflowConfigException from airflow.www import app as application - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index f8f0ae983382e..1ee0415b822a4 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -44,8 +44,7 @@ wrapped_markdown, ) from airflow.www.widgets import AirflowDateTimePickerROWidget, BS3TextAreaROWidget, BS3TextFieldROWidget - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/conftest.py b/tests/www/views/conftest.py index faa1cc23a8d36..e5bf5f947ac3b 100644 --- a/tests/www/views/conftest.py +++ b/tests/www/views/conftest.py @@ -27,11 +27,10 @@ from airflow import settings from airflow.models import DagBag from airflow.www.app import create_app - -from dev.tests_common.test_utils.api_connexion_utils import delete_user -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.api_connexion_utils import delete_user +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.www import ( client_with_login, client_without_login, client_without_login_as_admin, diff --git a/tests/www/views/test_anonymous_as_admin_role.py b/tests/www/views/test_anonymous_as_admin_role.py index 700f03f8e63c8..f881a35f95594 100644 --- a/tests/www/views/test_anonymous_as_admin_role.py +++ b/tests/www/views/test_anonymous_as_admin_role.py @@ -23,8 +23,7 @@ from airflow.models import Pool from airflow.utils.session import create_session - -from dev.tests_common.test_utils.www import check_content_in_response +from tests_common.test_utils.www import check_content_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_session.py b/tests/www/views/test_session.py index 009917606bda3..3bce5ebd9bf0d 100644 --- a/tests/www/views/test_session.py +++ b/tests/www/views/test_session.py @@ -22,9 +22,8 @@ from airflow.exceptions import AirflowConfigException from airflow.www import app - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index bae022316c73d..46d2ea7b194f9 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -42,11 +42,10 @@ get_task_stats_from_query, get_value_from_path, ) - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.mock_plugins import mock_plugin_manager -from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.mock_plugins import mock_plugin_manager +from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -330,8 +329,7 @@ def test_mark_task_instance_state(test_app): from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from airflow.www.views import Airflow - - from dev.tests_common.test_utils.db import clear_db_runs + from tests_common.test_utils.db import clear_db_runs clear_db_runs() start_date = datetime(2020, 1, 1) @@ -424,8 +422,7 @@ def test_mark_task_group_state(test_app): from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from airflow.www.views import Airflow - - from dev.tests_common.test_utils.db import clear_db_runs + from tests_common.test_utils.db import clear_db_runs clear_db_runs() start_date = datetime(2020, 1, 1) diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index 23fb26c18cf98..08e70b5727626 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -30,15 +30,15 @@ from airflow.utils.state import State from airflow.utils.types import DagRunType from airflow.www.views import FILTER_STATUS_COOKIE - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs -from dev.tests_common.test_utils.permissions import _resource_name -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.permissions import _resource_name +from tests_common.test_utils.www import ( check_content_in_response, check_content_not_in_response, client_with_login, ) + from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user_scope if AIRFLOW_V_3_0_PLUS: diff --git a/tests/www/views/test_views_base.py b/tests/www/views/test_views_base.py index 692f0301e2276..3254441c74da8 100644 --- a/tests/www/views/test_views_base.py +++ b/tests/www/views/test_views_base.py @@ -27,10 +27,9 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.www import app as application - -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_cluster_activity.py b/tests/www/views/test_views_cluster_activity.py index 6b83de3c7682a..64063e069db69 100644 --- a/tests/www/views/test_views_cluster_activity.py +++ b/tests/www/views/test_views_cluster_activity.py @@ -26,8 +26,7 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_configuration.py b/tests/www/views/test_views_configuration.py index 90ab9b7faa933..4bd1f2fc4c3c8 100644 --- a/tests/www/views/test_views_configuration.py +++ b/tests/www/views/test_views_configuration.py @@ -21,9 +21,8 @@ import pytest from airflow.configuration import conf - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 66e571780eb96..7b0229c315b96 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -27,8 +27,7 @@ from airflow.models import Connection from airflow.utils.session import create_session from airflow.www.views import ConnectionFormWidget, ConnectionModelView - -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.www import ( _check_last_log, _check_last_log_masked_connection, check_content_in_response, diff --git a/tests/www/views/test_views_custom_user_views.py b/tests/www/views/test_views_custom_user_views.py index 49ecc2c4f30b2..3c00242308b75 100644 --- a/tests/www/views/test_views_custom_user_views.py +++ b/tests/www/views/test_views_custom_user_views.py @@ -27,12 +27,12 @@ from airflow import settings from airflow.security import permissions from airflow.www import app as application - -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.www import ( check_content_in_response, check_content_not_in_response, client_with_login, ) + from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_role, diff --git a/tests/www/views/test_views_dagrun.py b/tests/www/views/test_views_dagrun.py index 3fb8eb02fb633..9a30e0a3511eb 100644 --- a/tests/www/views/test_views_dagrun.py +++ b/tests/www/views/test_views_dagrun.py @@ -24,19 +24,19 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.www.views import DagRunModelView -from tests.www.views.test_views_tasks import _get_appbuilder_pk_string - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.www import ( check_content_in_response, check_content_not_in_response, client_with_login, ) + from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, delete_user, ) +from tests.www.views.test_views_tasks import _get_appbuilder_pk_string if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_dataset.py b/tests/www/views/test_views_dataset.py index 899d64e9754ed..9d83715f3e210 100644 --- a/tests/www/views/test_views_dataset.py +++ b/tests/www/views/test_views_dataset.py @@ -24,9 +24,8 @@ from airflow.assets import Asset from airflow.models.asset import AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator - -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.db import clear_db_assets +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.db import clear_db_assets pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py index bdc402989fd0c..730fc55b4548b 100644 --- a/tests/www/views/test_views_decorators.py +++ b/tests/www/views/test_views_decorators.py @@ -23,10 +23,9 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_variables -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs, clear_db_variables +from tests_common.test_utils.www import ( _check_last_log, _check_last_log_masked_variable, check_content_in_response, diff --git a/tests/www/views/test_views_extra_links.py b/tests/www/views/test_views_extra_links.py index fbc11bf51f337..bc06a09c8f95b 100644 --- a/tests/www/views/test_views_extra_links.py +++ b/tests/www/views/test_views_extra_links.py @@ -28,10 +28,13 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink -from dev.tests_common.test_utils.db import clear_db_runs -from dev.tests_common.test_utils.mock_operators import AirflowLink, Dummy2TestOperator, Dummy3TestOperator +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink +from tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.mock_operators import ( + AirflowLink, + EmptyExtraLinkTestOperator, + EmptyNoExtraLinkTestOperator, +) if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -115,12 +118,12 @@ def task_1(dag): @pytest.fixture(scope="module", autouse=True) def task_2(dag): - return Dummy2TestOperator(task_id="some_dummy_task_2", dag=dag) + return EmptyExtraLinkTestOperator(task_id="some_dummy_task_2", dag=dag) @pytest.fixture(scope="module", autouse=True) def task_3(dag): - return Dummy3TestOperator(task_id="some_dummy_task_3", dag=dag) + return EmptyNoExtraLinkTestOperator(task_id="some_dummy_task_3", dag=dag) @pytest.fixture(scope="module", autouse=True) diff --git a/tests/www/views/test_views_grid.py b/tests/www/views/test_views_grid.py index 4f05f2a13456b..d896520422493 100644 --- a/tests/www/views/test_views_grid.py +++ b/tests/www/views/test_views_grid.py @@ -35,10 +35,9 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType from airflow.www.views import dag_to_grid - -from dev.tests_common.test_utils.asserts import assert_queries_count -from dev.tests_common.test_utils.db import clear_db_assets, clear_db_runs -from dev.tests_common.test_utils.mock_operators import MockOperator +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.db import clear_db_assets, clear_db_runs +from tests_common.test_utils.mock_operators import MockOperator pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_home.py b/tests/www/views/test_views_home.py index 4de032c58176e..b02099a668008 100644 --- a/tests/www/views/test_views_home.py +++ b/tests/www/views/test_views_home.py @@ -27,14 +27,14 @@ from airflow.utils.state import State from airflow.www.utils import UIAlert from airflow.www.views import FILTER_LASTRUN_COOKIE, FILTER_STATUS_COOKIE, FILTER_TAGS_COOKIE - -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_import_errors, clear_db_serialized_dags -from dev.tests_common.test_utils.permissions import _resource_name -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.db import clear_db_dags, clear_db_import_errors, clear_db_serialized_dags +from tests_common.test_utils.permissions import _resource_name +from tests_common.test_utils.www import ( check_content_in_response, check_content_not_in_response, client_with_login, ) + from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_log.py b/tests/www/views/test_views_log.py index c025ebcbbe8ed..b0fdf6a01d10c 100644 --- a/tests/www/views/test_views_log.py +++ b/tests/www/views/test_views_log.py @@ -40,12 +40,11 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType from airflow.www.app import create_app - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules -from dev.tests_common.test_utils.www import client_with_login +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.www import client_with_login if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_mount.py b/tests/www/views/test_views_mount.py index e58675439ec12..c20dc418d3a02 100644 --- a/tests/www/views/test_views_mount.py +++ b/tests/www/views/test_views_mount.py @@ -22,8 +22,7 @@ import werkzeug.wrappers from airflow.www.app import create_app - -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_paused.py b/tests/www/views/test_views_paused.py index 551cdd180e078..59eee1868b70e 100644 --- a/tests/www/views/test_views_paused.py +++ b/tests/www/views/test_views_paused.py @@ -19,8 +19,7 @@ import pytest from airflow.models.log import Log - -from dev.tests_common.test_utils.db import clear_db_dags +from tests_common.test_utils.db import clear_db_dags pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_pool.py b/tests/www/views/test_views_pool.py index 9b9ddfaeeb79e..7b7b20d57c9cf 100644 --- a/tests/www/views/test_views_pool.py +++ b/tests/www/views/test_views_pool.py @@ -23,8 +23,7 @@ from airflow.models import Pool from airflow.utils.session import create_session - -from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response +from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_rate_limit.py b/tests/www/views/test_views_rate_limit.py index d843c8ed54b09..eee9fed8dde00 100644 --- a/tests/www/views/test_views_rate_limit.py +++ b/tests/www/views/test_views_rate_limit.py @@ -20,10 +20,9 @@ import pytest from airflow.www.app import create_app - -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.decorators import dont_initialize_flask_app_submodules -from dev.tests_common.test_utils.www import client_with_login +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.www import client_with_login pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 6d96cdc4d625f..c23c300416a1f 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -34,15 +34,14 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator -from dev.tests_common.test_utils.db import ( +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator +from tests_common.test_utils.db import ( clear_db_dags, clear_db_runs, clear_rendered_ti_fields, initial_db_init, ) -from dev.tests_common.test_utils.www import check_content_in_response, check_content_not_in_response +from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_robots.py b/tests/www/views/test_views_robots.py index 41f963b3bf218..9f93ef5791f6f 100644 --- a/tests/www/views/test_views_robots.py +++ b/tests/www/views/test_views_robots.py @@ -18,7 +18,7 @@ import pytest -from dev.tests_common.test_utils.config import conf_vars +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_task_norun.py b/tests/www/views/test_views_task_norun.py index 2ff6ff9d7cd67..4cb195461fccc 100644 --- a/tests/www/views/test_views_task_norun.py +++ b/tests/www/views/test_views_task_norun.py @@ -22,7 +22,7 @@ import pytest -from dev.tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 4e9de5c00fc9c..b1a57ef290528 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -43,15 +43,15 @@ from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType from airflow.www.views import TaskInstanceModelView, _safe_parse_datetime - -from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.db import clear_db_runs, clear_db_xcom -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_runs, clear_db_xcom +from tests_common.test_utils.www import ( check_content_in_response, check_content_not_in_response, client_with_login, ) + from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, diff --git a/tests/www/views/test_views_trigger_dag.py b/tests/www/views/test_views_trigger_dag.py index d2c3817ede6db..f7e4d62599ea1 100644 --- a/tests/www/views/test_views_trigger_dag.py +++ b/tests/www/views/test_views_trigger_dag.py @@ -32,9 +32,9 @@ from airflow.utils.json import WebEncoder from airflow.utils.session import create_session from airflow.utils.types import DagRunType +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response -from dev.tests_common.test_utils.config import conf_vars -from dev.tests_common.test_utils.www import check_content_in_response from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_test_client pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_variable.py b/tests/www/views/test_views_variable.py index dae1f2e9b7315..2f1ec81fef1d6 100644 --- a/tests/www/views/test_views_variable.py +++ b/tests/www/views/test_views_variable.py @@ -25,13 +25,13 @@ from airflow.models import Variable from airflow.security import permissions from airflow.utils.session import create_session - -from dev.tests_common.test_utils.www import ( +from tests_common.test_utils.www import ( _check_last_log, check_content_in_response, check_content_not_in_response, client_with_login, ) + from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user pytestmark = pytest.mark.db_test diff --git a/dev/tests_common/__init__.py b/tests_common/__init__.py similarity index 100% rename from dev/tests_common/__init__.py rename to tests_common/__init__.py diff --git a/dev/tests_common/_internals/__init__.py b/tests_common/_internals/__init__.py similarity index 100% rename from dev/tests_common/_internals/__init__.py rename to tests_common/_internals/__init__.py diff --git a/dev/tests_common/_internals/capture_warnings.py b/tests_common/_internals/capture_warnings.py similarity index 99% rename from dev/tests_common/_internals/capture_warnings.py rename to tests_common/_internals/capture_warnings.py index 16b220989b89b..aaae4e54f0ce7 100644 --- a/dev/tests_common/_internals/capture_warnings.py +++ b/tests_common/_internals/capture_warnings.py @@ -179,7 +179,7 @@ def pytest_runtest_protocol(self, item: pytest.Item): @pytest.hookimpl(hookwrapper=True, trylast=True) def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int): - """Save warning captures in the session finish on xdist worker node""" + """Save warning captures in the session finish on xdist worker node.""" with CapturedWarning.capture_warnings("config", self.root_path, None) as records: yield self.add_captured_warnings(records) @@ -206,7 +206,7 @@ def pytest_testnodedown(self, node, error): @staticmethod def sorted_groupby(it, grouping_key: Callable): - """Helper for sort and group by.""" + """Sort and group by items by the grouping_key.""" for group, grouped_data in itertools.groupby(sorted(it, key=grouping_key), key=grouping_key): yield group, list(grouped_data) diff --git a/dev/tests_common/_internals/forbidden_warnings.py b/tests_common/_internals/forbidden_warnings.py similarity index 99% rename from dev/tests_common/_internals/forbidden_warnings.py rename to tests_common/_internals/forbidden_warnings.py index ce24878510667..0b0a11262b3b6 100644 --- a/dev/tests_common/_internals/forbidden_warnings.py +++ b/tests_common/_internals/forbidden_warnings.py @@ -81,7 +81,7 @@ def pytest_itemcollected(self, item: pytest.Item): @pytest.hookimpl(hookwrapper=True, trylast=True) def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int): - """Save set of test node ids in the session finish on xdist worker node""" + """Save set of test node ids in the session finish on xdist worker node.""" yield if self.is_worker_node and self.detected_cases and hasattr(self.config, "workeroutput"): self.config.workeroutput[self.node_key] = frozenset(self.detected_cases) diff --git a/dev/tests_common/pyproject.toml b/tests_common/pyproject.toml similarity index 98% rename from dev/tests_common/pyproject.toml rename to tests_common/pyproject.toml index 092ad7c7c5f5b..7cb353285ddd7 100644 --- a/dev/tests_common/pyproject.toml +++ b/tests_common/pyproject.toml @@ -35,4 +35,4 @@ disable = true include = ["**/*.py"] [tool.hatch.build.targets.wheel.sources] -"" = "dev/tests_common" +"" = "tests_common" diff --git a/dev/tests_common/pytest_plugin.py b/tests_common/pytest_plugin.py similarity index 97% rename from dev/tests_common/pytest_plugin.py rename to tests_common/pytest_plugin.py index 5694fe1b3403f..60638d8a5a756 100644 --- a/dev/tests_common/pytest_plugin.py +++ b/tests_common/pytest_plugin.py @@ -34,8 +34,8 @@ if TYPE_CHECKING: from itsdangerous import URLSafeSerializer - from dev.tests_common._internals.capture_warnings import CaptureWarningsPlugin # noqa: F401 - from dev.tests_common._internals.forbidden_warnings import ForbiddenWarningsPlugin # noqa: F401 + from tests_common._internals.capture_warnings import CaptureWarningsPlugin # noqa: F401 + from tests_common._internals.forbidden_warnings import ForbiddenWarningsPlugin # noqa: F401 # https://docs.pytest.org/en/stable/reference/reference.html#stash capture_warnings_key = pytest.StashKey["CaptureWarningsPlugin"]() @@ -133,8 +133,7 @@ @pytest.fixture def reset_db(): - """Resets Airflow db.""" - + """Reset Airflow db.""" from airflow.utils import db db.resetdb() @@ -145,7 +144,7 @@ def reset_db(): @pytest.fixture(autouse=True) def trace_sql(request): - from dev.tests_common.test_utils.perf.perf_kit.sqlalchemy import ( # isort: skip + from tests_common.test_utils.perf.perf_kit.sqlalchemy import ( # isort: skip count_queries, trace_queries, ) @@ -325,10 +324,10 @@ def pytest_addoption(parser: pytest.Parser): @pytest.fixture(autouse=True, scope="session") def initialize_airflow_tests(request): - """Helper that setups Airflow testing environment.""" + """Set up Airflow testing environment.""" print(" AIRFLOW ".center(60, "=")) - from dev.tests_common.test_utils.db import initial_db_init + from tests_common.test_utils.db import initial_db_init # Setup test environment for breeze home = os.path.expanduser("~") @@ -340,7 +339,7 @@ def initialize_airflow_tests(request): lock_file = os.path.join(airflow_home, ".airflow_db_initialised") if not skip_db_tests: if request.config.option.db_init: - from dev.tests_common.test_utils.db import initial_db_init + from tests_common.test_utils.db import initial_db_init print("Initializing the DB - forced with --with-db-init switch.") initial_db_init() @@ -431,7 +430,7 @@ def pytest_configure(config: pytest.Config) -> None: forbidden_warnings: list[str] | None = config.getini("forbidden_warnings") if not config.option.disable_forbidden_warnings and forbidden_warnings: - from dev.tests_common._internals.forbidden_warnings import ForbiddenWarningsPlugin + from tests_common._internals.forbidden_warnings import ForbiddenWarningsPlugin forbidden_warnings_plugin = ForbiddenWarningsPlugin( config=config, @@ -441,7 +440,7 @@ def pytest_configure(config: pytest.Config) -> None: config.stash[forbidden_warnings_key] = forbidden_warnings_plugin if not config.option.disable_capture_warnings: - from dev.tests_common._internals.capture_warnings import CaptureWarningsPlugin + from tests_common._internals.capture_warnings import CaptureWarningsPlugin capture_warnings_plugin = CaptureWarningsPlugin( config=config, output_path=config.getoption("warning_output_path", default=None) @@ -678,7 +677,8 @@ def pytest_runtest_setup(item): @pytest.fixture def frozen_sleep(monkeypatch): - """Use time-machine to "stub" sleep. + """ + Use time-machine to "stub" sleep. This means the ``sleep()`` takes no time, but ``datetime.now()`` appears to move forwards. @@ -717,7 +717,8 @@ def fake_sleep(seconds): @pytest.fixture def dag_maker(request): - """Fixture to help create DAG, DagModel, and SerializedDAG automatically. + """ + Fixture to help create DAG, DagModel, and SerializedDAG automatically. You have to use the dag_maker as a context manager and it takes the same argument as DAG:: @@ -824,8 +825,7 @@ def create_dagrun(self, **kwargs): from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType - - from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -922,8 +922,7 @@ def cleanup(self): from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskmap import TaskMap from airflow.utils.retries import run_with_db_retries - - from dev.tests_common.test_utils.compat import AssetEvent + from tests_common.test_utils.compat import AssetEvent for attempt in run_with_db_retries(logger=self.log): with attempt: @@ -970,7 +969,8 @@ def cleanup(self): @pytest.fixture def create_dummy_dag(dag_maker): - """Create a `DAG` with a single `EmptyOperator` task. + """ + Create a `DAG` with a single `EmptyOperator` task. DagRun and DagModel is also created. @@ -1006,7 +1006,7 @@ def create_dag( **kwargs, ): op_kwargs = {} - from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS if AIRFLOW_V_2_9_PLUS: op_kwargs["task_display_name"] = task_display_name @@ -1038,7 +1038,8 @@ def create_dag( @pytest.fixture def create_task_instance(dag_maker, create_dummy_dag): - """Create a TaskInstance, and associated DB rows (DagRun, DagModel, etc). + """ + Create a TaskInstance, and associated DB rows (DagRun, DagModel, etc). Uses ``create_dummy_dag`` to create the dag structure. """ @@ -1068,7 +1069,7 @@ def maker( map_index=-1, **kwargs, ) -> TaskInstance: - from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -1079,7 +1080,7 @@ def maker( execution_date = timezone.utcnow() with dag_maker(dag_id, **kwargs): op_kwargs = {} - from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS if AIRFLOW_V_2_9_PLUS: op_kwargs["task_display_name"] = task_display_name @@ -1243,7 +1244,7 @@ def reset_logging_config(): def suppress_info_logs_for_dag_and_fab(): import logging - from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS dag_logger = logging.getLogger("airflow.models.dag") dag_logger.setLevel(logging.WARNING) @@ -1259,7 +1260,7 @@ def suppress_info_logs_for_dag_and_fab(): @pytest.fixture(scope="module", autouse=True) def _clear_db(request): """Clear DB before each test module run.""" - from dev.tests_common.test_utils.db import clear_all, initial_db_init + from tests_common.test_utils.db import clear_all, initial_db_init if not request.config.option.db_cleanup: return @@ -1396,7 +1397,7 @@ def hook_lineage_collector(): @pytest.fixture def clean_dags_and_dagruns(): """Fixture that cleans the database before and after every test.""" - from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs + from tests_common.test_utils.db import clear_db_dags, clear_db_runs clear_db_runs() clear_db_dags() @@ -1407,7 +1408,7 @@ def clean_dags_and_dagruns(): @pytest.fixture(scope="session") def app(): - from dev.tests_common.test_utils.config import conf_vars + from tests_common.test_utils.config import conf_vars with conf_vars({("fab", "auth_rate_limited"): "False"}): from airflow.www import app diff --git a/dev/tests_common/test_utils/README.md b/tests_common/test_utils/README.md similarity index 100% rename from dev/tests_common/test_utils/README.md rename to tests_common/test_utils/README.md diff --git a/dev/tests_common/test_utils/__init__.py b/tests_common/test_utils/__init__.py similarity index 94% rename from dev/tests_common/test_utils/__init__.py rename to tests_common/test_utils/__init__.py index e440178fae6c7..c27a6b55209c1 100644 --- a/dev/tests_common/test_utils/__init__.py +++ b/tests_common/test_utils/__init__.py @@ -19,4 +19,4 @@ from pathlib import Path -AIRFLOW_MAIN_FOLDER = Path(__file__).parents[3] +AIRFLOW_MAIN_FOLDER = Path(__file__).parents[2] diff --git a/dev/tests_common/test_utils/api_connexion_utils.py b/tests_common/test_utils/api_connexion_utils.py similarity index 91% rename from dev/tests_common/test_utils/api_connexion_utils.py rename to tests_common/test_utils/api_connexion_utils.py index 2d273af2e6878..23a273d93557f 100644 --- a/dev/tests_common/test_utils/api_connexion_utils.py +++ b/tests_common/test_utils/api_connexion_utils.py @@ -20,8 +20,7 @@ from typing import TYPE_CHECKING from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP - -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES @@ -32,9 +31,7 @@ @contextmanager def create_test_client(app, user_name, role_name, permissions): - """ - Helper function to create a client with a temporary user which will be deleted once done - """ + """Create a client with a temporary user which will be deleted once done.""" client = app.test_client() with create_user_scope(app, username=user_name, role_name=role_name, permissions=permissions) as _: resp = client.post("/login/", data={"username": user_name, "password": user_name}) @@ -45,9 +42,10 @@ def create_test_client(app, user_name, role_name, permissions): @contextmanager def create_user_scope(app, username, **kwargs): """ - Helper function designed to be used with pytest fixture mainly. + Create user scope for use pytest fixture mainly. + It will create a user and provide it for the fixture via YIELD (generator) - then will tidy up once test is complete + then will tidy up once test is complete. """ test_user = create_user(app, username, **kwargs) diff --git a/dev/tests_common/test_utils/asserts.py b/tests_common/test_utils/asserts.py similarity index 95% rename from dev/tests_common/test_utils/asserts.py rename to tests_common/test_utils/asserts.py index 56bf8cc1fee57..59ca9bad968ea 100644 --- a/dev/tests_common/test_utils/asserts.py +++ b/tests_common/test_utils/asserts.py @@ -46,6 +46,8 @@ def _trim(s): class QueriesTraceRecord(NamedTuple): + """QueriesTraceRecord holds information about the query executed in the context.""" + module: str name: str lineno: int | None @@ -63,6 +65,8 @@ def __str__(self): class QueriesTraceInfo(NamedTuple): + """QueriesTraceInfo holds information about the queries executed in the context.""" + traces: tuple[QueriesTraceRecord, ...] @classmethod @@ -144,14 +148,15 @@ def assert_queries_count( session: Session | None = None, ): """ - Asserts that the number of queries is as expected with the margin applied + Assert that the number of queries is as expected with the margin applied. + The margin is helpful in case of complex cases where we do not want to change it every time we changed queries, but we want to catch cases where we spin out of control :param expected_count: expected number of queries :param message_fmt: message printed optionally if the number is exceeded :param margin: margin to add to expected number of calls :param stacklevel: limits the output stack trace to that numbers of frame - :param stacklevel_from_module: Filter stack trace from specific module + :param stacklevel_from_module: Filter stack trace from specific module. """ with count_queries( stacklevel=stacklevel, stacklevel_from_module=stacklevel_from_module, session=session diff --git a/dev/tests_common/test_utils/azure_system_helpers.py b/tests_common/test_utils/azure_system_helpers.py similarity index 93% rename from dev/tests_common/test_utils/azure_system_helpers.py rename to tests_common/test_utils/azure_system_helpers.py index b1a46617c082f..057124dea1f29 100644 --- a/dev/tests_common/test_utils/azure_system_helpers.py +++ b/tests_common/test_utils/azure_system_helpers.py @@ -28,9 +28,8 @@ from airflow.models import Connection from airflow.providers.microsoft.azure.hooks.fileshare import AzureFileShareHook from airflow.utils.process_utils import patch_environ - -from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER -from dev.tests_common.test_utils.system_tests_class import SystemTest +from tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from tests_common.test_utils.system_tests_class import SystemTest AZURE_DAG_FOLDER = os.path.join( AIRFLOW_MAIN_FOLDER, "airflow", "providers", "microsoft", "azure", "example_dags" @@ -44,7 +43,7 @@ @contextmanager def provide_wasb_default_connection(key_file_path: str): """ - Context manager to provide a temporary value for wasb_default connection + Context manager to provide a temporary value for wasb_default connection. :param key_file_path: Path to file with wasb_default credentials .json file. """ @@ -67,7 +66,8 @@ def provide_wasb_default_connection(key_file_path: str): @contextmanager def provide_azure_data_lake_default_connection(key_file_path: str): """ - Context manager to provide a temporary value for azure_data_lake_default connection + Provide a temporary value for azure_data_lake_default connection. + :param key_file_path: Path to file with azure_data_lake_default credentials .json file. """ required_fields = {"login", "password", "extra"} @@ -106,6 +106,8 @@ def provide_azure_fileshare(share_name: str, azure_fileshare_conn_id: str, file_ @pytest.mark.system("azure") class AzureSystemTest(SystemTest): + """Base class for Azure system tests.""" + @classmethod def create_share(cls, share_name: str, azure_fileshare_conn_id: str): hook = AzureFileShareHook(azure_fileshare_conn_id=azure_fileshare_conn_id) @@ -138,9 +140,7 @@ def upload_file_from_string( @classmethod def prepare_share(cls, share_name: str, azure_fileshare_conn_id: str, file_name: str, directory: str): - """ - Create share with a file in given directory. If directory is None, file is in root dir. - """ + """Create share with a file in given directory. If directory is None, file is in root dir.""" hook = AzureFileShareHook( azure_fileshare_conn_id=azure_fileshare_conn_id, share_name=share_name, diff --git a/dev/tests_common/test_utils/compat.py b/tests_common/test_utils/compat.py similarity index 99% rename from dev/tests_common/test_utils/compat.py rename to tests_common/test_utils/compat.py index 67a111350daff..bc04f798e027e 100644 --- a/dev/tests_common/test_utils/compat.py +++ b/tests_common/test_utils/compat.py @@ -116,7 +116,7 @@ def connection_to_dict( connection: Connection, *, prune_empty: bool = False, validate: bool = True ) -> dict[str, Any]: """ - Convert Connection to json-serializable dictionary (compatibility code for Airflow 2.7 tests) + Convert Connection to json-serializable dictionary (compatibility code for Airflow 2.7 tests). :param connection: connection to convert to dict :param prune_empty: Whether or not remove empty values. diff --git a/dev/tests_common/test_utils/config.py b/tests_common/test_utils/config.py similarity index 100% rename from dev/tests_common/test_utils/config.py rename to tests_common/test_utils/config.py diff --git a/dev/tests_common/test_utils/db.py b/tests_common/test_utils/db.py similarity index 97% rename from dev/tests_common/test_utils/db.py rename to tests_common/test_utils/db.py index 4b2dede05ead7..bf92c3a2582d8 100644 --- a/dev/tests_common/test_utils/db.py +++ b/tests_common/test_utils/db.py @@ -44,8 +44,7 @@ from airflow.security.permissions import RESOURCE_DAG_PREFIX from airflow.utils.db import add_default_pool_if_not_exists, create_default_connections, reflect_tables from airflow.utils.session import create_session - -from dev.tests_common.test_utils.compat import ( +from tests_common.test_utils.compat import ( AIRFLOW_V_2_10_PLUS, AssetDagRunQueue, AssetEvent, @@ -63,8 +62,7 @@ def initial_db_init(): from airflow.utils import db from airflow.www.extensions.init_appbuilder import init_appbuilder from airflow.www.extensions.init_auth_manager import get_auth_manager - - from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS db.resetdb() db.bootstrap_dagbag() @@ -106,7 +104,7 @@ def clear_db_assets(): session.query(DagScheduleAssetReference).delete() session.query(TaskOutletAssetReference).delete() if AIRFLOW_V_2_10_PLUS: - from dev.tests_common.test_utils.compat import AssetAliasModel + from tests_common.test_utils.compat import AssetAliasModel session.query(AssetAliasModel).delete() diff --git a/dev/tests_common/test_utils/decorators.py b/tests_common/test_utils/decorators.py similarity index 100% rename from dev/tests_common/test_utils/decorators.py rename to tests_common/test_utils/decorators.py diff --git a/dev/tests_common/test_utils/fake_datetime.py b/tests_common/test_utils/fake_datetime.py similarity index 92% rename from dev/tests_common/test_utils/fake_datetime.py rename to tests_common/test_utils/fake_datetime.py index 6af41e15efc0c..3aba78e767077 100644 --- a/dev/tests_common/test_utils/fake_datetime.py +++ b/tests_common/test_utils/fake_datetime.py @@ -21,9 +21,7 @@ class FakeDatetime(datetime): - """ - A fake replacement for datetime that can be mocked for testing. - """ + """A fake replacement for datetime that can be mocked for testing.""" def __new__(cls, *args, **kwargs): return datetime.__new__(datetime, *args, **kwargs) diff --git a/dev/tests_common/test_utils/gcp_system_helpers.py b/tests_common/test_utils/gcp_system_helpers.py similarity index 95% rename from dev/tests_common/test_utils/gcp_system_helpers.py rename to tests_common/test_utils/gcp_system_helpers.py index e17679bd8eb56..82b16da8490b7 100644 --- a/dev/tests_common/test_utils/gcp_system_helpers.py +++ b/tests_common/test_utils/gcp_system_helpers.py @@ -30,10 +30,10 @@ import airflow.providers.google from airflow.providers.google.cloud.utils.credentials_provider import provide_gcp_conn_and_credentials +from tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from tests_common.test_utils.logging_command_executor import CommandExecutor +from tests_common.test_utils.system_tests_class import SystemTest -from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER -from dev.tests_common.test_utils.logging_command_executor import CommandExecutor -from dev.tests_common.test_utils.system_tests_class import SystemTest from providers.tests.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY, GCP_SECRET_MANAGER_KEY GCP_DIR = Path(airflow.providers.google.__file__).parent @@ -49,7 +49,7 @@ def resolve_full_gcp_key_path(key: str) -> str: """ - Returns path full path to provided GCP key. + Return path full path to provided GCP key. :param key: Name of the GCP key, for example ``my_service.json`` :returns: Full path to the key @@ -66,7 +66,7 @@ def provide_gcp_context( project_id: str | None = None, ): """ - Context manager that provides: + Provide context manager for GCP. - GCP credentials for application supporting `Application Default Credentials (ADC) strategy `__. @@ -115,6 +115,8 @@ def provide_gcs_bucket(bucket_name: str): @pytest.mark.system("google") class GoogleSystemTest(SystemTest): + """Base class for Google system tests.""" + @staticmethod def execute_cmd(*args, **kwargs): executor = CommandExecutor() @@ -132,10 +134,7 @@ def _service_key(): def execute_with_ctx( cls, cmd: list[str], key: str = GCP_GCS_KEY, project_id=None, scopes=None, silent: bool = False ): - """ - Executes command with context created by provide_gcp_context and activated - service key. - """ + """Execute command with context created by provide_gcp_context and activated service key.""" current_project_id = project_id or cls._project_id() with provide_gcp_context(key, project_id=current_project_id, scopes=scopes): cls.execute_cmd(cmd=cmd, silent=silent) diff --git a/dev/tests_common/test_utils/get_all_tests.py b/tests_common/test_utils/get_all_tests.py similarity index 92% rename from dev/tests_common/test_utils/get_all_tests.py rename to tests_common/test_utils/get_all_tests.py index 5081243c9e93a..000f2e63b34bd 100644 --- a/dev/tests_common/test_utils/get_all_tests.py +++ b/tests_common/test_utils/get_all_tests.py @@ -16,9 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -Gets all tests cases from xunit file. -""" +"""Gets all tests cases from xunit file.""" from __future__ import annotations @@ -28,7 +26,7 @@ def last_replace(s, old, new, number_of_occurrences): """ - Replaces last n occurrences of the old string with the new one within the string provided + Replace last n occurrences of the old string with the new one within the string provided. :param s: string to replace occurrences with :param old: old string @@ -42,7 +40,7 @@ def last_replace(s, old, new, number_of_occurrences): def print_all_cases(xunit_test_file_path): """ - Prints all test cases read from the xunit test file + Print all test cases read from the xunit test file. :param xunit_test_file_path: path of the xunit file :return: None diff --git a/dev/tests_common/test_utils/hdfs_utils.py b/tests_common/test_utils/hdfs_utils.py similarity index 95% rename from dev/tests_common/test_utils/hdfs_utils.py rename to tests_common/test_utils/hdfs_utils.py index f429c9ab0cf45..26eb3a78a47de 100644 --- a/dev/tests_common/test_utils/hdfs_utils.py +++ b/tests_common/test_utils/hdfs_utils.py @@ -19,6 +19,8 @@ class FakeWebHDFSHook: + """Fake WebHDFSHook for testing purposes.""" + def __init__(self, conn_id): self.conn_id = conn_id diff --git a/dev/tests_common/test_utils/logging_command_executor.py b/tests_common/test_utils/logging_command_executor.py similarity index 97% rename from dev/tests_common/test_utils/logging_command_executor.py rename to tests_common/test_utils/logging_command_executor.py index 1773cda30d6f6..d41a3ea1ad9dc 100644 --- a/dev/tests_common/test_utils/logging_command_executor.py +++ b/tests_common/test_utils/logging_command_executor.py @@ -25,10 +25,12 @@ class CommandExecutionError(Exception): - """Raise in case of error during command execution""" + """Raise in case of error during command execution.""" class LoggingCommandExecutor(LoggingMixin): + """Logging command executor.""" + def execute_cmd(self, cmd, silent=False, cwd=None, env=None): if silent: self.log.info("Executing in silent mode: '%s'", " ".join(shlex.quote(c) for c in cmd)) @@ -70,6 +72,8 @@ def check_output(self, cmd): class CommandExecutor(LoggingCommandExecutor): """ + Command executor. + Due to 'LoggingCommandExecutor' class just returns the status code of command execution ('execute_cmd' method) and continues to perform code with possible errors, separate inherited 'CommandExecutor' class was created to use it if you need to break code performing diff --git a/dev/tests_common/test_utils/mapping.py b/tests_common/test_utils/mapping.py similarity index 100% rename from dev/tests_common/test_utils/mapping.py rename to tests_common/test_utils/mapping.py diff --git a/dev/tests_common/test_utils/mock_executor.py b/tests_common/test_utils/mock_executor.py similarity index 97% rename from dev/tests_common/test_utils/mock_executor.py rename to tests_common/test_utils/mock_executor.py index 5080290500c9c..506c0447589ac 100644 --- a/dev/tests_common/test_utils/mock_executor.py +++ b/tests_common/test_utils/mock_executor.py @@ -28,9 +28,7 @@ class MockExecutor(BaseExecutor): - """ - TestExecutor is used for unit testing purposes. - """ + """TestExecutor is used for unit testing purposes.""" supports_pickling = False mock_module_path = "mock.executor.path" @@ -98,6 +96,8 @@ def change_state(self, key, state, info=None): def mock_task_fail(self, dag_id, task_id, run_id: str, try_number=1): """ + Mock for test failures. + Set the mock outcome of running this particular task instances to FAILED. diff --git a/dev/tests_common/test_utils/mock_operators.py b/tests_common/test_utils/mock_operators.py similarity index 84% rename from dev/tests_common/test_utils/mock_operators.py rename to tests_common/test_utils/mock_operators.py index 0df0afec824c3..7cc807a3a485b 100644 --- a/dev/tests_common/test_utils/mock_operators.py +++ b/tests_common/test_utils/mock_operators.py @@ -23,8 +23,7 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.xcom import XCom - -from dev.tests_common.test_utils.compat import BaseOperatorLink +from tests_common.test_utils.compat import BaseOperatorLink if TYPE_CHECKING: import jinja2 @@ -47,12 +46,16 @@ def execute(self, context: Context): class NestedFields: + """Nested fields for testing purposes.""" + def __init__(self, field_1, field_2): self.field_1 = field_1 self.field_2 = field_2 class MockOperatorWithNestedFields(BaseOperator): + """Operator with nested fields for testing purposes.""" + template_fields: Sequence[str] = ("arg1", "arg2") def __init__(self, arg1: str = "", arg2: NestedFields | None = None, **kwargs): @@ -85,9 +88,7 @@ def execute(self, context: Context): class AirflowLink(BaseOperatorLink): - """ - Operator Link for Apache Airflow Website - """ + """Operator Link for Apache Airflow Website.""" name = "airflow" @@ -95,19 +96,23 @@ def get_link(self, operator, *, ti_key): return "https://airflow.apache.org" -class Dummy2TestOperator(BaseOperator): +class EmptyExtraLinkTestOperator(BaseOperator): """ + Empty test operator with extra link. + Example of an Operator that has an extra operator link - and will be overridden by the one defined in tests/plugins/test_plugin.py + and will be overridden by the one defined in tests/plugins/test_plugin.py. """ operator_extra_links = (AirflowLink(),) -class Dummy3TestOperator(BaseOperator): +class EmptyNoExtraLinkTestOperator(BaseOperator): """ + Empty test operator without extra operator link. + Example of an operator that has no extra Operator link. - An operator link would be added to this operator via Airflow plugin + An operator link would be added to this operator via Airflow plugin. """ operator_extra_links = () @@ -115,6 +120,8 @@ class Dummy3TestOperator(BaseOperator): @attr.s(auto_attribs=True) class CustomBaseIndexOpLink(BaseOperatorLink): + """Custom Operator Link for Google BigQuery Console.""" + index: int = attr.ib() @property @@ -134,6 +141,8 @@ def get_link(self, operator, *, ti_key): class CustomOpLink(BaseOperatorLink): + """Custom Operator with Link for Google Custom Search.""" + name = "Google Custom" def get_link(self, operator, *, ti_key): @@ -146,14 +155,14 @@ def get_link(self, operator, *, ti_key): class CustomOperator(BaseOperator): + """Custom Operator for testing purposes.""" + template_fields = ["bash_command"] custom_operator_name = "@custom" @property def operator_extra_links(self): - """ - Return operator extra links - """ + """Return operator extra links.""" if isinstance(self.bash_command, str) or self.bash_command is None: return (CustomOpLink(),) return (CustomBaseIndexOpLink(i) for i, _ in enumerate(self.bash_command)) @@ -168,33 +177,27 @@ def execute(self, context: Context): class GoogleLink(BaseOperatorLink): - """ - Operator Link for Apache Airflow Website for Google - """ + """Operator Link for Apache Airflow Website for Google.""" name = "google" - operators = [Dummy3TestOperator, CustomOperator] + operators = [EmptyNoExtraLinkTestOperator, CustomOperator] def get_link(self, operator, *, ti_key): return "https://www.google.com" class AirflowLink2(BaseOperatorLink): - """ - Operator Link for Apache Airflow Website for 1.10.5 - """ + """Operator Link for Apache Airflow Website for 1.10.5.""" name = "airflow" - operators = [Dummy2TestOperator, Dummy3TestOperator] + operators = [EmptyExtraLinkTestOperator, EmptyNoExtraLinkTestOperator] def get_link(self, operator, *, ti_key): return "https://airflow.apache.org/1.10.5/" class GithubLink(BaseOperatorLink): - """ - Operator Link for Apache Airflow GitHub - """ + """Operator Link for Apache Airflow GitHub.""" name = "github" @@ -203,6 +206,8 @@ def get_link(self, operator, *, ti_key): class DeprecatedOperator(BaseOperator): + """Deprecated Operator for testing purposes.""" + def __init__(self, **kwargs): warnings.warn("This operator is deprecated.", DeprecationWarning, stacklevel=2) super().__init__(**kwargs) diff --git a/dev/tests_common/test_utils/mock_plugins.py b/tests_common/test_utils/mock_plugins.py similarity index 100% rename from dev/tests_common/test_utils/mock_plugins.py rename to tests_common/test_utils/mock_plugins.py diff --git a/dev/tests_common/test_utils/mock_security_manager.py b/tests_common/test_utils/mock_security_manager.py similarity index 89% rename from dev/tests_common/test_utils/mock_security_manager.py rename to tests_common/test_utils/mock_security_manager.py index 6b9f45e3d8410..ee0f49372afa9 100644 --- a/dev/tests_common/test_utils/mock_security_manager.py +++ b/tests_common/test_utils/mock_security_manager.py @@ -16,13 +16,15 @@ # under the License. from __future__ import annotations -from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0", __file__): from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride class MockSecurityManager(FabAirflowSecurityManagerOverride): + """Mock Security Manager for testing purposes.""" + VIEWER_VMS = { "Airflow", } diff --git a/dev/tests_common/test_utils/operators/__init__.py b/tests_common/test_utils/operators/__init__.py similarity index 100% rename from dev/tests_common/test_utils/operators/__init__.py rename to tests_common/test_utils/operators/__init__.py diff --git a/dev/tests_common/test_utils/operators/postgres_local_executor.cfg b/tests_common/test_utils/operators/postgres_local_executor.cfg similarity index 100% rename from dev/tests_common/test_utils/operators/postgres_local_executor.cfg rename to tests_common/test_utils/operators/postgres_local_executor.cfg diff --git a/dev/tests_common/test_utils/perf/__init__.py b/tests_common/test_utils/perf/__init__.py similarity index 100% rename from dev/tests_common/test_utils/perf/__init__.py rename to tests_common/test_utils/perf/__init__.py diff --git a/dev/tests_common/test_utils/perf/perf_kit/__init__.py b/tests_common/test_utils/perf/perf_kit/__init__.py similarity index 99% rename from dev/tests_common/test_utils/perf/perf_kit/__init__.py rename to tests_common/test_utils/perf/perf_kit/__init__.py index 011276155429f..84f0f335b1b19 100644 --- a/dev/tests_common/test_utils/perf/perf_kit/__init__.py +++ b/tests_common/test_utils/perf/perf_kit/__init__.py @@ -15,13 +15,10 @@ # specific language governing permissions and limitations # under the License. """ - -Perf-kit -======== +Perf-kit. Useful decorators and context managers used when testing the performance of various Airflow components. - Content ======== diff --git a/dev/tests_common/test_utils/perf/perf_kit/memory.py b/tests_common/test_utils/perf/perf_kit/memory.py similarity index 94% rename from dev/tests_common/test_utils/perf/perf_kit/memory.py rename to tests_common/test_utils/perf/perf_kit/memory.py index b236630f5f6c7..195a22d0bd9ff 100644 --- a/dev/tests_common/test_utils/perf/perf_kit/memory.py +++ b/tests_common/test_utils/perf/perf_kit/memory.py @@ -37,7 +37,7 @@ def _human_readable_size(size, decimal_places=3): class TraceMemoryResult: - """Trace results of memory,""" + """Trace results of memory,.""" def __init__(self): self.before = 0 @@ -48,7 +48,7 @@ def __init__(self): @contextmanager def trace_memory(human_readable=True, gc_collect=False): """ - Calculates the amount of difference in free memory before and after script execution. + Decorate function and calculate the amount of difference in free memory before and after script execution. In other words, how much data the code snippet has used up memory. diff --git a/dev/tests_common/test_utils/perf/perf_kit/python.py b/tests_common/test_utils/perf/perf_kit/python.py similarity index 90% rename from dev/tests_common/test_utils/perf/perf_kit/python.py rename to tests_common/test_utils/perf/perf_kit/python.py index 2589d9753451b..054fd0c0897a3 100644 --- a/dev/tests_common/test_utils/perf/perf_kit/python.py +++ b/tests_common/test_utils/perf/perf_kit/python.py @@ -30,8 +30,9 @@ @contextlib.contextmanager def pyspy(): """ - This decorator provide deterministic profiling. It generate and save flame graph to file. It uses``pyspy`` - internally. + Decorate methods for deterministic profiling. + + It generates and saves flame graph to file. It uses``pyspy` internally. Running py-spy inside of a docker container will also usually bring up a permissions denied error even when running as root. @@ -65,8 +66,9 @@ def pyspy(): @contextlib.contextmanager def profiled(print_callers=False): """ - This decorator provide deterministic profiling. It uses ``cProfile`` internally. It generates statistic - and print on the screen. + Decorate function with deterministic profiling. + + It uses ``cProfile`` internally. It generates statistics and prints on the screen. """ profile = cProfile.Profile() profile.enable() @@ -86,10 +88,7 @@ def profiled(print_callers=False): if __name__ == "__main__": def case(): - """ - Load modules. - :return: - """ + """Load modules.""" import logging import airflow diff --git a/dev/tests_common/test_utils/perf/perf_kit/repeat_and_time.py b/tests_common/test_utils/perf/perf_kit/repeat_and_time.py similarity index 92% rename from dev/tests_common/test_utils/perf/perf_kit/repeat_and_time.py rename to tests_common/test_utils/perf/perf_kit/repeat_and_time.py index 290cdeabc8766..ef179d8ea835f 100644 --- a/dev/tests_common/test_utils/perf/perf_kit/repeat_and_time.py +++ b/tests_common/test_utils/perf/perf_kit/repeat_and_time.py @@ -36,7 +36,7 @@ def __init__(self): @contextlib.contextmanager def timing(repeat_count: int = 1): """ - Measures code execution time. + Measure code execution time. :param repeat_count: If passed, the result will be divided by the value. """ @@ -60,7 +60,7 @@ def timing(repeat_count: int = 1): def repeat(repeat_count=5): """ - Function decorators that repeat function many times. + Decorate functions that repeats many times. :param repeat_count: The repeat count """ @@ -79,13 +79,13 @@ def wrap(*args, **kwargs): class TimeoutException(Exception): - """Exception when the test timeo uts""" + """The exception raised when the test timeouts.""" @contextlib.contextmanager def timeout(seconds=1): """ - Executes code only limited seconds. If the code does not end during this time, it will be interrupted. + Execute code only limited seconds. If the code does not end during this time, it will be interrupted. :param seconds: Number of seconds """ @@ -114,7 +114,7 @@ def handle_timeout(signum, frame): if __name__ == "__main__": def monte_carlo(total=10000): - """Monte Carlo""" + """Monte Carlo.""" inside = 0 for _ in range(total): @@ -140,7 +140,7 @@ def monte_carlo(total=10000): @repeat(REPEAT_COUNT) @timing() def get_pi(): - """Returns PI value:""" + """Return PI value.""" return monte_carlo() res = get_pi() diff --git a/dev/tests_common/test_utils/perf/perf_kit/sqlalchemy.py b/tests_common/test_utils/perf/perf_kit/sqlalchemy.py similarity index 97% rename from dev/tests_common/test_utils/perf/perf_kit/sqlalchemy.py rename to tests_common/test_utils/perf/perf_kit/sqlalchemy.py index f7fe68d779371..ff17f17b1962b 100644 --- a/dev/tests_common/test_utils/perf/perf_kit/sqlalchemy.py +++ b/tests_common/test_utils/perf/perf_kit/sqlalchemy.py @@ -35,7 +35,7 @@ def _pretty_format_sql(text: str): class TraceQueries: """ - Tracking SQL queries in a code block. + Track SQL queries in a code block. :param display_num: If True, displays the query number. :param display_time: If True, displays the query execution time. @@ -73,7 +73,7 @@ def before_cursor_execute( executemany, ): """ - Executed before cursor. + Execute before cursor. :param conn: connection :param cursor: cursor @@ -83,7 +83,6 @@ def before_cursor_execute( :param executemany: whether many statements executed :return: """ - conn.info.setdefault("query_start_time", []).append(time.monotonic()) self.query_count += 1 @@ -97,7 +96,7 @@ def after_cursor_execute( executemany, ): """ - Executed after cursor. + Execute after cursor. :param conn: connection :param cursor: cursor @@ -154,9 +153,7 @@ def __exit__(self, type_, value, traceback): class CountQueriesResult: - """ - Counter for number of queries. - """ + """Counter for number of queries.""" def __init__(self): self.count = 0 @@ -198,7 +195,7 @@ def after_cursor_execute( executemany, ): """ - Executed after cursor. + Execute after cursor. :param conn: connection :param cursor: cursor @@ -215,7 +212,7 @@ def after_cursor_execute( if __name__ == "__main__": # Example: def case(): - """Case of logging om/""" + """Case of logging om/.""" import logging from unittest import mock diff --git a/dev/tests_common/test_utils/permissions.py b/tests_common/test_utils/permissions.py similarity index 93% rename from dev/tests_common/test_utils/permissions.py rename to tests_common/test_utils/permissions.py index 46e964c1f6190..3abb5d0fbcdab 100644 --- a/dev/tests_common/test_utils/permissions.py +++ b/tests_common/test_utils/permissions.py @@ -21,6 +21,8 @@ def _resource_name(dag_id: str, resource_name: str) -> str: """ + Return resource name for given dag_id and resource_name in backwards compatible way. + This method is to keep compatibility with new FAB versions running with old airflow versions. """ diff --git a/dev/tests_common/test_utils/providers.py b/tests_common/test_utils/providers.py similarity index 94% rename from dev/tests_common/test_utils/providers.py rename to tests_common/test_utils/providers.py index 02ca9bed41989..34288d8680d29 100644 --- a/dev/tests_common/test_utils/providers.py +++ b/tests_common/test_utils/providers.py @@ -21,7 +21,7 @@ def object_exists(path: str): - """Returns true if importable python object is there.""" + """Return true if importable python object is there.""" from airflow.utils.module_loading import import_string try: @@ -33,7 +33,7 @@ def object_exists(path: str): def get_provider_version(provider_name): """ - Returns provider version given provider package name. + Return provider version given provider package name. Example:: if provider_version("apache-airflow-providers-cncf-kubernetes") >= (6, 0): diff --git a/dev/tests_common/test_utils/remote_user_api_auth_backend.py b/tests_common/test_utils/remote_user_api_auth_backend.py similarity index 97% rename from dev/tests_common/test_utils/remote_user_api_auth_backend.py rename to tests_common/test_utils/remote_user_api_auth_backend.py index 59df201e530e4..74f705bcf709a 100644 --- a/dev/tests_common/test_utils/remote_user_api_auth_backend.py +++ b/tests_common/test_utils/remote_user_api_auth_backend.py @@ -46,7 +46,7 @@ def _lookup_user(username: str): def requires_authentication(function: T): - """Decorator for functions that require authentication""" + """Decorate functions that require authentication.""" @wraps(function) def decorated(*args, **kwargs): diff --git a/dev/tests_common/test_utils/reset_warning_registry.py b/tests_common/test_utils/reset_warning_registry.py similarity index 96% rename from dev/tests_common/test_utils/reset_warning_registry.py rename to tests_common/test_utils/reset_warning_registry.py index d7d5c1ffe577b..4c6ae8d26b1a2 100644 --- a/dev/tests_common/test_utils/reset_warning_registry.py +++ b/tests_common/test_utils/reset_warning_registry.py @@ -34,7 +34,9 @@ # https://stackoverflow.com/questions/19428761/python-showing-once-warnings-again-resetting-all-warning-registries class reset_warning_registry: """ - context manager which archives & clears warning registry for duration of + Reset warning registry context manager. + + The context manager archives & clears warning registry for duration of context. :param pattern: diff --git a/dev/tests_common/test_utils/salesforce_system_helpers.py b/tests_common/test_utils/salesforce_system_helpers.py similarity index 100% rename from dev/tests_common/test_utils/salesforce_system_helpers.py rename to tests_common/test_utils/salesforce_system_helpers.py diff --git a/dev/tests_common/test_utils/sftp_system_helpers.py b/tests_common/test_utils/sftp_system_helpers.py similarity index 99% rename from dev/tests_common/test_utils/sftp_system_helpers.py rename to tests_common/test_utils/sftp_system_helpers.py index baed591b9004b..a599f120cb51b 100644 --- a/dev/tests_common/test_utils/sftp_system_helpers.py +++ b/tests_common/test_utils/sftp_system_helpers.py @@ -30,7 +30,7 @@ @contextmanager def provide_sftp_default_connection(key_file_path: str): """ - Context manager to provide a temporary value for sftp_default connection + Context manager to provide a temporary value for sftp_default connection. :param key_file_path: Path to file with sftp_default credentials .json file. """ diff --git a/dev/tests_common/test_utils/system_tests.py b/tests_common/test_utils/system_tests.py similarity index 100% rename from dev/tests_common/test_utils/system_tests.py rename to tests_common/test_utils/system_tests.py diff --git a/dev/tests_common/test_utils/system_tests_class.py b/tests_common/test_utils/system_tests_class.py similarity index 95% rename from dev/tests_common/test_utils/system_tests_class.py rename to tests_common/test_utils/system_tests_class.py index 5abdca96bee06..61f0b71d4c781 100644 --- a/dev/tests_common/test_utils/system_tests_class.py +++ b/tests_common/test_utils/system_tests_class.py @@ -28,9 +28,8 @@ from airflow.configuration import AIRFLOW_HOME, AirflowConfigParser, get_airflow_config from airflow.exceptions import AirflowException - -from dev.tests_common.test_utils import AIRFLOW_MAIN_FOLDER -from dev.tests_common.test_utils.logging_command_executor import get_executor +from tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from tests_common.test_utils.logging_command_executor import get_executor DEFAULT_DAG_FOLDER = os.path.join(AIRFLOW_MAIN_FOLDER, "airflow", "example_dags") @@ -42,9 +41,7 @@ def get_default_logs_if_none(logs: str | None) -> str: def resolve_logs_folder() -> str: - """ - Returns LOGS folder specified in current Airflow config. - """ + """Return LOGS folder specified in current Airflow config.""" config_file = get_airflow_config(AIRFLOW_HOME) conf = AirflowConfigParser() conf.read(config_file) @@ -59,6 +56,8 @@ def resolve_logs_folder() -> str: class SystemTest: + """Base class for system tests.""" + log: logging.Logger @staticmethod @@ -70,6 +69,8 @@ def setup_logger(request): @pytest.fixture(autouse=True) def setup_system(self): """ + Set up system tests. + We want to avoid random errors while database got reset - those Are apparently triggered by parser trying to parse DAGs while The tables are dropped. We move the dags temporarily out of the dags folder diff --git a/dev/tests_common/test_utils/terraform.py b/tests_common/test_utils/terraform.py similarity index 93% rename from dev/tests_common/test_utils/terraform.py rename to tests_common/test_utils/terraform.py index b600ef6643b19..387d8cb28cafc 100644 --- a/dev/tests_common/test_utils/terraform.py +++ b/tests_common/test_utils/terraform.py @@ -16,10 +16,12 @@ # under the License. from __future__ import annotations -from dev.tests_common.test_utils.system_tests_class import SystemTest +from tests_common.test_utils.system_tests_class import SystemTest class Terraform(SystemTest): + """Base class for Terraform tests.""" + TERRAFORM_DIR: str def setup_method(self) -> None: diff --git a/dev/tests_common/test_utils/timetables.py b/tests_common/test_utils/timetables.py similarity index 97% rename from dev/tests_common/test_utils/timetables.py rename to tests_common/test_utils/timetables.py index 849968e76110a..f2053d83b24ac 100644 --- a/dev/tests_common/test_utils/timetables.py +++ b/tests_common/test_utils/timetables.py @@ -30,6 +30,8 @@ def delta_timetable(delta) -> DeltaDataIntervalTimetable: class CustomSerializationTimetable(Timetable): + """Custom timetable for testing serialization.""" + def __init__(self, value: str): self.value = value diff --git a/dev/tests_common/test_utils/watcher.py b/tests_common/test_utils/watcher.py similarity index 86% rename from dev/tests_common/test_utils/watcher.py rename to tests_common/test_utils/watcher.py index 4ee2e2e665006..6c6402cc6bd38 100644 --- a/dev/tests_common/test_utils/watcher.py +++ b/tests_common/test_utils/watcher.py @@ -23,6 +23,10 @@ @task(trigger_rule=TriggerRule.ONE_FAILED, retries=0) def watcher(): - """Watcher task raises an AirflowException and is used to 'watch' tasks for failures - and propagates fail status to the whole DAG Run""" + """ + Watcher task. + + Watcher task raises an AirflowException and is used to 'watch' tasks for failures + and propagates fail status to the whole DAG Run. + """ raise AirflowException("Failing task because one or more upstream tasks failed.") diff --git a/dev/tests_common/test_utils/www.py b/tests_common/test_utils/www.py similarity index 100% rename from dev/tests_common/test_utils/www.py rename to tests_common/test_utils/www.py From 94c14ae1667e884fa18e541e576d3de3b6870fe3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 15 Oct 2024 11:29:40 +0800 Subject: [PATCH 113/199] Add AssetActive model (#42612) --- airflow/api_fastapi/views/ui/assets.py | 2 +- airflow/dag_processing/collection.py | 20 +- airflow/jobs/scheduler_job_runner.py | 21 +- ...5_3_0_0_add_name_field_to_dataset_model.py | 4 +- .../versions/0037_3_0_0_add_asset_active.py | 88 ++ airflow/models/asset.py | 59 +- airflow/models/dag.py | 1 + airflow/serialization/pydantic/asset.py | 1 - airflow/utils/db.py | 2 +- airflow/www/views.py | 4 +- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 1253 +++++++++-------- docs/apache-airflow/migrations-ref.rst | 4 +- tests/jobs/test_scheduler_job.py | 21 +- tests/models/test_dag.py | 12 +- tests/utils/test_db_cleanup.py | 1 + tests/www/views/test_views_dataset.py | 116 +- 17 files changed, 878 insertions(+), 733 deletions(-) create mode 100644 airflow/migrations/versions/0037_3_0_0_add_asset_active.py diff --git a/airflow/api_fastapi/views/ui/assets.py b/airflow/api_fastapi/views/ui/assets.py index 4a4ad1d0df9b4..1a198745f11f3 100644 --- a/airflow/api_fastapi/views/ui/assets.py +++ b/airflow/api_fastapi/views/ui/assets.py @@ -77,7 +77,7 @@ async def next_run_assets( ), isouter=True, ) - .where(DagScheduleAssetReference.dag_id == dag_id, ~AssetModel.is_orphaned) + .where(DagScheduleAssetReference.dag_id == dag_id, AssetModel.active.has()) .group_by(AssetModel.id, AssetModel.uri) .order_by(AssetModel.uri) ) diff --git a/airflow/dag_processing/collection.py b/airflow/dag_processing/collection.py index c8ce5dc873afa..f68ffbf331dd2 100644 --- a/airflow/dag_processing/collection.py +++ b/airflow/dag_processing/collection.py @@ -31,13 +31,13 @@ import logging from typing import TYPE_CHECKING, NamedTuple -from sqlalchemy import func, select +from sqlalchemy import func, select, tuple_ from sqlalchemy.orm import joinedload, load_only -from sqlalchemy.sql import expression from airflow.assets import Asset, AssetAlias from airflow.assets.manager import asset_manager from airflow.models.asset import ( + AssetActive, AssetAliasModel, AssetModel, DagScheduleAssetAliasReference, @@ -298,8 +298,6 @@ def add_assets(self, *, session: Session) -> dict[str, AssetModel]: orm_assets: dict[str, AssetModel] = { am.uri: am for am in session.scalars(select(AssetModel).where(AssetModel.uri.in_(self.assets))) } - for model in orm_assets.values(): - model.is_orphaned = expression.false() orm_assets.update( (model.uri, model) for model in asset_manager.create_assets( @@ -328,6 +326,20 @@ def add_asset_aliases(self, *, session: Session) -> dict[str, AssetAliasModel]: ) return orm_aliases + def add_asset_active_references(self, assets: Collection[AssetModel], *, session: Session) -> None: + existing_entries = set( + session.execute( + select(AssetActive.name, AssetActive.uri).where( + tuple_(AssetActive.name, AssetActive.uri).in_((asset.name, asset.uri) for asset in assets) + ) + ) + ) + session.add_all( + AssetActive.for_asset(asset) + for asset in assets + if (asset.name, asset.uri) not in existing_entries + ) + def add_dag_asset_references( self, dags: dict[str, DagModel], diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index f6821f57aa83c..30f58885a9aa2 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -45,6 +45,7 @@ from airflow.jobs.job import Job, perform_heartbeat from airflow.models import Log from airflow.models.asset import ( + AssetActive, AssetDagRunQueue, AssetEvent, AssetModel, @@ -2062,15 +2063,14 @@ def _cleanup_stale_dags(self, session: Session = NEW_SESSION) -> None: SerializedDagModel.remove_dag(dag_id=dag.dag_id, session=session) session.flush() - def _set_orphaned(self, asset: AssetModel) -> int: - self.log.info("Orphaning unreferenced asset '%s'", asset.uri) - asset.is_orphaned = expression.true() - return 1 + def _get_orphaning_identifier(self, asset: AssetModel) -> tuple[str, str]: + self.log.info("Orphaning unreferenced %s", asset) + return asset.name, asset.uri @provide_session def _orphan_unreferenced_assets(self, session: Session = NEW_SESSION) -> None: """ - Detect orphaned assets and set is_orphaned flag to True. + Detect orphaned assets and remove their active entry. An orphaned asset is no longer referenced in any DAG schedule parameters or task outlets. """ @@ -2085,7 +2085,7 @@ def _orphan_unreferenced_assets(self, session: Session = NEW_SESSION) -> None: isouter=True, ) .group_by(AssetModel.id) - .where(~AssetModel.is_orphaned) + .where(AssetModel.active.has()) .having( and_( func.count(DagScheduleAssetReference.dag_id) == 0, @@ -2094,8 +2094,13 @@ def _orphan_unreferenced_assets(self, session: Session = NEW_SESSION) -> None: ) ) - updated_count = sum(self._set_orphaned(asset) for asset in orphaned_asset_query) - Stats.gauge("asset.orphaned", updated_count) + orphaning_identifiers = [self._get_orphaning_identifier(asset) for asset in orphaned_asset_query] + session.execute( + delete(AssetActive).where( + tuple_in_condition((AssetActive.name, AssetActive.uri), orphaning_identifiers) + ) + ) + Stats.gauge("asset.orphaned", len(orphaning_identifiers)) def _executor_to_tis(self, tis: list[TaskInstance]) -> dict[BaseExecutor, list[TaskInstance]]: """Organize TIs into lists per their respective executor.""" diff --git a/airflow/migrations/versions/0035_3_0_0_add_name_field_to_dataset_model.py b/airflow/migrations/versions/0035_3_0_0_add_name_field_to_dataset_model.py index 6016dd9658908..2460b6956cff6 100644 --- a/airflow/migrations/versions/0035_3_0_0_add_name_field_to_dataset_model.py +++ b/airflow/migrations/versions/0035_3_0_0_add_name_field_to_dataset_model.py @@ -63,7 +63,9 @@ def upgrade(): batch_op.add_column(sa.Column("name", _STRING_COLUMN_TYPE)) batch_op.add_column(sa.Column("group", _STRING_COLUMN_TYPE, default=str, nullable=False)) # Fill name from uri column. - Session(bind=op.get_bind()).execute(sa.text("update dataset set name=uri")) + with Session(bind=op.get_bind()) as session: + session.execute(sa.text("update dataset set name=uri")) + session.commit() # Set the name column non-nullable. # Now with values in there, we can create the new unique constraint and index. # Due to MySQL restrictions, we are also reducing the length on uri. diff --git a/airflow/migrations/versions/0037_3_0_0_add_asset_active.py b/airflow/migrations/versions/0037_3_0_0_add_asset_active.py new file mode 100644 index 0000000000000..422bc440dba85 --- /dev/null +++ b/airflow/migrations/versions/0037_3_0_0_add_asset_active.py @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Add AssetActive to track orphaning instead of a flag. + +Revision ID: 5a5d66100783 +Revises: c3389cd7793f +Create Date: 2024-10-01 08:39:48.997198 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = "5a5d66100783" +down_revision = "c3389cd7793f" +branch_labels = None +depends_on = None +airflow_version = "3.0.0" + +_STRING_COLUMN_TYPE = sa.String(length=1500).with_variant( + sa.String(length=1500, collation="latin1_general_cs"), + dialect_name="mysql", +) + + +def upgrade(): + op.create_table( + "asset_active", + sa.Column("name", _STRING_COLUMN_TYPE, nullable=False), + sa.Column("uri", _STRING_COLUMN_TYPE, nullable=False), + sa.PrimaryKeyConstraint("name", "uri", name="asset_active_pkey"), + sa.ForeignKeyConstraint( + columns=["name", "uri"], + refcolumns=["dataset.name", "dataset.uri"], + name="asset_active_asset_name_uri_fkey", + ondelete="CASCADE", + ), + sa.Index("idx_asset_active_name_unique", "name", unique=True), + sa.Index("idx_asset_active_uri_unique", "uri", unique=True), + ) + with Session(bind=op.get_bind()) as session: + session.execute( + sa.text( + "insert into asset_active (name, uri) " + "select name, uri from dataset where is_orphaned = false" + ) + ) + session.commit() + with op.batch_alter_table("dataset", schema=None) as batch_op: + batch_op.drop_column("is_orphaned") + + +def downgrade(): + with op.batch_alter_table("dataset", schema=None) as batch_op: + batch_op.add_column( + sa.Column("is_orphaned", sa.Boolean, default=False, nullable=False, server_default="0") + ) + with Session(bind=op.get_bind()) as session: + session.execute( + sa.text( + "update dataset set is_orphaned = true " + "where exists (select 1 from asset_active " + "where dataset.name = asset_active.name and dataset.uri = asset_active.uri)" + ) + ) + session.commit() + op.drop_table("asset_active") diff --git a/airflow/models/asset.py b/airflow/models/asset.py index fb56bc4bf1ecf..d5ca0ea513f12 100644 --- a/airflow/models/asset.py +++ b/airflow/models/asset.py @@ -21,7 +21,6 @@ import sqlalchemy_jsonfield from sqlalchemy import ( - Boolean, Column, ForeignKey, ForeignKeyConstraint, @@ -192,7 +191,8 @@ class AssetModel(Base): created_at = Column(UtcDateTime, default=timezone.utcnow, nullable=False) updated_at = Column(UtcDateTime, default=timezone.utcnow, onupdate=timezone.utcnow, nullable=False) - is_orphaned = Column(Boolean, default=False, nullable=False, server_default="0") + + active = relationship("AssetActive", uselist=False, viewonly=True) consuming_dags = relationship("DagScheduleAssetReference", back_populates="dataset") producing_tasks = relationship("TaskOutletAssetReference", back_populates="dataset") @@ -232,6 +232,61 @@ def to_public(self) -> Asset: return Asset(uri=self.uri, extra=self.extra) +class AssetActive(Base): + """ + Collection of active assets. + + An asset is considered active if it is declared by the user in any DAG files. + AssetModel entries that are not active (also called orphaned in some parts + of the code base) are still kept in the database, but have their corresponding + entries in this table removed. This ensures we keep all possible history on + distinct assets (those with non-matching name-URI pairs), but still ensure + *name and URI are each unique* within active assets. + """ + + name = Column( + String(length=1500).with_variant( + String( + length=1500, + # latin1 allows for more indexed length in mysql + # and this field should only be ascii chars + collation="latin1_general_cs", + ), + "mysql", + ), + nullable=False, + ) + uri = Column( + String(length=1500).with_variant( + String( + length=1500, + # latin1 allows for more indexed length in mysql + # and this field should only be ascii chars + collation="latin1_general_cs", + ), + "mysql", + ), + nullable=False, + ) + + __tablename__ = "asset_active" + __table_args__ = ( + PrimaryKeyConstraint(name, uri, name="asset_active_pkey"), + ForeignKeyConstraint( + columns=[name, uri], + refcolumns=["dataset.name", "dataset.uri"], + name="asset_active_asset_name_uri_fkey", + ondelete="CASCADE", + ), + Index("idx_asset_active_name_unique", name, unique=True), + Index("idx_asset_active_uri_unique", uri, unique=True), + ) + + @classmethod + def for_asset(cls, asset: AssetModel) -> AssetActive: + return cls(name=asset.name, uri=asset.uri) + + class DagScheduleAssetAliasReference(Base): """References from a DAG to an asset alias of which it is a consumer.""" diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 1dfb3b2e9114d..f8d9f55e56e18 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2593,6 +2593,7 @@ def bulk_write_to_db( orm_asset_aliases = asset_op.add_asset_aliases(session=session) session.flush() # This populates id so we can create fks in later calls. + asset_op.add_asset_active_references(orm_assets.values(), session=session) asset_op.add_dag_asset_references(orm_dags, orm_assets, session=session) asset_op.add_dag_asset_alias_references(orm_dags, orm_asset_aliases, session=session) asset_op.add_task_asset_references(orm_dags, orm_assets, session=session) diff --git a/airflow/serialization/pydantic/asset.py b/airflow/serialization/pydantic/asset.py index 29806d3bdf911..4cd264902091a 100644 --- a/airflow/serialization/pydantic/asset.py +++ b/airflow/serialization/pydantic/asset.py @@ -51,7 +51,6 @@ class AssetPydantic(BaseModelPydantic): extra: Optional[dict] created_at: datetime updated_at: datetime - is_orphaned: bool consuming_dags: List[DagScheduleAssetReferencePydantic] producing_tasks: List[TaskOutletAssetReferencePydantic] diff --git a/airflow/utils/db.py b/airflow/utils/db.py index fde641fa9b424..26af566c7e2bd 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -96,7 +96,7 @@ class MappedClassProtocol(Protocol): "2.9.0": "1949afb29106", "2.9.2": "686269002441", "2.10.0": "22ed7efa9da2", - "3.0.0": "c3389cd7793f", + "3.0.0": "5a5d66100783", } diff --git a/airflow/www/views.py b/airflow/www/views.py index 8dba4c4fcc0d8..23ccae6224960 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -3450,7 +3450,7 @@ def next_run_datasets(self, dag_id): ), isouter=True, ) - .where(DagScheduleAssetReference.dag_id == dag_id, ~AssetModel.is_orphaned) + .where(DagScheduleAssetReference.dag_id == dag_id, AssetModel.active.has()) .group_by(AssetModel.id, AssetModel.uri) .order_by(AssetModel.uri) ) @@ -3583,7 +3583,7 @@ def datasets_summary(self): if has_event_filters: count_query = count_query.join(AssetEvent, AssetEvent.dataset_id == AssetModel.id) - filters = [~AssetModel.is_orphaned] + filters = [AssetModel.active.has()] if uri_pattern: filters.append(AssetModel.uri.ilike(f"%{uri_pattern}%")) if updated_after: diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 67d7cad45e23e..a751eca916568 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -8bd129828ba299ef05d70305eee66d15b6c0c79dc6ae82f654b9657464e3682a \ No newline at end of file +0ed26236c783f7524416c1377638fe18ff3520bd355160db48656585ff58524e \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 8f180c819fe32..73452a069c040 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -4,11 +4,11 @@ - - + + %3 - + log @@ -433,48 +433,48 @@ dataset_alias - -dataset_alias - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(3000)] - NOT NULL + +dataset_alias + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL dataset_alias_dataset - -dataset_alias_dataset - -alias_id - - [INTEGER] - NOT NULL - -dataset_id - - [INTEGER] - NOT NULL + +dataset_alias_dataset + +alias_id + + [INTEGER] + NOT NULL + +dataset_id + + [INTEGER] + NOT NULL dataset_alias--dataset_alias_dataset - -0..N -1 + +0..N +1 dataset_alias--dataset_alias_dataset - -0..N -1 + +0..N +1 @@ -495,16 +495,16 @@ dataset_alias--dataset_alias_dataset_event - -0..N -1 + +0..N +1 dataset_alias--dataset_alias_dataset_event - -0..N -1 + +0..N +1 @@ -535,72 +535,97 @@ dataset_alias--dag_schedule_dataset_alias_reference - -0..N -1 + +0..N +1 dataset - -dataset - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -extra - - [JSON] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -is_orphaned - - [BOOLEAN] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +dataset + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +extra + + [JSON] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL dataset--dataset_alias_dataset - -0..N -1 + +0..N +1 dataset--dataset_alias_dataset - -0..N -1 + +0..N +1 - + +asset_active + +asset_active + +name + + [VARCHAR(1500)] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL + + + +dataset--asset_active + +1 +1 + + + +dataset--asset_active + +1 +1 + + + dag_schedule_dataset_reference dag_schedule_dataset_reference @@ -626,14 +651,14 @@ NOT NULL - + dataset--dag_schedule_dataset_reference - -0..N -1 + +0..N +1 - + task_outlet_dataset_reference task_outlet_dataset_reference @@ -664,14 +689,14 @@ NOT NULL - + dataset--task_outlet_dataset_reference - -0..N -1 + +0..N +1 - + dataset_dag_run_queue dataset_dag_run_queue @@ -692,70 +717,70 @@ NOT NULL - + dataset--dataset_dag_run_queue - -0..N -1 + +0..N +1 - + dataset_event - -dataset_event - -id - - [INTEGER] - NOT NULL - -dataset_id - - [INTEGER] - NOT NULL - -extra - - [JSON] - NOT NULL - -source_dag_id - - [VARCHAR(250)] - -source_map_index - - [INTEGER] - -source_run_id - - [VARCHAR(250)] - -source_task_id - - [VARCHAR(250)] - -timestamp - - [TIMESTAMP] - NOT NULL + +dataset_event + +id + + [INTEGER] + NOT NULL + +dataset_id + + [INTEGER] + NOT NULL + +extra + + [JSON] + NOT NULL + +source_dag_id + + [VARCHAR(250)] + +source_map_index + + [INTEGER] + +source_run_id + + [VARCHAR(250)] + +source_task_id + + [VARCHAR(250)] + +timestamp + + [TIMESTAMP] + NOT NULL - + dataset_event--dataset_alias_dataset_event - -0..N -1 + +0..N +1 - + dataset_event--dataset_alias_dataset_event - -0..N -1 + +0..N +1 - + dagrun_dataset_event dagrun_dataset_event @@ -771,156 +796,156 @@ NOT NULL - + dataset_event--dagrun_dataset_event - -0..N -1 + +0..N +1 - + dag - -dag - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_display_name - - [VARCHAR(2000)] - -dataset_expression - - [JSON] - -default_view - - [VARCHAR(25)] - -description - - [TEXT] - -fileloc - - [VARCHAR(2000)] - -has_import_errors - - [BOOLEAN] - -has_task_concurrency_limits - - [BOOLEAN] - NOT NULL - -is_active - - [BOOLEAN] - -is_paused - - [BOOLEAN] - -last_expired - - [TIMESTAMP] - -last_parsed_time - - [TIMESTAMP] - -last_pickled - - [TIMESTAMP] - -max_active_runs - - [INTEGER] - -max_active_tasks - - [INTEGER] - NOT NULL - -max_consecutive_failed_dag_runs - - [INTEGER] - NOT NULL - -next_dagrun - - [TIMESTAMP] - -next_dagrun_create_after - - [TIMESTAMP] - -next_dagrun_data_interval_end - - [TIMESTAMP] - -next_dagrun_data_interval_start - - [TIMESTAMP] - -owners - - [VARCHAR(2000)] - -pickle_id - - [INTEGER] - -processor_subdir - - [VARCHAR(2000)] - -scheduler_lock - - [BOOLEAN] - -timetable_description - - [VARCHAR(1000)] - -timetable_summary - - [TEXT] + +dag + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_display_name + + [VARCHAR(2000)] + +dataset_expression + + [JSON] + +default_view + + [VARCHAR(25)] + +description + + [TEXT] + +fileloc + + [VARCHAR(2000)] + +has_import_errors + + [BOOLEAN] + +has_task_concurrency_limits + + [BOOLEAN] + NOT NULL + +is_active + + [BOOLEAN] + +is_paused + + [BOOLEAN] + +last_expired + + [TIMESTAMP] + +last_parsed_time + + [TIMESTAMP] + +last_pickled + + [TIMESTAMP] + +max_active_runs + + [INTEGER] + +max_active_tasks + + [INTEGER] + NOT NULL + +max_consecutive_failed_dag_runs + + [INTEGER] + NOT NULL + +next_dagrun + + [TIMESTAMP] + +next_dagrun_create_after + + [TIMESTAMP] + +next_dagrun_data_interval_end + + [TIMESTAMP] + +next_dagrun_data_interval_start + + [TIMESTAMP] + +owners + + [VARCHAR(2000)] + +pickle_id + + [INTEGER] + +processor_subdir + + [VARCHAR(2000)] + +scheduler_lock + + [BOOLEAN] + +timetable_description + + [VARCHAR(1000)] + +timetable_summary + + [TEXT] - + dag--dag_schedule_dataset_alias_reference - -0..N -1 + +0..N +1 - + dag--dag_schedule_dataset_reference - -0..N -1 + +0..N +1 - + dag--task_outlet_dataset_reference - -0..N -1 + +0..N +1 - + dag--dataset_dag_run_queue - -0..N -1 + +0..N +1 - + dag_tag dag_tag @@ -936,14 +961,14 @@ NOT NULL - + dag--dag_tag - -0..N -1 + +0..N +1 - + dag_owner_attributes dag_owner_attributes @@ -964,14 +989,14 @@ NOT NULL - + dag--dag_owner_attributes - -0..N -1 + +0..N +1 - + dag_warning dag_warning @@ -997,14 +1022,14 @@ NOT NULL - + dag--dag_warning - -0..N -1 + +0..N +1 - + log_template log_template @@ -1030,7 +1055,7 @@ NOT NULL - + dag_run dag_run @@ -1126,21 +1151,21 @@ [TIMESTAMP] - + log_template--dag_run 0..N {0,1} - + dag_run--dagrun_dataset_event 0..N 1 - + task_instance task_instance @@ -1280,21 +1305,21 @@ [TIMESTAMP] - + dag_run--task_instance 0..N 1 - + dag_run--task_instance 0..N 1 - + dag_run_note dag_run_note @@ -1323,14 +1348,14 @@ [VARCHAR(128)] - + dag_run--dag_run_note 1 1 - + backfill_dag_run backfill_dag_run @@ -1355,14 +1380,14 @@ NOT NULL - + dag_run--backfill_dag_run 0..N {0,1} - + task_reschedule task_reschedule @@ -1418,49 +1443,49 @@ NOT NULL - + dag_run--task_reschedule 0..N 1 - + dag_run--task_reschedule 0..N 1 - + task_instance--task_reschedule 0..N 1 - + task_instance--task_reschedule -0..N +0..N 1 - + task_instance--task_reschedule -0..N +0..N 1 - + task_instance--task_reschedule 0..N 1 - + rendered_task_instance_fields rendered_task_instance_fields @@ -1495,35 +1520,35 @@ NOT NULL - + task_instance--rendered_task_instance_fields 0..N 1 - + task_instance--rendered_task_instance_fields 0..N 1 - + task_instance--rendered_task_instance_fields 0..N 1 - + task_instance--rendered_task_instance_fields 0..N 1 - + task_fail task_fail @@ -1566,35 +1591,35 @@ NOT NULL - + task_instance--task_fail 0..N 1 - + task_instance--task_fail 0..N 1 - + task_instance--task_fail 0..N 1 - + task_instance--task_fail 0..N 1 - + task_map task_map @@ -1629,35 +1654,35 @@ NOT NULL - + task_instance--task_map 0..N 1 - + task_instance--task_map 0..N 1 - + task_instance--task_map 0..N 1 - + task_instance--task_map 0..N 1 - + xcom xcom @@ -1702,35 +1727,35 @@ [BYTEA] - + task_instance--xcom 0..N 1 - + task_instance--xcom 0..N 1 - + task_instance--xcom 0..N 1 - + task_instance--xcom 0..N 1 - + task_instance_note task_instance_note @@ -1774,35 +1799,35 @@ [VARCHAR(128)] - + task_instance--task_instance_note 0..N 1 - + task_instance--task_instance_note 0..N 1 - + task_instance--task_instance_note 0..N 1 - + task_instance--task_instance_note 0..N 1 - + task_instance_history task_instance_history @@ -1948,35 +1973,35 @@ [TIMESTAMP] - + task_instance--task_instance_history 0..N 1 - + task_instance--task_instance_history 0..N 1 - + task_instance--task_instance_history 0..N 1 - + task_instance--task_instance_history 0..N 1 - + backfill backfill @@ -2029,21 +2054,21 @@ NOT NULL - + backfill--dag_run 0..N {0,1} - + backfill--backfill_dag_run 0..N 1 - + trigger trigger @@ -2073,327 +2098,327 @@ [INTEGER] - + trigger--task_instance 0..N {0,1} - + session - -session - -id - - [INTEGER] - NOT NULL - -data - - [BYTEA] - -expiry - - [TIMESTAMP] - -session_id - - [VARCHAR(255)] + +session + +id + + [INTEGER] + NOT NULL + +data + + [BYTEA] + +expiry + + [TIMESTAMP] + +session_id + + [VARCHAR(255)] - + alembic_version - -alembic_version - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version + +version_num + + [VARCHAR(32)] + NOT NULL - + ab_user - -ab_user - -id - - [INTEGER] - NOT NULL - -active - - [BOOLEAN] - -changed_by_fk - - [INTEGER] - -changed_on - - [TIMESTAMP] - -created_by_fk - - [INTEGER] - -created_on - - [TIMESTAMP] - -email - - [VARCHAR(512)] - NOT NULL - -fail_login_count - - [INTEGER] - -first_name - - [VARCHAR(256)] - NOT NULL - -last_login - - [TIMESTAMP] - -last_name - - [VARCHAR(256)] - NOT NULL - -login_count - - [INTEGER] - -password - - [VARCHAR(256)] - -username - - [VARCHAR(512)] - NOT NULL + +ab_user + +id + + [INTEGER] + NOT NULL + +active + + [BOOLEAN] + +changed_by_fk + + [INTEGER] + +changed_on + + [TIMESTAMP] + +created_by_fk + + [INTEGER] + +created_on + + [TIMESTAMP] + +email + + [VARCHAR(512)] + NOT NULL + +fail_login_count + + [INTEGER] + +first_name + + [VARCHAR(256)] + NOT NULL + +last_login + + [TIMESTAMP] + +last_name + + [VARCHAR(256)] + NOT NULL + +login_count + + [INTEGER] + +password + + [VARCHAR(256)] + +username + + [VARCHAR(512)] + NOT NULL - + ab_user--ab_user - -0..N -{0,1} + +0..N +{0,1} - + ab_user--ab_user - -0..N -{0,1} + +0..N +{0,1} - + ab_user_role - -ab_user_role - -id - - [INTEGER] - NOT NULL - -role_id - - [INTEGER] - -user_id - - [INTEGER] + +ab_user_role + +id + + [INTEGER] + NOT NULL + +role_id + + [INTEGER] + +user_id + + [INTEGER] - + ab_user--ab_user_role - -0..N -{0,1} + +0..N +{0,1} - + ab_register_user - -ab_register_user - -id - - [INTEGER] - NOT NULL - -email - - [VARCHAR(512)] - NOT NULL - -first_name - - [VARCHAR(256)] - NOT NULL - -last_name - - [VARCHAR(256)] - NOT NULL - -password - - [VARCHAR(256)] - -registration_date - - [TIMESTAMP] - -registration_hash - - [VARCHAR(256)] - -username - - [VARCHAR(512)] - NOT NULL + +ab_register_user + +id + + [INTEGER] + NOT NULL + +email + + [VARCHAR(512)] + NOT NULL + +first_name + + [VARCHAR(256)] + NOT NULL + +last_name + + [VARCHAR(256)] + NOT NULL + +password + + [VARCHAR(256)] + +registration_date + + [TIMESTAMP] + +registration_hash + + [VARCHAR(256)] + +username + + [VARCHAR(512)] + NOT NULL - + ab_permission - -ab_permission - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(100)] - NOT NULL + +ab_permission + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(100)] + NOT NULL - + ab_permission_view - -ab_permission_view - -id - - [INTEGER] - NOT NULL - -permission_id - - [INTEGER] - -view_menu_id - - [INTEGER] + +ab_permission_view + +id + + [INTEGER] + NOT NULL + +permission_id + + [INTEGER] + +view_menu_id + + [INTEGER] - + ab_permission--ab_permission_view - -0..N -{0,1} + +0..N +{0,1} - + ab_permission_view_role - -ab_permission_view_role - -id - - [INTEGER] - NOT NULL - -permission_view_id - - [INTEGER] - -role_id - - [INTEGER] + +ab_permission_view_role + +id + + [INTEGER] + NOT NULL + +permission_view_id + + [INTEGER] + +role_id + + [INTEGER] - + ab_permission_view--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} - + ab_view_menu - -ab_view_menu - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(250)] - NOT NULL + +ab_view_menu + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(250)] + NOT NULL - + ab_view_menu--ab_permission_view - -0..N -{0,1} + +0..N +{0,1} - + ab_role - -ab_role - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(64)] - NOT NULL + +ab_role + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(64)] + NOT NULL - + ab_role--ab_user_role - -0..N -{0,1} + +0..N +{0,1} - + ab_role--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} - + alembic_version_fab - -alembic_version_fab - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version_fab + +version_num + + [VARCHAR(32)] + NOT NULL diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 6b348013bd3ce..f917717d49efe 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``c3389cd7793f`` (head) | ``0d9e73a75ee4`` | ``3.0.0`` | Add backfill to dag run model. | +| ``5a5d66100783`` (head) | ``c3389cd7793f`` | ``3.0.0`` | Add AssetActive to track orphaning instead of a flag. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``c3389cd7793f`` | ``0d9e73a75ee4`` | ``3.0.0`` | Add backfill to dag run model. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``0d9e73a75ee4`` | ``44eabb1904b4`` | ``3.0.0`` | Add name and group fields to DatasetModel. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index f47f4497d5aab..5244e96490984 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -6086,9 +6086,9 @@ def test_asset_orphaning(self, dag_maker, session): with dag_maker(dag_id="assets-1", schedule=[asset1, asset2], session=session): BashOperator(task_id="task", bash_command="echo 1", outlets=[asset3, asset4]) - non_orphaned_asset_count = session.query(AssetModel).filter(~AssetModel.is_orphaned).count() + non_orphaned_asset_count = session.query(AssetModel).filter(AssetModel.active.has()).count() assert non_orphaned_asset_count == 4 - orphaned_asset_count = session.query(AssetModel).filter(AssetModel.is_orphaned).count() + orphaned_asset_count = session.query(AssetModel).filter(~AssetModel.active.has()).count() assert orphaned_asset_count == 0 # now remove 2 asset references @@ -6105,14 +6105,13 @@ def test_asset_orphaning(self, dag_maker, session): non_orphaned_assets = [ asset.uri for asset in session.query(AssetModel.uri) - .filter(~AssetModel.is_orphaned) + .filter(AssetModel.active.has()) .order_by(AssetModel.uri) ] assert non_orphaned_assets == ["ds1", "ds3"] - orphaned_assets = [ - asset.uri - for asset in session.query(AssetModel.uri).filter(AssetModel.is_orphaned).order_by(AssetModel.uri) - ] + orphaned_assets = session.scalars( + select(AssetModel.uri).where(~AssetModel.active.has()).order_by(AssetModel.uri) + ).all() assert orphaned_assets == ["ds2", "ds4"] def test_asset_orphaning_ignore_orphaned_assets(self, dag_maker, session): @@ -6121,9 +6120,9 @@ def test_asset_orphaning_ignore_orphaned_assets(self, dag_maker, session): with dag_maker(dag_id="assets-1", schedule=[asset1], session=session): BashOperator(task_id="task", bash_command="echo 1") - non_orphaned_asset_count = session.query(AssetModel).filter(~AssetModel.is_orphaned).count() + non_orphaned_asset_count = session.query(AssetModel).filter(AssetModel.active.has()).count() assert non_orphaned_asset_count == 1 - orphaned_asset_count = session.query(AssetModel).filter(AssetModel.is_orphaned).count() + orphaned_asset_count = session.query(AssetModel).filter(~AssetModel.active.has()).count() assert orphaned_asset_count == 0 # now remove asset1 reference @@ -6138,7 +6137,7 @@ def test_asset_orphaning_ignore_orphaned_assets(self, dag_maker, session): orphaned_assets_before_rerun = ( session.query(AssetModel.updated_at, AssetModel.uri) - .filter(AssetModel.is_orphaned) + .filter(~AssetModel.active.has()) .order_by(AssetModel.uri) ) assert [asset.uri for asset in orphaned_assets_before_rerun] == ["ds1"] @@ -6151,7 +6150,7 @@ def test_asset_orphaning_ignore_orphaned_assets(self, dag_maker, session): orphaned_assets_after_rerun = ( session.query(AssetModel.updated_at, AssetModel.uri) - .filter(AssetModel.is_orphaned) + .filter(~AssetModel.active.has()) .order_by(AssetModel.uri) ) assert [asset.uri for asset in orphaned_assets_after_rerun] == ["ds1"] diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 0fe22ccf77fe3..b439487b0161e 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -1080,10 +1080,10 @@ def test_bulk_write_to_db_unorphan_assets(self): # orphans asset1 = Asset(uri="ds1") asset2 = Asset(uri="ds2") - session.add(AssetModel(uri=asset2.uri, is_orphaned=True)) + session.add(AssetModel(uri=asset2.uri)) asset3 = Asset(uri="ds3") asset4 = Asset(uri="ds4") - session.add(AssetModel(uri=asset4.uri, is_orphaned=True)) + session.add(AssetModel(uri=asset4.uri)) session.flush() dag1 = DAG(dag_id="assets-1", start_date=DEFAULT_DATE, schedule=[asset1]) @@ -1095,14 +1095,14 @@ def test_bulk_write_to_db_unorphan_assets(self): non_orphaned_assets = [ asset.uri for asset in session.query(AssetModel.uri) - .filter(~AssetModel.is_orphaned) + .filter(AssetModel.active.has()) .order_by(AssetModel.uri) ] assert non_orphaned_assets == ["ds1", "ds3"] orphaned_assets = [ asset.uri for asset in session.query(AssetModel.uri) - .filter(AssetModel.is_orphaned) + .filter(~AssetModel.active.has()) .order_by(AssetModel.uri) ] assert orphaned_assets == ["ds2", "ds4"] @@ -1114,9 +1114,9 @@ def test_bulk_write_to_db_unorphan_assets(self): DAG.bulk_write_to_db([dag1], session=session) # and count the orphans and non-orphans - non_orphaned_asset_count = session.query(AssetModel).filter(~AssetModel.is_orphaned).count() + non_orphaned_asset_count = session.query(AssetModel).filter(AssetModel.active.has()).count() assert non_orphaned_asset_count == 4 - orphaned_asset_count = session.query(AssetModel).filter(AssetModel.is_orphaned).count() + orphaned_asset_count = session.query(AssetModel).filter(~AssetModel.active.has()).count() assert orphaned_asset_count == 0 def test_bulk_write_to_db_asset_aliases(self): diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py index c7533827d805d..ba7cc9476c933 100644 --- a/tests/utils/test_db_cleanup.py +++ b/tests/utils/test_db_cleanup.py @@ -329,6 +329,7 @@ def test_no_models_missing(self): "backfill_dag_run", # todo: AIP-78 "ab_user", "variable", # leave alone + "asset_active", # not good way to know if "stale" "dataset", # not good way to know if "stale" "dataset_alias", # not good way to know if "stale" "task_map", # keys to TI, so no need diff --git a/tests/www/views/test_views_dataset.py b/tests/www/views/test_views_dataset.py index 9d83715f3e210..bc40b9010c29c 100644 --- a/tests/www/views/test_views_dataset.py +++ b/tests/www/views/test_views_dataset.py @@ -22,7 +22,7 @@ from dateutil.tz import UTC from airflow.assets import Asset -from airflow.models.asset import AssetEvent, AssetModel +from airflow.models.asset import AssetActive, AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.db import clear_db_assets @@ -37,17 +37,22 @@ def _cleanup(self): yield clear_db_assets() + @pytest.fixture + def create_assets(self, session): + def create(indexes): + assets = [AssetModel(id=i, uri=f"s3://bucket/key/{i}") for i in indexes] + session.add_all(assets) + session.flush() + session.add_all(AssetActive.for_asset(a) for a in assets) + session.flush() + return assets + + return create + class TestGetDatasets(TestDatasetEndpoint): - def test_should_respond_200(self, admin_client, session): - assets = [ - AssetModel( - id=i, - uri=f"s3://bucket/key/{i}", - ) - for i in [1, 2] - ] - session.add_all(assets) + def test_should_respond_200(self, admin_client, create_assets, session): + create_assets([1, 2]) session.commit() assert session.query(AssetModel).count() == 2 @@ -74,14 +79,8 @@ def test_should_respond_200(self, admin_client, session): "total_entries": 2, } - def test_order_by_raises_400_for_invalid_attr(self, admin_client, session): - assets = [ - AssetModel( - uri=f"s3://bucket/key/{i}", - ) - for i in [1, 2] - ] - session.add_all(assets) + def test_order_by_raises_400_for_invalid_attr(self, admin_client, create_assets, session): + create_assets([1, 2]) session.commit() assert session.query(AssetModel).count() == 2 @@ -91,9 +90,8 @@ def test_order_by_raises_400_for_invalid_attr(self, admin_client, session): msg = "Ordering with 'fake' is disallowed or the attribute does not exist on the model" assert response.json["detail"] == msg - def test_order_by_raises_400_for_invalid_datetimes(self, admin_client, session): - assets = [AssetModel(uri=f"s3://bucket/key/{i}") for i in [1, 2]] - session.add_all(assets) + def test_order_by_raises_400_for_invalid_datetimes(self, admin_client, create_assets, session): + create_assets([1, 2]) session.commit() assert session.query(AssetModel).count() == 2 @@ -107,24 +105,17 @@ def test_order_by_raises_400_for_invalid_datetimes(self, admin_client, session): assert response.status_code == 400 assert "Invalid datetime:" in response.text - def test_filter_by_datetimes(self, admin_client, session): + def test_filter_by_datetimes(self, admin_client, create_assets, session): today = pendulum.today("UTC") - assets = [ - AssetModel( - id=i, - uri=f"s3://bucket/key/{i}", - ) - for i in range(1, 4) - ] - session.add_all(assets) + assets = create_assets(range(1, 4)) # Update assets, one per day, starting with assets[0], ending with assets[2] asset_events = [ AssetEvent( - dataset_id=assets[i].id, + dataset_id=a.id, timestamp=today.add(days=-len(assets) + i + 1), ) - for i in range(len(assets)) + for i, a in enumerate(assets) ] session.add_all(asset_events) session.commit() @@ -153,15 +144,8 @@ def test_filter_by_datetimes(self, admin_client, session): ("-last_dataset_update", [2, 3, 1, 4]), ], ) - def test_order_by(self, admin_client, session, order_by, ordered_asset_ids): - assets = [ - AssetModel( - id=i, - uri=f"s3://bucket/key/{i}", - ) - for i in range(1, len(ordered_asset_ids) + 1) - ] - session.add_all(assets) + def test_order_by(self, admin_client, session, create_assets, order_by, ordered_asset_ids): + assets = create_assets(range(1, len(ordered_asset_ids) + 1)) asset_events = [ AssetEvent( dataset_id=assets[2].id, @@ -186,15 +170,8 @@ def test_order_by(self, admin_client, session, order_by, ordered_asset_ids): assert ordered_asset_ids == [json_dict["id"] for json_dict in response.json["datasets"]] assert response.json["total_entries"] == len(ordered_asset_ids) - def test_search_uri_pattern(self, admin_client, session): - assets = [ - AssetModel( - id=i, - uri=f"s3://bucket/key_{i}", - ) - for i in [1, 2] - ] - session.add_all(assets) + def test_search_uri_pattern(self, admin_client, create_assets, session): + create_assets([1, 2]) session.commit() assert session.query(AssetModel).count() == 2 @@ -207,7 +184,7 @@ def test_search_uri_pattern(self, admin_client, session): "datasets": [ { "id": 2, - "uri": "s3://bucket/key_2", + "uri": "s3://bucket/key/2", "last_dataset_update": None, "total_updates": 0, }, @@ -224,13 +201,13 @@ def test_search_uri_pattern(self, admin_client, session): "datasets": [ { "id": 1, - "uri": "s3://bucket/key_1", + "uri": "s3://bucket/key/1", "last_dataset_update": None, "total_updates": 0, }, { "id": 2, - "uri": "s3://bucket/key_2", + "uri": "s3://bucket/key/2", "last_dataset_update": None, "total_updates": 0, }, @@ -389,15 +366,8 @@ class TestGetDatasetsEndpointPagination(TestDatasetEndpoint): ("/object/datasets_summary?offset=3&limit=3", [f"s3://bucket/key/{i}" for i in [4, 5, 6]]), ], ) - def test_limit_and_offset(self, admin_client, session, url, expected_dataset_uris): - assets = [ - AssetModel( - uri=f"s3://bucket/key/{i}", - extra={"foo": "bar"}, - ) - for i in range(1, 10) - ] - session.add_all(assets) + def test_limit_and_offset(self, admin_client, create_assets, session, url, expected_dataset_uris): + create_assets(range(1, 10)) session.commit() response = admin_client.get(url) @@ -406,15 +376,8 @@ def test_limit_and_offset(self, admin_client, session, url, expected_dataset_uri dataset_uris = [dataset["uri"] for dataset in response.json["datasets"]] assert dataset_uris == expected_dataset_uris - def test_should_respect_page_size_limit_default(self, admin_client, session): - assets = [ - AssetModel( - uri=f"s3://bucket/key/{i}", - extra={"foo": "bar"}, - ) - for i in range(1, 60) - ] - session.add_all(assets) + def test_should_respect_page_size_limit_default(self, admin_client, create_assets, session): + create_assets(range(1, 60)) session.commit() response = admin_client.get("/object/datasets_summary") @@ -422,15 +385,8 @@ def test_should_respect_page_size_limit_default(self, admin_client, session): assert response.status_code == 200 assert len(response.json["datasets"]) == 25 - def test_should_return_max_if_req_above(self, admin_client, session): - assets = [ - AssetModel( - uri=f"s3://bucket/key/{i}", - extra={"foo": "bar"}, - ) - for i in range(1, 60) - ] - session.add_all(assets) + def test_should_return_max_if_req_above(self, admin_client, create_assets, session): + create_assets(range(1, 60)) session.commit() response = admin_client.get("/object/datasets_summary?limit=180") From dabc4f2aa96936649a471f521c0508ffb0ed8fe5 Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:15:51 +0530 Subject: [PATCH 114/199] Migrate the public endpoint Delete DAG to FastAPI (#42914) * Migrate the public endpoint Delete DAG to FastAPI * Refactor tests --- .../api_connexion/endpoints/dag_endpoint.py | 2 + airflow/api_fastapi/openapi/v1-generated.yaml | 49 ++++ airflow/api_fastapi/views/public/dags.py | 19 +- airflow/ui/openapi-gen/queries/common.ts | 3 + airflow/ui/openapi-gen/queries/queries.ts | 37 +++ .../ui/openapi-gen/requests/services.gen.ts | 29 +++ airflow/ui/openapi-gen/requests/types.gen.ts | 35 +++ airflow/utils/api_migration.py | 2 +- tests/api_fastapi/views/public/test_dags.py | 242 ++++++++++++------ 9 files changed, 336 insertions(+), 82 deletions(-) diff --git a/airflow/api_connexion/endpoints/dag_endpoint.py b/airflow/api_connexion/endpoints/dag_endpoint.py index 3d0d3dd8bfabe..0352297bfffd4 100644 --- a/airflow/api_connexion/endpoints/dag_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_endpoint.py @@ -52,6 +52,7 @@ from airflow.api_connexion.types import APIResponse, UpdateMask +@mark_fastapi_migration_done @security.requires_access_dag("GET") @provide_session def get_dag( @@ -215,6 +216,7 @@ def patch_dags(limit, session, offset=0, only_active=True, tags=None, dag_id_pat return dags_collection_schema.dump(DAGCollection(dags=dags, total_entries=total_entries)) +@mark_fastapi_migration_done @security.requires_access_dag("DELETE") @action_logging @provide_session diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index 235410a6d347a..56f48c73e987f 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -408,6 +408,55 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - DAG + summary: Delete Dag + description: Delete the specific DAG. + operationId: delete_dag + parameters: + - name: dag_id + in: path + required: true + schema: + type: string + title: Dag Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Bad Request + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unprocessable Entity /public/dags/{dag_id}/details: get: tags: diff --git a/airflow/api_fastapi/views/public/dags.py b/airflow/api_fastapi/views/public/dags.py index ca0f44162eb2d..eb8233a7f700d 100644 --- a/airflow/api_fastapi/views/public/dags.py +++ b/airflow/api_fastapi/views/public/dags.py @@ -17,11 +17,12 @@ from __future__ import annotations -from fastapi import Depends, HTTPException, Query, Request +from fastapi import Depends, HTTPException, Query, Request, Response from sqlalchemy import update from sqlalchemy.orm import Session from typing_extensions import Annotated +from airflow.api.common import delete_dag as delete_dag_module from airflow.api_fastapi.db.common import ( get_session, paginated_select, @@ -48,6 +49,7 @@ DAGResponse, ) from airflow.api_fastapi.views.router import AirflowRouter +from airflow.exceptions import AirflowException, DagNotFound from airflow.models import DAG, DagModel dags_router = AirflowRouter(tags=["DAG"], prefix="/dags") @@ -204,3 +206,18 @@ async def patch_dags( dags=[DAGResponse.model_validate(dag, from_attributes=True) for dag in dags], total_entries=total_entries, ) + + +@dags_router.delete("/{dag_id}", responses=create_openapi_http_exception_doc([400, 401, 403, 404, 422])) +async def delete_dag( + dag_id: str, + session: Annotated[Session, Depends(get_session)], +) -> Response: + """Delete the specific DAG.""" + try: + delete_dag_module.delete_dag(dag_id, session=session) + except DagNotFound: + raise HTTPException(404, f"Dag with id: {dag_id} was not found") + except AirflowException: + raise HTTPException(409, f"Task instances of dag with id: '{dag_id}' are still running") + return Response(status_code=204) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 426e28447fd9c..2f1c6a78d92f0 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -197,6 +197,9 @@ export type DagServicePatchDagMutationResult = Awaited< export type VariableServicePatchVariableMutationResult = Awaited< ReturnType >; +export type DagServiceDeleteDagMutationResult = Awaited< + ReturnType +>; export type ConnectionServiceDeleteConnectionMutationResult = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 557a7ba8ffa17..a16bdf165b182 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -514,6 +514,43 @@ export const useVariableServicePatchVariable = < }) as unknown as Promise, ...options, }); +/** + * Delete Dag + * Delete the specific DAG. + * @param data The data for the request. + * @param data.dagId + * @returns unknown Successful Response + * @throws ApiError + */ +export const useDagServiceDeleteDag = < + TData = Common.DagServiceDeleteDagMutationResult, + TError = unknown, + TContext = unknown, +>( + options?: Omit< + UseMutationOptions< + TData, + TError, + { + dagId: string; + }, + TContext + >, + "mutationFn" + >, +) => + useMutation< + TData, + TError, + { + dagId: string; + }, + TContext + >({ + mutationFn: ({ dagId }) => + DagService.deleteDag({ dagId }) as unknown as Promise, + ...options, + }); /** * Delete Connection * Delete a connection entry. diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 78b113c7f2f80..8d7f0cee2b295 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -15,6 +15,8 @@ import type { GetDagResponse, PatchDagData, PatchDagResponse, + DeleteDagData, + DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, DeleteConnectionData, @@ -234,6 +236,33 @@ export class DagService { }); } + /** + * Delete Dag + * Delete the specific DAG. + * @param data The data for the request. + * @param data.dagId + * @returns unknown Successful Response + * @throws ApiError + */ + public static deleteDag( + data: DeleteDagData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "DELETE", + url: "/public/dags/{dag_id}", + path: { + dag_id: data.dagId, + }, + errors: { + 400: "Bad Request", + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Unprocessable Entity", + }, + }); + } + /** * Get Dag Details * Get details of DAG. diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 856517d560fe4..7f603a1adb4d3 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -327,6 +327,12 @@ export type PatchDagData = { export type PatchDagResponse = DAGResponse; +export type DeleteDagData = { + dagId: string; +}; + +export type DeleteDagResponse = unknown; + export type GetDagDetailsData = { dagId: string; }; @@ -525,6 +531,35 @@ export type $OpenApiTs = { 422: HTTPValidationError; }; }; + delete: { + req: DeleteDagData; + res: { + /** + * Successful Response + */ + 200: unknown; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Unprocessable Entity + */ + 422: HTTPExceptionResponse; + }; + }; }; "/public/dags/{dag_id}/details": { get: { diff --git a/airflow/utils/api_migration.py b/airflow/utils/api_migration.py index d6b61a933deed..3e6ba3881cbed 100644 --- a/airflow/utils/api_migration.py +++ b/airflow/utils/api_migration.py @@ -31,7 +31,7 @@ RT = TypeVar("RT") -def mark_fastapi_migration_done(function: Callable[PS, RT]) -> Callable[PS, RT]: +def mark_fastapi_migration_done(function: Callable[..., RT]) -> Callable[..., RT]: """ Mark an endpoint as migrated over to the new FastAPI API. diff --git a/tests/api_fastapi/views/public/test_dags.py b/tests/api_fastapi/views/public/test_dags.py index cd1809cb70d3d..ab0c54f51764f 100644 --- a/tests/api_fastapi/views/public/test_dags.py +++ b/tests/api_fastapi/views/public/test_dags.py @@ -25,7 +25,7 @@ from airflow.models.dagrun import DagRun from airflow.operators.empty import EmptyOperator from airflow.utils.session import provide_session -from airflow.utils.state import DagRunState +from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags @@ -36,84 +36,100 @@ DAG2_ID = "test_dag2" DAG2_START_DATE = datetime(2021, 6, 15, tzinfo=timezone.utc) DAG3_ID = "test_dag3" +DAG4_ID = "test_dag4" +DAG4_DISPLAY_NAME = "display4" +DAG5_ID = "test_dag5" +DAG5_DISPLAY_NAME = "display5" TASK_ID = "op1" UTC_JSON_REPR = "UTC" if pendulum.__version__.startswith("3") else "Timezone('UTC')" +API_PREFIX = "/public/dags" + + +class TestDagEndpoint: + """Common class for /public/dags related unit tests.""" + + @staticmethod + def _clear_db(): + clear_db_runs() + clear_db_dags() + clear_db_serialized_dags() + + def _create_deactivated_paused_dag(self, session=None): + dag_model = DagModel( + dag_id=DAG3_ID, + fileloc="/tmp/dag_del_1.py", + timetable_summary="2 2 * * *", + is_active=False, + is_paused=True, + owners="test_owner,another_test_owner", + next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + ) + + dagrun_failed = DagRun( + dag_id=DAG3_ID, + run_id="run1", + execution_date=datetime(2018, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + start_date=datetime(2018, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + run_type=DagRunType.SCHEDULED, + state=DagRunState.FAILED, + ) + + dagrun_success = DagRun( + dag_id=DAG3_ID, + run_id="run2", + execution_date=datetime(2019, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + start_date=datetime(2019, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + run_type=DagRunType.MANUAL, + state=DagRunState.SUCCESS, + ) + + session.add(dag_model) + session.add(dagrun_failed) + session.add(dagrun_success) + + @pytest.fixture(autouse=True) + @provide_session + def setup(self, dag_maker, session=None) -> None: + self._clear_db() + + with dag_maker( + DAG1_ID, + dag_display_name=DAG1_DISPLAY_NAME, + schedule=None, + start_date=datetime(2018, 6, 15, 0, 0, tzinfo=timezone.utc), + doc_md="details", + params={"foo": 1}, + tags=["example"], + ): + EmptyOperator(task_id=TASK_ID) + + dag_maker.create_dagrun(state=DagRunState.FAILED) + + with dag_maker( + DAG2_ID, + schedule=None, + start_date=DAG2_START_DATE, + doc_md="details", + params={"foo": 1}, + max_active_tasks=16, + max_active_runs=16, + ): + EmptyOperator(task_id=TASK_ID) + + self._create_deactivated_paused_dag(session) + + dag_maker.dagbag.sync_to_db() + dag_maker.dag_model.has_task_concurrency_limits = True + session.merge(dag_maker.dag_model) + session.commit() + + def teardown_method(self) -> None: + self._clear_db() + + +class TestGetDags(TestDagEndpoint): + """Unit tests for Get DAGs.""" - -@provide_session -def _create_deactivated_paused_dag(session=None): - dag_model = DagModel( - dag_id=DAG3_ID, - fileloc="/tmp/dag_del_1.py", - timetable_summary="2 2 * * *", - is_active=False, - is_paused=True, - owners="test_owner,another_test_owner", - next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc), - ) - - dagrun_failed = DagRun( - dag_id=DAG3_ID, - run_id="run1", - execution_date=datetime(2018, 1, 1, 12, 0, 0, tzinfo=timezone.utc), - start_date=datetime(2018, 1, 1, 12, 0, 0, tzinfo=timezone.utc), - run_type=DagRunType.SCHEDULED, - state=DagRunState.FAILED, - ) - - dagrun_success = DagRun( - dag_id=DAG3_ID, - run_id="run2", - execution_date=datetime(2019, 1, 1, 12, 0, 0, tzinfo=timezone.utc), - start_date=datetime(2019, 1, 1, 12, 0, 0, tzinfo=timezone.utc), - run_type=DagRunType.MANUAL, - state=DagRunState.SUCCESS, - ) - - session.add(dag_model) - session.add(dagrun_failed) - session.add(dagrun_success) - - -@pytest.fixture(autouse=True) -@provide_session -def setup(dag_maker, session=None) -> None: - clear_db_runs() - clear_db_dags() - clear_db_serialized_dags() - - with dag_maker( - DAG1_ID, - dag_display_name=DAG1_DISPLAY_NAME, - schedule=None, - start_date=datetime(2018, 6, 15, 0, 0, tzinfo=timezone.utc), - doc_md="details", - params={"foo": 1}, - tags=["example"], - ): - EmptyOperator(task_id=TASK_ID) - - dag_maker.create_dagrun(state=DagRunState.FAILED) - - with dag_maker( - DAG2_ID, - schedule=None, - start_date=DAG2_START_DATE, - doc_md="details", - params={"foo": 1}, - max_active_tasks=16, - max_active_runs=16, - ): - EmptyOperator(task_id=TASK_ID) - - dag_maker.dagbag.sync_to_db() - dag_maker.dag_model.has_task_concurrency_limits = True - session.merge(dag_maker.dag_model) - session.commit() - _create_deactivated_paused_dag() - - -class TestGetDags: @pytest.mark.parametrize( "query_params, expected_total_entries, expected_ids", [ @@ -161,7 +177,9 @@ def test_get_dags(self, test_client, query_params, expected_total_entries, expec assert [dag["dag_id"] for dag in body["dags"]] == expected_ids -class TestPatchDag: +class TestPatchDag(TestDagEndpoint): + """Unit tests for Patch DAG.""" + @pytest.mark.parametrize( "query_params, dag_id, body, expected_status_code, expected_is_paused", [ @@ -184,7 +202,9 @@ def test_patch_dag( assert body["is_paused"] == expected_is_paused -class TestPatchDags: +class TestPatchDags(TestDagEndpoint): + """Unit tests for Patch DAGs.""" + @pytest.mark.parametrize( "query_params, body, expected_status_code, expected_ids, expected_paused_ids", [ @@ -239,7 +259,9 @@ def test_patch_dags( assert paused_dag_ids == expected_paused_ids -class TestDagDetails: +class TestDagDetails(TestDagEndpoint): + """Unit tests for DAG Details.""" + @pytest.mark.parametrize( "query_params, dag_id, expected_status_code, dag_display_name, start_date", [ @@ -312,7 +334,9 @@ def test_dag_details( assert res_json == expected -class TestGetDag: +class TestGetDag(TestDagEndpoint): + """Unit tests for Get DAG.""" + @pytest.mark.parametrize( "query_params, dag_id, expected_status_code, dag_display_name", [ @@ -359,3 +383,61 @@ def test_get_dag(self, test_client, query_params, dag_id, expected_status_code, "pickle_id": None, } assert res_json == expected + + +class TestDeleteDAG(TestDagEndpoint): + """Unit tests for Delete DAG.""" + + def _create_dag_for_deletion( + self, + dag_maker, + dag_id=None, + dag_display_name=None, + has_running_dagruns=False, + ): + with dag_maker( + dag_id, + dag_display_name=dag_display_name, + start_date=datetime(2024, 10, 10, tzinfo=timezone.utc), + ): + EmptyOperator(task_id="dummy") + + if has_running_dagruns: + dr = dag_maker.create_dagrun() + ti = dr.get_task_instances()[0] + ti.set_state(TaskInstanceState.RUNNING) + + dag_maker.dagbag.sync_to_db() + + @pytest.mark.parametrize( + "dag_id, dag_display_name, status_code_delete, status_code_details, has_running_dagruns, is_create_dag", + [ + ("test_nonexistent_dag_id", "nonexistent_display_name", 404, 404, False, False), + (DAG4_ID, DAG4_DISPLAY_NAME, 204, 404, False, True), + (DAG5_ID, DAG5_DISPLAY_NAME, 409, 200, True, True), + ], + ) + def test_delete_dag( + self, + dag_maker, + test_client, + dag_id, + dag_display_name, + status_code_delete, + status_code_details, + has_running_dagruns, + is_create_dag, + ): + if is_create_dag: + self._create_dag_for_deletion( + dag_maker=dag_maker, + dag_id=dag_id, + dag_display_name=dag_display_name, + has_running_dagruns=has_running_dagruns, + ) + + delete_response = test_client.delete(f"{API_PREFIX}/{dag_id}") + assert delete_response.status_code == status_code_delete + + details_response = test_client.get(f"{API_PREFIX}/{dag_id}/details") + assert details_response.status_code == status_code_details From 613c7d93816df5bdc9dfcd64f099cf51524804a9 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Tue, 15 Oct 2024 15:10:52 +0530 Subject: [PATCH 115/199] Clarifying PLUGINS_FOLDER permissions by DAG authors (#43022) --- docs/apache-airflow/security/security_model.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/apache-airflow/security/security_model.rst b/docs/apache-airflow/security/security_model.rst index 0181710bda9dc..ebe1b35c54fab 100644 --- a/docs/apache-airflow/security/security_model.rst +++ b/docs/apache-airflow/security/security_model.rst @@ -212,12 +212,15 @@ DAG author to choose the code that will be executed in the scheduler or webserve should not be arbitrary code that DAG author can add in DAG folder. All those functionalities are only available via ``plugins`` and ``providers`` mechanisms where the code that is executed can only be provided by installed packages (or in case of plugins it can also be added to PLUGINS folder where DAG -authors should not have write access to). PLUGINS FOLDER is a legacy mechanism coming from Airflow 1.10 +authors should not have write access to). PLUGINS_FOLDER is a legacy mechanism coming from Airflow 1.10 - but we recommend using entrypoint mechanism that allows the Deployment Manager to - effectively - choose and register the code that will be executed in those contexts. DAG Author has no access to install or modify packages installed in Webserver and Scheduler, and this is the way to prevent the DAG Author to execute arbitrary code in those processes. +Additionally, if you decide to utilize and configure the PLUGINS_FOLDER, it is essential for the Deployment +Manager to ensure that the DAG author does not have write access to this folder. + The Deployment Manager might decide to introduce additional control mechanisms to prevent DAG authors from executing arbitrary code. This is all fully in hands of the Deployment Manager and it is discussed in the following chapter. From b21e83e8d0edc5f668bceafe45c00d25b361fb9d Mon Sep 17 00:00:00 2001 From: Gopal Dirisala <39794726+dirrao@users.noreply.github.com> Date: Tue, 15 Oct 2024 15:39:17 +0530 Subject: [PATCH 116/199] kubernetes package version bump to 31.0.0 (#42907) --- generated/provider_dependencies.json | 4 ++-- providers/src/airflow/providers/cncf/kubernetes/provider.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 49b457cd81a70..c91323180f7cc 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -370,8 +370,8 @@ "asgiref>=3.5.2", "cryptography>=41.0.0", "google-re2>=1.0", - "kubernetes>=29.0.0,<=30.1.0", - "kubernetes_asyncio>=29.0.0,<=30.1.0" + "kubernetes>=29.0.0,<=31.0.0", + "kubernetes_asyncio>=29.0.0,<=31.0.0" ], "devel-deps": [], "plugins": [], diff --git a/providers/src/airflow/providers/cncf/kubernetes/provider.yaml b/providers/src/airflow/providers/cncf/kubernetes/provider.yaml index 0849609b8ff80..980d552e15453 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/provider.yaml +++ b/providers/src/airflow/providers/cncf/kubernetes/provider.yaml @@ -104,13 +104,13 @@ dependencies: # limiting minimum airflow version supported in cncf.kubernetes provider, due to the # potential breaking changes in Airflow Core as well (kubernetes is added as extra, so Airflow # core is not hard-limited via install-requirements, only by extra). - - kubernetes>=29.0.0,<=30.1.0 + - kubernetes>=29.0.0,<=31.0.0 # The Kubernetes_asyncio package is used for providing Asynchronous (AsyncIO) client library for # standard Kubernetes API. The version is limited by minimum 18.20.1 because of introducing the ability to # load kubernetes config file from dictionary in that release and is limited to the next MAJOR version # (started from current 24.2.2 version) to prevent introducing some problems that could be due to some # major changes in the package. - - kubernetes_asyncio>=29.0.0,<=30.1.0 + - kubernetes_asyncio>=29.0.0,<=31.0.0 - google-re2>=1.0 integrations: From 08981991da1ea617665a182c384b8b5fe7dc1880 Mon Sep 17 00:00:00 2001 From: saucoide <32314353+saucoide@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:43:50 +1300 Subject: [PATCH 117/199] Make google provider pyarrow dependency explicit (#42996) * Make google provider pyarrow dependency explicit The provider already depends on it directly but it was not listed in provider.yaml * Add comment to the change * rerun hooks --- generated/provider_dependencies.json | 1 + providers/src/airflow/providers/google/provider.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index c91323180f7cc..e025266eebf81 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -686,6 +686,7 @@ "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", "proto-plus>=1.19.6", + "pyarrow>=14.0.1", "python-slugify>=7.0.0", "sqlalchemy-bigquery>=1.2.1", "sqlalchemy-spanner>=1.6.2", diff --git a/providers/src/airflow/providers/google/provider.yaml b/providers/src/airflow/providers/google/provider.yaml index 196a035badc4d..86a98a6962757 100644 --- a/providers/src/airflow/providers/google/provider.yaml +++ b/providers/src/airflow/providers/google/provider.yaml @@ -170,6 +170,8 @@ dependencies: # A transient dependency of google-cloud-bigquery-datatransfer, but we # further constrain it since older versions are buggy. - proto-plus>=1.19.6 + # Used to write parquet files by BaseSqlToGCSOperator + - pyarrow>=14.0.1 - python-slugify>=7.0.0 - PyOpenSSL>=23.0.0 - sqlalchemy-bigquery>=1.2.1 From b3611f01bb2616cd46aee93d4846955e6ce04276 Mon Sep 17 00:00:00 2001 From: Luyang Liu Date: Tue, 15 Oct 2024 23:30:19 +1100 Subject: [PATCH 118/199] Add search by dag_display_name_pattern on dag list page with rebase (#42896) * Add search by `dag_display_name_pattern` on dag list page. * refactor search parameter update logic * Refactor SearchBar component and simplify debounce logic in DagsList. * Refactor search input change handling for SearchBar and DagsList components * Update airflow/ui/src/pages/DagsList/DagsList.tsx Co-authored-by: Brent Bovenzi * Refactor search parameter handling and improve type consistency * Remove typo * Add `LAST_DAG_RUN_STATE` to `SearchParamsKeys` and update filters * Add LAST_DAG_RUN_STATE to SearchParamsKeys and update filters * Fix missing change for add SearchBar component by removing forwardRef and adding debounced search logic. * minor change for SearchBar and DagsFilters components * Optimize imports and improve formatting across components * refactor: move query options from autogenerated useDagServiceGetDags to DagsList. * Fix formatting. --------- Co-authored-by: Brent Bovenzi --- airflow/ui/package.json | 3 +- airflow/ui/pnpm-lock.yaml | 13 ++++ .../src/components/DataTable/searchParams.ts | 13 +++- airflow/ui/src/components/SearchBar.tsx | 62 +++++++++------ airflow/ui/src/constants/searchParams.ts | 31 ++++++++ airflow/ui/src/pages/DagsList/DagsFilters.tsx | 16 ++-- airflow/ui/src/pages/DagsList/DagsList.tsx | 78 +++++++++++++++---- 7 files changed, 169 insertions(+), 47 deletions(-) create mode 100644 airflow/ui/src/constants/searchParams.ts diff --git a/airflow/ui/package.json b/airflow/ui/package.json index 014564c1604bc..a7cf90bb5731b 100644 --- a/airflow/ui/package.json +++ b/airflow/ui/package.json @@ -28,7 +28,8 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-icons": "^5.3.0", - "react-router-dom": "^6.26.2" + "react-router-dom": "^6.26.2", + "use-debounce": "^10.0.3" }, "devDependencies": { "@7nohe/openapi-react-query-codegen": "^1.6.0", diff --git a/airflow/ui/pnpm-lock.yaml b/airflow/ui/pnpm-lock.yaml index c9f94d35f7358..3b73df0fa8049 100644 --- a/airflow/ui/pnpm-lock.yaml +++ b/airflow/ui/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: react-router-dom: specifier: ^6.26.2 version: 6.26.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + use-debounce: + specifier: ^10.0.3 + version: 10.0.3(react@18.3.1) devDependencies: '@7nohe/openapi-react-query-codegen': specifier: ^1.6.0 @@ -3221,6 +3224,12 @@ packages: '@types/react': optional: true + use-debounce@10.0.3: + resolution: {integrity: sha512-DxQSI9ZKso689WM1mjgGU3ozcxU1TJElBJ3X6S4SMzMNcm2lVH0AHmyXB+K7ewjz2BSUKJTDqTcwtSMRfB89dg==} + engines: {node: '>= 16.0.0'} + peerDependencies: + react: '*' + use-isomorphic-layout-effect@1.1.2: resolution: {integrity: sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==} peerDependencies: @@ -6892,6 +6901,10 @@ snapshots: optionalDependencies: '@types/react': 18.3.5 + use-debounce@10.0.3(react@18.3.1): + dependencies: + react: 18.3.1 + use-isomorphic-layout-effect@1.1.2(@types/react@18.3.5)(react@18.3.1): dependencies: react: 18.3.1 diff --git a/airflow/ui/src/components/DataTable/searchParams.ts b/airflow/ui/src/components/DataTable/searchParams.ts index 8cc57ad7b5bcf..39001b097f349 100644 --- a/airflow/ui/src/components/DataTable/searchParams.ts +++ b/airflow/ui/src/components/DataTable/searchParams.ts @@ -18,11 +18,18 @@ */ import type { SortingState } from "@tanstack/react-table"; +import { + SearchParamsKeys, + type SearchParamsKeysType, +} from "src/constants/searchParams"; + import type { TableState } from "./types"; -export const LIMIT_PARAM = "limit"; -export const OFFSET_PARAM = "offset"; -export const SORT_PARAM = "sort"; +const { + LIMIT: LIMIT_PARAM, + OFFSET: OFFSET_PARAM, + SORT: SORT_PARAM, +}: SearchParamsKeysType = SearchParamsKeys; export const stateToSearchParams = ( state: TableState, diff --git a/airflow/ui/src/components/SearchBar.tsx b/airflow/ui/src/components/SearchBar.tsx index 830f942677123..ad50a65b4311b 100644 --- a/airflow/ui/src/components/SearchBar.tsx +++ b/airflow/ui/src/components/SearchBar.tsx @@ -18,15 +18,19 @@ */ import { Button, - type ButtonProps, Input, InputGroup, - type InputGroupProps, InputLeftElement, - type InputProps, InputRightElement, + type ButtonProps, + type InputGroupProps, + type InputProps, } from "@chakra-ui/react"; +import type { ChangeEvent } from "react"; import { FiSearch } from "react-icons/fi"; +import { useDebouncedCallback } from "use-debounce"; + +const debounceDelay = 200; export const SearchBar = ({ buttonProps, @@ -36,23 +40,35 @@ export const SearchBar = ({ readonly buttonProps?: ButtonProps; readonly groupProps?: InputGroupProps; readonly inputProps?: InputProps; -}) => ( - - - - - - - - - -); +}) => { + const handleSearchChange = useDebouncedCallback( + (event: ChangeEvent) => inputProps?.onChange?.(event), + debounceDelay, + ); + + return ( + + + + + + + + + + ); +}; diff --git a/airflow/ui/src/constants/searchParams.ts b/airflow/ui/src/constants/searchParams.ts new file mode 100644 index 0000000000000..893a4461bffe7 --- /dev/null +++ b/airflow/ui/src/constants/searchParams.ts @@ -0,0 +1,31 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export enum SearchParamsKeys { + LAST_DAG_RUN_STATE = "last_dag_run_state", + LIMIT = "limit", + NAME_PATTERN = "name_pattern", + OFFSET = "offset", + PAUSED = "paused", + SORT = "sort", +} + +export type SearchParamsKeysType = Record< + keyof typeof SearchParamsKeys, + string +>; diff --git a/airflow/ui/src/pages/DagsList/DagsFilters.tsx b/airflow/ui/src/pages/DagsList/DagsFilters.tsx index 6316816f48f95..3d507ace365db 100644 --- a/airflow/ui/src/pages/DagsList/DagsFilters.tsx +++ b/airflow/ui/src/pages/DagsList/DagsFilters.tsx @@ -23,15 +23,21 @@ import { useSearchParams } from "react-router-dom"; import { useTableURLState } from "src/components/DataTable/useTableUrlState"; import { QuickFilterButton } from "src/components/QuickFilterButton"; +import { + SearchParamsKeys, + type SearchParamsKeysType, +} from "src/constants/searchParams"; -const PAUSED_PARAM = "paused"; -const STATE_PARAM = "last_dag_run_state"; +const { + LAST_DAG_RUN_STATE: LAST_DAG_RUN_STATE_PARAM, + PAUSED: PAUSED_PARAM, +}: SearchParamsKeysType = SearchParamsKeys; export const DagsFilters = () => { const [searchParams, setSearchParams] = useSearchParams(); const showPaused = searchParams.get(PAUSED_PARAM); - const state = searchParams.get(STATE_PARAM); + const state = searchParams.get(LAST_DAG_RUN_STATE_PARAM); const isAll = state === null; const isRunning = state === "running"; const isFailed = state === "failed"; @@ -61,9 +67,9 @@ export const DagsFilters = () => { useCallback( ({ currentTarget: { value } }) => { if (value === "all") { - searchParams.delete(STATE_PARAM); + searchParams.delete(LAST_DAG_RUN_STATE_PARAM); } else { - searchParams.set(STATE_PARAM, value); + searchParams.set(LAST_DAG_RUN_STATE_PARAM, value); } setSearchParams(searchParams); setTableURLState({ diff --git a/airflow/ui/src/pages/DagsList/DagsList.tsx b/airflow/ui/src/pages/DagsList/DagsList.tsx index 60d6ef9c4f437..623b8a3b4bacc 100644 --- a/airflow/ui/src/pages/DagsList/DagsList.tsx +++ b/airflow/ui/src/pages/DagsList/DagsList.tsx @@ -25,7 +25,12 @@ import { VStack, } from "@chakra-ui/react"; import type { ColumnDef } from "@tanstack/react-table"; -import { type ChangeEventHandler, useCallback, useState } from "react"; +import { + type ChangeEvent, + type ChangeEventHandler, + useCallback, + useState, +} from "react"; import { useSearchParams } from "react-router-dom"; import { useDagServiceGetDags } from "openapi/queries"; @@ -37,6 +42,10 @@ import { useTableURLState } from "src/components/DataTable/useTableUrlState"; import { ErrorAlert } from "src/components/ErrorAlert"; import { SearchBar } from "src/components/SearchBar"; import { TogglePause } from "src/components/TogglePause"; +import { + SearchParamsKeys, + type SearchParamsKeysType, +} from "src/constants/searchParams"; import { pluralize } from "src/utils/pluralize"; import { DagCard } from "./DagCard"; @@ -90,6 +99,12 @@ const columns: Array> = [ }, ]; +const { + LAST_DAG_RUN_STATE: LAST_DAG_RUN_STATE_PARAM, + NAME_PATTERN: NAME_PATTERN_PARAM, + PAUSED: PAUSED_PARAM, +}: SearchParamsKeysType = SearchParamsKeys; + const cardDef: CardDef = { card: ({ row }) => , meta: { @@ -97,31 +112,61 @@ const cardDef: CardDef = { }, }; -const PAUSED_PARAM = "paused"; -const STATE_PARAM = "last_dag_run_state"; - export const DagsList = () => { - const [searchParams] = useSearchParams(); + const [searchParams, setSearchParams] = useSearchParams(); const [display, setDisplay] = useState<"card" | "table">("card"); const showPaused = searchParams.get(PAUSED_PARAM); - const lastDagRunState = searchParams.get(STATE_PARAM) as DagRunState; + const lastDagRunState = searchParams.get( + LAST_DAG_RUN_STATE_PARAM, + ) as DagRunState; const { setTableURLState, tableURLState } = useTableURLState(); const { pagination, sorting } = tableURLState; + const [dagDisplayNamePattern, setDagDisplayNamePattern] = useState( + searchParams.get(NAME_PATTERN_PARAM) ?? undefined, + ); // TODO: update API to accept multiple orderBy params const [sort] = sorting; const orderBy = sort ? `${sort.desc ? "-" : ""}${sort.id}` : undefined; - const { data, error, isFetching, isLoading } = useDagServiceGetDags({ - lastDagRunState, - limit: pagination.pageSize, - offset: pagination.pageIndex * pagination.pageSize, - onlyActive: true, - orderBy, - paused: showPaused === null ? undefined : showPaused === "true", - }); + const handleSearchChange = ({ + target: { value }, + }: ChangeEvent) => { + if (value) { + searchParams.set(NAME_PATTERN_PARAM, value); + } else { + searchParams.delete(NAME_PATTERN_PARAM); + } + setSearchParams(searchParams); + setTableURLState({ + pagination: { ...pagination, pageIndex: 0 }, + sorting, + }); + setDagDisplayNamePattern(value); + }; + + const { data, error, isFetching, isLoading } = useDagServiceGetDags( + { + dagDisplayNamePattern: Boolean(dagDisplayNamePattern) + ? `%${dagDisplayNamePattern}%` + : undefined, + lastDagRunState, + limit: pagination.pageSize, + offset: pagination.pageIndex * pagination.pageSize, + onlyActive: true, + orderBy, + paused: showPaused === null ? undefined : showPaused === "true", + }, + [dagDisplayNamePattern, showPaused], + { + refetchOnMount: true, + refetchOnReconnect: false, + refetchOnWindowFocus: false, + staleTime: 5 * 60 * 1000, + }, + ); const handleSortChange = useCallback>( ({ currentTarget: { value } }) => { @@ -140,7 +185,10 @@ export const DagsList = () => { From b47703314423ccadcc31c34862bcb7e81368f0ca Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:52:35 -0400 Subject: [PATCH 119/199] Deprecate session auth backend (#42909) --- airflow/api/auth/backend/session.py | 31 +++++++------------ .../auth_manager/api_endpoints/test_auth.py | 2 +- providers/tests/fab/auth_manager/conftest.py | 2 +- tests/api_connexion/test_auth.py | 4 ++- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/airflow/api/auth/backend/session.py b/airflow/api/auth/backend/session.py index d51f7bf1cf4c9..e6b4da691213e 100644 --- a/airflow/api/auth/backend/session.py +++ b/airflow/api/auth/backend/session.py @@ -18,30 +18,21 @@ from __future__ import annotations -from functools import wraps -from typing import Any, Callable, TypeVar, cast +import warnings +from typing import Any -from flask import Response - -from airflow.www.extensions.init_auth_manager import get_auth_manager +import airflow.providers.fab.auth_manager.api.auth.backend.session as fab_session +from airflow.exceptions import RemovedInAirflow3Warning CLIENT_AUTH: tuple[str, str] | Any | None = None -def init_app(_): - """Initialize authentication backend.""" - - -T = TypeVar("T", bound=Callable) - - -def requires_authentication(function: T): - """Decorate functions that require authentication.""" +warnings.warn( + "This module is deprecated. Please use `airflow.providers.fab.auth_manager.api.auth.backend.session` instead.", + RemovedInAirflow3Warning, + stacklevel=2, +) - @wraps(function) - def decorated(*args, **kwargs): - if not get_auth_manager().is_logged_in(): - return Response("Unauthorized", 401, {}) - return function(*args, **kwargs) - return cast(T, decorated) +init_app = fab_session.init_app +requires_authentication = fab_session.requires_authentication diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_auth.py b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py index fd29120070268..6a90b7ec4b30e 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_auth.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py @@ -147,7 +147,7 @@ def with_basic_auth_backend(self, minimal_app_for_auth_api): ( "api", "auth_backends", - ): "airflow.api.auth.backend.session,airflow.providers.fab.auth_manager.api.auth.backend.basic_auth" + ): "airflow.providers.fab.auth_manager.api.auth.backend.session,airflow.providers.fab.auth_manager.api.auth.backend.basic_auth" } ): init_api_auth(minimal_app_for_auth_api) diff --git a/providers/tests/fab/auth_manager/conftest.py b/providers/tests/fab/auth_manager/conftest.py index 5818ecf435db6..970889e2ec397 100644 --- a/providers/tests/fab/auth_manager/conftest.py +++ b/providers/tests/fab/auth_manager/conftest.py @@ -42,7 +42,7 @@ def factory(): ( "api", "auth_backends", - ): "providers.tests.fab.auth_manager.api_endpoints.remote_user_api_auth_backend,airflow.api.auth.backend.session", + ): "providers.tests.fab.auth_manager.api_endpoints.remote_user_api_auth_backend,airflow.providers.fab.auth_manager.api.auth.backend.session", ( "core", "auth_manager", diff --git a/tests/api_connexion/test_auth.py b/tests/api_connexion/test_auth.py index 65e98c18c974d..f3c35bc5fb372 100644 --- a/tests/api_connexion/test_auth.py +++ b/tests/api_connexion/test_auth.py @@ -43,7 +43,9 @@ def with_session_backend(self, minimal_app_for_api): old_auth = getattr(minimal_app_for_api, "api_auth") try: - with conf_vars({("api", "auth_backends"): "airflow.api.auth.backend.session"}): + with conf_vars( + {("api", "auth_backends"): "airflow.providers.fab.auth_manager.api.auth.backend.session"} + ): init_api_auth(minimal_app_for_api) yield finally: From 96916fc7fc44b1865548b0494bb8b490e8ec6c30 Mon Sep 17 00:00:00 2001 From: Geonwoo Kim <78521993+kgw7401@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:16:08 +0900 Subject: [PATCH 120/199] fix: Change CustomSecurityManager method name (#43034) * oauth_user_info -> get_oauth_user_info --- .../auth-manager/webserver-authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst b/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst index 9bdcd6178273a..8cc4c3823e2d4 100644 --- a/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst +++ b/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst @@ -289,7 +289,7 @@ Here is an example of what you might have in your webserver_config.py: class CustomSecurityManager(AirflowSecurityManager): - def oauth_user_info(self, provider, response): + def get_oauth_user_info(self, provider, response): if provider == "keycloak": token = response["access_token"] me = jwt.decode(token, public_key, algorithms=["HS256", "RS256"]) From 37a541433813ae1caebfe5df361388e0148bcbfc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 15 Oct 2024 16:59:02 +0200 Subject: [PATCH 121/199] Update to 0.4.21 of UV (#43032) --- Dockerfile | 2 +- Dockerfile.ci | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c762cae900f2f..e0261b8f69d20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.10.2" ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.20 +ARG AIRFLOW_UV_VERSION=0.4.21 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index 0cdf9899799f3..bbdf044158089 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1278,7 +1278,7 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.20 +ARG AIRFLOW_UV_VERSION=0.4.21 ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller @@ -1302,7 +1302,7 @@ ARG AIRFLOW_VERSION="" ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.7 +ARG AIRFLOW_UV_VERSION=0.4.21 ARG AIRFLOW_USE_UV="true" ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ From 8fb844c9dd1dabe060e6f7be1474c04296c6b7be Mon Sep 17 00:00:00 2001 From: majorosdonat Date: Tue, 15 Oct 2024 17:50:56 +0200 Subject: [PATCH 122/199] Add retry on error 502 and 504 (#42994) * Add retry on error 502 and 504 * fix mypy findings * Add pytest * Convert response code to HTTPStatus * Add docs to retriable exception * extend docs for AirflowHttpException * Fix syntax and typos * fix pytest * fix static checks * fix some static checks * Fix ruff * fix pre-commit --------- Co-authored-by: Majoros Donat (XC-DX/EET2-Bp) --- airflow/api_internal/internal_api_call.py | 33 ++++++++++++++++++-- tests/api_internal/test_internal_api_call.py | 23 ++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/airflow/api_internal/internal_api_call.py b/airflow/api_internal/internal_api_call.py index 8838377877bec..064834d7c8673 100644 --- a/airflow/api_internal/internal_api_call.py +++ b/airflow/api_internal/internal_api_call.py @@ -21,6 +21,7 @@ import json import logging from functools import wraps +from http import HTTPStatus from typing import Callable, TypeVar from urllib.parse import urlparse @@ -40,6 +41,14 @@ logger = logging.getLogger(__name__) +class AirflowHttpException(AirflowException): + """Raise when there is a problem during an http request on the internal API decorator.""" + + def __init__(self, message: str, status_code: HTTPStatus): + super().__init__(message) + self.status_code = status_code + + class InternalApiConfig: """Stores and caches configuration for Internal API.""" @@ -105,10 +114,27 @@ def internal_api_call(func: Callable[PS, RT]) -> Callable[PS, RT]: """ from requests.exceptions import ConnectionError + def _is_retryable_exception(exception: BaseException) -> bool: + """ + Evaluate which exception types to retry. + + This is especially demanded for cases where an application gateway or Kubernetes ingress can + not find a running instance of a webserver hosting the API (HTTP 502+504) or when the + HTTP request fails in general on network level. + + Note that we want to fail on other general errors on the webserver not to send bad requests in an endless loop. + """ + retryable_status_codes = (HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT) + return ( + isinstance(exception, AirflowHttpException) + and exception.status_code in retryable_status_codes + or isinstance(exception, (ConnectionError, NewConnectionError)) + ) + @tenacity.retry( stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_exponential(min=1), - retry=tenacity.retry_if_exception_type((NewConnectionError, ConnectionError)), + retry=tenacity.retry_if_exception(_is_retryable_exception), before_sleep=tenacity.before_log(logger, logging.WARNING), ) def make_jsonrpc_request(method_name: str, params_json: str) -> bytes: @@ -126,9 +152,10 @@ def make_jsonrpc_request(method_name: str, params_json: str) -> bytes: internal_api_endpoint = InternalApiConfig.get_internal_api_endpoint() response = requests.post(url=internal_api_endpoint, data=json.dumps(data), headers=headers) if response.status_code != 200: - raise AirflowException( + raise AirflowHttpException( f"Got {response.status_code}:{response.reason} when sending " - f"the internal api request: {response.text}" + f"the internal api request: {response.text}", + HTTPStatus(response.status_code), ) return response.content diff --git a/tests/api_internal/test_internal_api_call.py b/tests/api_internal/test_internal_api_call.py index 880e1e89b0388..ff9c01e3b554d 100644 --- a/tests/api_internal/test_internal_api_call.py +++ b/tests/api_internal/test_internal_api_call.py @@ -25,6 +25,7 @@ import pytest import requests +from tenacity import RetryError from airflow.__main__ import configure_internal_api from airflow.api_internal.internal_api_call import InternalApiConfig, internal_api_call @@ -266,6 +267,28 @@ def test_remote_classmethod_call_with_params(self, mock_requests): assert call_kwargs["headers"]["Content-Type"] == "application/json" assert "Authorization" in call_kwargs["headers"] + @conf_vars( + { + ("core", "database_access_isolation"): "true", + ("core", "internal_api_url"): "http://localhost:8888", + ("database", "sql_alchemy_conn"): "none://", + } + ) + @mock.patch("airflow.api_internal.internal_api_call.requests") + @mock.patch("tenacity.time.sleep") + def test_retry_on_bad_gateway(self, mock_sleep, mock_requests): + configure_internal_api(Namespace(subcommand="dag-processor"), conf) + response = requests.Response() + response.status_code = 502 + response.reason = "Bad Gateway" + response._content = b"Bad Gateway" + + mock_sleep = lambda *_, **__: None # noqa: F841 + mock_requests.post.return_value = response + with pytest.raises(RetryError): + TestInternalApiCall.fake_method_with_params("fake-dag", task_id=123, session="session") + assert mock_requests.post.call_count == 10 + @conf_vars( { ("core", "database_access_isolation"): "true", From 1824f18c493bf5dbc756218f39633f3635955dae Mon Sep 17 00:00:00 2001 From: Bugra Ozturk Date: Tue, 15 Oct 2024 18:05:58 +0200 Subject: [PATCH 123/199] AIP-84 Migrate get connections to FastAPI API #42571 (#42782) * Make SortParam parent for Model Specific SortParams, Include get connections endpoint to fastapi * Change depends() method regular method in SortParam due to parent class already have abstract * Remove subclass, get default order_by from primary key, change alias strategy for backcompat * pre-commit hooks * Dynamic return value of SortParam generated within openapi specs and removed unnecessary attribute mapping keys * Include connection_id to attr_mapping again * Dynamic depends with correct documentation * Add more tests --------- Co-authored-by: pierrejeambrun --- .../endpoints/connection_endpoint.py | 1 + airflow/api_fastapi/openapi/v1-generated.yaml | 82 ++++++++++++++++++- airflow/api_fastapi/parameters.py | 48 +++++++++-- .../api_fastapi/serializers/connections.py | 9 +- .../api_fastapi/views/public/connections.py | 42 +++++++++- airflow/api_fastapi/views/public/dags.py | 5 +- airflow/ui/openapi-gen/queries/common.ts | 24 ++++++ airflow/ui/openapi-gen/queries/prefetch.ts | 30 +++++++ airflow/ui/openapi-gen/queries/queries.ts | 36 ++++++++ airflow/ui/openapi-gen/queries/suspense.ts | 36 ++++++++ .../ui/openapi-gen/requests/schemas.gen.ts | 26 +++++- .../ui/openapi-gen/requests/services.gen.ts | 32 ++++++++ airflow/ui/openapi-gen/requests/types.gen.ts | 45 +++++++++- .../views/public/test_connections.py | 74 ++++++++++++++++- 14 files changed, 466 insertions(+), 24 deletions(-) diff --git a/airflow/api_connexion/endpoints/connection_endpoint.py b/airflow/api_connexion/endpoints/connection_endpoint.py index 7cea4cf9837c5..37c91c44eb698 100644 --- a/airflow/api_connexion/endpoints/connection_endpoint.py +++ b/airflow/api_connexion/endpoints/connection_endpoint.py @@ -92,6 +92,7 @@ def get_connection(*, connection_id: str, session: Session = NEW_SESSION) -> API @security.requires_access_connection("GET") @format_parameters({"limit": check_limit}) @provide_session +@mark_fastapi_migration_done def get_connections( *, limit: int, diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index 56f48c73e987f..d9339fdb56cff 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -593,6 +593,66 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/connections/: + get: + tags: + - Connection + summary: Get Connections + description: Get all connection entries. + operationId: get_connections + parameters: + - name: limit + in: query + required: false + schema: + type: integer + default: 100 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + - name: order_by + in: query + required: false + schema: + type: string + default: id + title: Order By + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ConnectionCollectionResponse' + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /public/variables/{variable_key}: delete: tags: @@ -886,11 +946,27 @@ paths: $ref: '#/components/schemas/HTTPValidationError' components: schemas: + ConnectionCollectionResponse: + properties: + connections: + items: + $ref: '#/components/schemas/ConnectionResponse' + type: array + title: Connections + total_entries: + type: integer + title: Total Entries + type: object + required: + - connections + - total_entries + title: ConnectionCollectionResponse + description: DAG Collection serializer for responses. ConnectionResponse: properties: - conn_id: + connection_id: type: string - title: Conn Id + title: Connection Id conn_type: type: string title: Conn Type @@ -926,7 +1002,7 @@ components: title: Extra type: object required: - - conn_id + - connection_id - conn_type - description - host diff --git a/airflow/api_fastapi/parameters.py b/airflow/api_fastapi/parameters.py index 59d61ad68600d..07d8a76c79c3e 100644 --- a/airflow/api_fastapi/parameters.py +++ b/airflow/api_fastapi/parameters.py @@ -17,16 +17,19 @@ from __future__ import annotations +import importlib from abc import ABC, abstractmethod from datetime import datetime -from typing import TYPE_CHECKING, Any, Generic, List, TypeVar +from typing import TYPE_CHECKING, Any, Callable, Generic, List, TypeVar from fastapi import Depends, HTTPException, Query from pendulum.parsing.exceptions import ParserError from pydantic import AfterValidator -from sqlalchemy import case, or_ +from sqlalchemy import Column, case, or_ +from sqlalchemy.inspection import inspect from typing_extensions import Annotated, Self +from airflow.models import Base, Connection from airflow.models.dag import DagModel, DagTag from airflow.models.dagrun import DagRun from airflow.utils import timezone @@ -154,11 +157,17 @@ class SortParam(BaseParam[str]): attr_mapping = { "last_run_state": DagRun.state, "last_run_start_date": DagRun.start_date, + "connection_id": Connection.conn_id, } - def __init__(self, allowed_attrs: list[str]) -> None: + def __init__( + self, + allowed_attrs: list[str], + model: Base, + ) -> None: super().__init__() self.allowed_attrs = allowed_attrs + self.model = model def to_orm(self, select: Select) -> Select: if self.skip_none is False: @@ -175,7 +184,9 @@ def to_orm(self, select: Select) -> Select: f"the attribute does not exist on the model", ) - column = self.attr_mapping.get(lstriped_orderby, None) or getattr(DagModel, lstriped_orderby) + column: Column = self.attr_mapping.get(lstriped_orderby, None) or getattr( + self.model, lstriped_orderby + ) # MySQL does not support `nullslast`, and True/False ordering depends on the # database implementation. @@ -185,12 +196,33 @@ def to_orm(self, select: Select) -> Select: select = select.order_by(None) if self.value[0] == "-": - return select.order_by(nullscheck, column.desc(), DagModel.dag_id.desc()) + return select.order_by(nullscheck, column.desc(), column.desc()) else: - return select.order_by(nullscheck, column.asc(), DagModel.dag_id.asc()) + return select.order_by(nullscheck, column.asc(), column.asc()) + + def get_primary_key(self) -> str: + """Get the primary key of the model of SortParam object.""" + return inspect(self.model).primary_key[0].name + + @staticmethod + def get_primary_key_of_given_model_string(model_string: str) -> str: + """ + Get the primary key of given 'airflow.models' class as a string. The class should have driven be from 'airflow.models.base'. + + :param model_string: The string representation of the model class. + :return: The primary key of the model class. + """ + dynamic_return_model = getattr(importlib.import_module("airflow.models"), model_string) + return inspect(dynamic_return_model).primary_key[0].name + + def depends(self, *args: Any, **kwargs: Any) -> Self: + raise NotImplementedError("Use dynamic_depends, depends not implemented.") + + def dynamic_depends(self) -> Callable: + def inner(order_by: str = self.get_primary_key()) -> SortParam: + return self.set_value(self.get_primary_key() if order_by == "" else order_by) - def depends(self, order_by: str = "dag_id") -> SortParam: - return self.set_value(order_by) + return inner class _TagsFilter(BaseParam[List[str]]): diff --git a/airflow/api_fastapi/serializers/connections.py b/airflow/api_fastapi/serializers/connections.py index e40b2fa1b2113..1c801607299fe 100644 --- a/airflow/api_fastapi/serializers/connections.py +++ b/airflow/api_fastapi/serializers/connections.py @@ -27,7 +27,7 @@ class ConnectionResponse(BaseModel): """Connection serializer for responses.""" - connection_id: str = Field(alias="conn_id") + connection_id: str = Field(serialization_alias="connection_id", validation_alias="conn_id") conn_type: str description: str | None host: str | None @@ -48,3 +48,10 @@ def redact_extra(cls, v: str | None) -> str | None: except json.JSONDecodeError: # we can't redact fields in an unstructured `extra` return v + + +class ConnectionCollectionResponse(BaseModel): + """DAG Collection serializer for responses.""" + + connections: list[ConnectionResponse] + total_entries: int diff --git a/airflow/api_fastapi/views/public/connections.py b/airflow/api_fastapi/views/public/connections.py index 94e9b614e9c02..6fca43aca26c7 100644 --- a/airflow/api_fastapi/views/public/connections.py +++ b/airflow/api_fastapi/views/public/connections.py @@ -21,9 +21,10 @@ from sqlalchemy.orm import Session from typing_extensions import Annotated -from airflow.api_fastapi.db.common import get_session +from airflow.api_fastapi.db.common import get_session, paginated_select from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.serializers.connections import ConnectionResponse +from airflow.api_fastapi.parameters import QueryLimit, QueryOffset, SortParam +from airflow.api_fastapi.serializers.connections import ConnectionCollectionResponse, ConnectionResponse from airflow.api_fastapi.views.router import AirflowRouter from airflow.models import Connection @@ -63,3 +64,40 @@ async def get_connection( raise HTTPException(404, f"The Connection with connection_id: `{connection_id}` was not found") return ConnectionResponse.model_validate(connection, from_attributes=True) + + +@connections_router.get( + "/", + responses=create_openapi_http_exception_doc([401, 403, 404]), +) +async def get_connections( + limit: QueryLimit, + offset: QueryOffset, + order_by: Annotated[ + SortParam, + Depends( + SortParam( + ["connection_id", "conn_type", "description", "host", "port", "id"], Connection + ).dynamic_depends() + ), + ], + session: Annotated[Session, Depends(get_session)], +) -> ConnectionCollectionResponse: + """Get all connection entries.""" + connection_select, total_entries = paginated_select( + select(Connection), + [], + order_by=order_by, + offset=offset, + limit=limit, + session=session, + ) + + connections = session.scalars(connection_select).all() + + return ConnectionCollectionResponse( + connections=[ + ConnectionResponse.model_validate(connection, from_attributes=True) for connection in connections + ], + total_entries=total_entries, + ) diff --git a/airflow/api_fastapi/views/public/dags.py b/airflow/api_fastapi/views/public/dags.py index eb8233a7f700d..46d4d8b3540c0 100644 --- a/airflow/api_fastapi/views/public/dags.py +++ b/airflow/api_fastapi/views/public/dags.py @@ -70,8 +70,9 @@ async def get_dags( SortParam, Depends( SortParam( - ["dag_id", "dag_display_name", "next_dagrun", "last_run_state", "last_run_start_date"] - ).depends + ["dag_id", "dag_display_name", "next_dagrun", "last_run_state", "last_run_start_date"], + DagModel, + ).dynamic_depends() ), ], session: Annotated[Session, Depends(get_session)], diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 2f1c6a78d92f0..30a20826109c6 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -151,6 +151,30 @@ export const UseConnectionServiceGetConnectionKeyFn = ( useConnectionServiceGetConnectionKey, ...(queryKey ?? [{ connectionId }]), ]; +export type ConnectionServiceGetConnectionsDefaultResponse = Awaited< + ReturnType +>; +export type ConnectionServiceGetConnectionsQueryResult< + TData = ConnectionServiceGetConnectionsDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useConnectionServiceGetConnectionsKey = + "ConnectionServiceGetConnections"; +export const UseConnectionServiceGetConnectionsKeyFn = ( + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, + queryKey?: Array, +) => [ + useConnectionServiceGetConnectionsKey, + ...(queryKey ?? [{ limit, offset, orderBy }]), +]; export type VariableServiceGetVariableDefaultResponse = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index 3e194302f4be0..79b310107d27c 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -187,6 +187,36 @@ export const prefetchUseConnectionServiceGetConnection = ( queryKey: Common.UseConnectionServiceGetConnectionKeyFn({ connectionId }), queryFn: () => ConnectionService.getConnection({ connectionId }), }); +/** + * Get Connections + * Get all connection entries. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @param data.orderBy + * @returns ConnectionCollectionResponse Successful Response + * @throws ApiError + */ +export const prefetchUseConnectionServiceGetConnections = ( + queryClient: QueryClient, + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseConnectionServiceGetConnectionsKeyFn({ + limit, + offset, + orderBy, + }), + queryFn: () => ConnectionService.getConnections({ limit, offset, orderBy }), + }); /** * Get Variable * Get a variable entry. diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index a16bdf165b182..1e9586e436018 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -235,6 +235,42 @@ export const useConnectionServiceGetConnection = < queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, ...options, }); +/** + * Get Connections + * Get all connection entries. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @param data.orderBy + * @returns ConnectionCollectionResponse Successful Response + * @throws ApiError + */ +export const useConnectionServiceGetConnections = < + TData = Common.ConnectionServiceGetConnectionsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseConnectionServiceGetConnectionsKeyFn( + { limit, offset, orderBy }, + queryKey, + ), + queryFn: () => + ConnectionService.getConnections({ limit, offset, orderBy }) as TData, + ...options, + }); /** * Get Variable * Get a variable entry. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index 79ad479f0a42f..af2a571871eb1 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -230,6 +230,42 @@ export const useConnectionServiceGetConnectionSuspense = < queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, ...options, }); +/** + * Get Connections + * Get all connection entries. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @param data.orderBy + * @returns ConnectionCollectionResponse Successful Response + * @throws ApiError + */ +export const useConnectionServiceGetConnectionsSuspense = < + TData = Common.ConnectionServiceGetConnectionsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseConnectionServiceGetConnectionsKeyFn( + { limit, offset, orderBy }, + queryKey, + ), + queryFn: () => + ConnectionService.getConnections({ limit, offset, orderBy }) as TData, + ...options, + }); /** * Get Variable * Get a variable entry. diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index e42a3f6572ca9..44a29d3899d2a 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -1,10 +1,30 @@ // This file is auto-generated by @hey-api/openapi-ts +export const $ConnectionCollectionResponse = { + properties: { + connections: { + items: { + $ref: "#/components/schemas/ConnectionResponse", + }, + type: "array", + title: "Connections", + }, + total_entries: { + type: "integer", + title: "Total Entries", + }, + }, + type: "object", + required: ["connections", "total_entries"], + title: "ConnectionCollectionResponse", + description: "DAG Collection serializer for responses.", +} as const; + export const $ConnectionResponse = { properties: { - conn_id: { + connection_id: { type: "string", - title: "Conn Id", + title: "Connection Id", }, conn_type: { type: "string", @@ -79,7 +99,7 @@ export const $ConnectionResponse = { }, type: "object", required: [ - "conn_id", + "connection_id", "conn_type", "description", "host", diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 8d7f0cee2b295..6f9b0aa5cb8a1 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -23,6 +23,8 @@ import type { DeleteConnectionResponse, GetConnectionData, GetConnectionResponse, + GetConnectionsData, + GetConnectionsResponse, DeleteVariableData, DeleteVariableResponse, GetVariableData, @@ -343,6 +345,36 @@ export class ConnectionService { }, }); } + + /** + * Get Connections + * Get all connection entries. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @param data.orderBy + * @returns ConnectionCollectionResponse Successful Response + * @throws ApiError + */ + public static getConnections( + data: GetConnectionsData = {}, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/connections/", + query: { + limit: data.limit, + offset: data.offset, + order_by: data.orderBy, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Validation Error", + }, + }); + } } export class VariableService { diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 7f603a1adb4d3..2fae25c7660c4 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1,10 +1,18 @@ // This file is auto-generated by @hey-api/openapi-ts +/** + * DAG Collection serializer for responses. + */ +export type ConnectionCollectionResponse = { + connections: Array; + total_entries: number; +}; + /** * Connection serializer for responses. */ export type ConnectionResponse = { - conn_id: string; + connection_id: string; conn_type: string; description: string | null; host: string | null; @@ -351,6 +359,14 @@ export type GetConnectionData = { export type GetConnectionResponse = ConnectionResponse; +export type GetConnectionsData = { + limit?: number; + offset?: number; + orderBy?: string; +}; + +export type GetConnectionsResponse = ConnectionCollectionResponse; + export type DeleteVariableData = { variableKey: string; }; @@ -644,6 +660,33 @@ export type $OpenApiTs = { }; }; }; + "/public/connections/": { + get: { + req: GetConnectionsData; + res: { + /** + * Successful Response + */ + 200: ConnectionCollectionResponse; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; "/public/variables/{variable_key}": { delete: { req: DeleteVariableData; diff --git a/tests/api_fastapi/views/public/test_connections.py b/tests/api_fastapi/views/public/test_connections.py index a0da0d2b9b28e..a80f2467b9b72 100644 --- a/tests/api_fastapi/views/public/test_connections.py +++ b/tests/api_fastapi/views/public/test_connections.py @@ -26,14 +26,43 @@ TEST_CONN_ID = "test_connection_id" TEST_CONN_TYPE = "test_type" +TEST_CONN_DESCRIPTION = "some_description_a" +TEST_CONN_HOST = "some_host_a" +TEST_CONN_PORT = 8080 + + +TEST_CONN_ID_2 = "test_connection_id_2" +TEST_CONN_TYPE_2 = "test_type_2" +TEST_CONN_DESCRIPTION_2 = "some_description_b" +TEST_CONN_HOST_2 = "some_host_b" +TEST_CONN_PORT_2 = 8081 @provide_session def _create_connection(session) -> None: - connection_model = Connection(conn_id=TEST_CONN_ID, conn_type=TEST_CONN_TYPE) + connection_model = Connection( + conn_id=TEST_CONN_ID, + conn_type=TEST_CONN_TYPE, + description=TEST_CONN_DESCRIPTION, + host=TEST_CONN_HOST, + port=TEST_CONN_PORT, + ) session.add(connection_model) +@provide_session +def _create_connections(session) -> None: + _create_connection(session) + connection_model_2 = Connection( + conn_id=TEST_CONN_ID_2, + conn_type=TEST_CONN_TYPE_2, + description=TEST_CONN_DESCRIPTION_2, + host=TEST_CONN_HOST_2, + port=TEST_CONN_PORT_2, + ) + session.add(connection_model_2) + + class TestConnectionEndpoint: @pytest.fixture(autouse=True) def setup(self) -> None: @@ -45,6 +74,9 @@ def teardown_method(self) -> None: def create_connection(self): _create_connection() + def create_connections(self): + _create_connections() + class TestDeleteConnection(TestConnectionEndpoint): def test_delete_should_respond_204(self, test_client, session): @@ -69,7 +101,7 @@ def test_get_should_respond_200(self, test_client, session): response = test_client.get(f"/public/connections/{TEST_CONN_ID}") assert response.status_code == 200 body = response.json() - assert body["conn_id"] == TEST_CONN_ID + assert body["connection_id"] == TEST_CONN_ID assert body["conn_type"] == TEST_CONN_TYPE def test_get_should_respond_404(self, test_client): @@ -86,7 +118,7 @@ def test_get_should_respond_200_with_extra(self, test_client, session): response = test_client.get(f"/public/connections/{TEST_CONN_ID}") assert response.status_code == 200 body = response.json() - assert body["conn_id"] == TEST_CONN_ID + assert body["connection_id"] == TEST_CONN_ID assert body["conn_type"] == TEST_CONN_TYPE assert body["extra"] == '{"extra_key": "extra_value"}' @@ -99,6 +131,40 @@ def test_get_should_respond_200_with_extra_redacted(self, test_client, session): response = test_client.get(f"/public/connections/{TEST_CONN_ID}") assert response.status_code == 200 body = response.json() - assert body["conn_id"] == TEST_CONN_ID + assert body["connection_id"] == TEST_CONN_ID assert body["conn_type"] == TEST_CONN_TYPE assert body["extra"] == '{"password": "***"}' + + +class TestGetConnections(TestConnectionEndpoint): + @pytest.mark.parametrize( + "query_params, expected_total_entries, expected_ids", + [ + # Filters + ({}, 2, [TEST_CONN_ID, TEST_CONN_ID_2]), + ({"limit": 1}, 2, [TEST_CONN_ID]), + ({"limit": 1, "offset": 1}, 2, [TEST_CONN_ID_2]), + # Sort + ({"order_by": "-connection_id"}, 2, [TEST_CONN_ID_2, TEST_CONN_ID]), + ({"order_by": "conn_type"}, 2, [TEST_CONN_ID, TEST_CONN_ID_2]), + ({"order_by": "-conn_type"}, 2, [TEST_CONN_ID_2, TEST_CONN_ID]), + ({"order_by": "description"}, 2, [TEST_CONN_ID, TEST_CONN_ID_2]), + ({"order_by": "-description"}, 2, [TEST_CONN_ID_2, TEST_CONN_ID]), + ({"order_by": "host"}, 2, [TEST_CONN_ID, TEST_CONN_ID_2]), + ({"order_by": "-host"}, 2, [TEST_CONN_ID_2, TEST_CONN_ID]), + ({"order_by": "port"}, 2, [TEST_CONN_ID, TEST_CONN_ID_2]), + ({"order_by": "-port"}, 2, [TEST_CONN_ID_2, TEST_CONN_ID]), + ({"order_by": "id"}, 2, [TEST_CONN_ID, TEST_CONN_ID_2]), + ({"order_by": "-id"}, 2, [TEST_CONN_ID_2, TEST_CONN_ID]), + ], + ) + def test_should_respond_200( + self, test_client, session, query_params, expected_total_entries, expected_ids + ): + self.create_connections() + response = test_client.get("/public/connections/", params=query_params) + assert response.status_code == 200 + + body = response.json() + assert body["total_entries"] == expected_total_entries + assert [dag["connection_id"] for dag in body["connections"]] == expected_ids From dca05348f2ebb4dd0acfde1ae34e1e49300386b6 Mon Sep 17 00:00:00 2001 From: Jake Roach <116606359+jroachgolf84@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:17:11 -0400 Subject: [PATCH 124/199] Make `RedshiftDataOperator` handle multiple queries (#42900) * Adding updates to RedshiftDataOperator and unit tests * Adding updates to RedshiftDataOperator and unit tests * Updating how statement_id is used * Ruff formatting * Fixing nit --- .../amazon/aws/operators/redshift_data.py | 63 ++++++++---- .../aws/operators/test_redshift_data.py | 95 +++++++++++++------ .../system/amazon/aws/example_redshift.py | 38 ++++---- 3 files changed, 130 insertions(+), 66 deletions(-) diff --git a/providers/src/airflow/providers/amazon/aws/operators/redshift_data.py b/providers/src/airflow/providers/amazon/aws/operators/redshift_data.py index 3d00c6d22edf7..0813918854464 100644 --- a/providers/src/airflow/providers/amazon/aws/operators/redshift_data.py +++ b/providers/src/airflow/providers/amazon/aws/operators/redshift_data.py @@ -28,7 +28,10 @@ from airflow.providers.amazon.aws.utils.mixins import aws_template_fields if TYPE_CHECKING: - from mypy_boto3_redshift_data.type_defs import GetStatementResultResponseTypeDef + from mypy_boto3_redshift_data.type_defs import ( + DescribeStatementResponseTypeDef, + GetStatementResultResponseTypeDef, + ) from airflow.utils.context import Context @@ -37,7 +40,7 @@ class RedshiftDataOperator(AwsBaseOperator[RedshiftDataHook]): """ Executes SQL Statements against an Amazon Redshift cluster using Redshift Data. - .. seealso:: + ... see also:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:RedshiftDataOperator` @@ -84,7 +87,6 @@ class RedshiftDataOperator(AwsBaseOperator[RedshiftDataHook]): ) template_ext = (".sql",) template_fields_renderers = {"sql": "sql"} - statement_id: str | None def __init__( self, @@ -124,12 +126,11 @@ def __init__( poll_interval, ) self.return_sql_result = return_sql_result - self.statement_id: str | None = None self.deferrable = deferrable self.session_id = session_id self.session_keep_alive_seconds = session_keep_alive_seconds - def execute(self, context: Context) -> GetStatementResultResponseTypeDef | str: + def execute(self, context: Context) -> list[GetStatementResultResponseTypeDef] | list[str]: """Execute a statement against Amazon Redshift.""" self.log.info("Executing statement: %s", self.sql) @@ -154,13 +155,14 @@ def execute(self, context: Context) -> GetStatementResultResponseTypeDef | str: session_keep_alive_seconds=self.session_keep_alive_seconds, ) - self.statement_id = query_execution_output.statement_id + # Pull the statement ID, session ID + self.statement_id: str = query_execution_output.statement_id if query_execution_output.session_id: self.xcom_push(context, key="session_id", value=query_execution_output.session_id) if self.deferrable and self.wait_for_completion: - is_finished = self.hook.check_query_is_finished(self.statement_id) + is_finished: bool = self.hook.check_query_is_finished(self.statement_id) if not is_finished: self.defer( timeout=self.execution_timeout, @@ -176,16 +178,13 @@ def execute(self, context: Context) -> GetStatementResultResponseTypeDef | str: method_name="execute_complete", ) - if self.return_sql_result: - result = self.hook.conn.get_statement_result(Id=self.statement_id) - self.log.debug("Statement result: %s", result) - return result - else: - return self.statement_id + # Use the get_sql_results method to return the results of the SQL query, or the statement_ids, + # depending on the value of self.return_sql_result + return self.get_sql_results(statement_id=self.statement_id, return_sql_result=self.return_sql_result) def execute_complete( self, context: Context, event: dict[str, Any] | None = None - ) -> GetStatementResultResponseTypeDef | str: + ) -> list[GetStatementResultResponseTypeDef] | list[str]: event = validate_execute_complete_event(event) if event["status"] == "error": @@ -197,16 +196,40 @@ def execute_complete( raise AirflowException("statement_id should not be empty.") self.log.info("%s completed successfully.", self.task_id) - if self.return_sql_result: - result = self.hook.conn.get_statement_result(Id=statement_id) - self.log.debug("Statement result: %s", result) - return result - return statement_id + # Use the get_sql_results method to return the results of the SQL query, or the statement_ids, + # depending on the value of self.return_sql_result + return self.get_sql_results(statement_id=statement_id, return_sql_result=self.return_sql_result) + + def get_sql_results( + self, statement_id: str, return_sql_result: bool + ) -> list[GetStatementResultResponseTypeDef] | list[str]: + """ + Retrieve either the result of the SQL query, or the statement ID(s). + + :param statement_id: Statement ID of the running queries + :param return_sql_result: Boolean, true if results should be returned + """ + # ISSUE-40427: Pull the statement, and check to see if there are sub-statements. If that is the + # case, pull each of the sub-statement ID's, and grab the results. Otherwise, just use statement_id + statement: DescribeStatementResponseTypeDef = self.hook.conn.describe_statement(Id=statement_id) + statement_ids: list[str] = ( + [sub_statement["Id"] for sub_statement in statement["SubStatements"]] + if len(statement.get("SubStatements", [])) > 0 + else [statement_id] + ) + + # If returning the SQL result, use get_statement_result to return the records for each query + if return_sql_result: + results: list = [self.hook.conn.get_statement_result(Id=sid) for sid in statement_ids] + self.log.debug("Statement result(s): %s", results) + return results + else: + return statement_ids def on_kill(self) -> None: """Cancel the submitted redshift query.""" - if self.statement_id: + if hasattr(self, "statement_id"): self.log.info("Received a kill signal.") self.log.info("Stopping Query with statementId - %s", self.statement_id) diff --git a/providers/tests/amazon/aws/operators/test_redshift_data.py b/providers/tests/amazon/aws/operators/test_redshift_data.py index d367fc3ca9596..da9a486df43af 100644 --- a/providers/tests/amazon/aws/operators/test_redshift_data.py +++ b/providers/tests/amazon/aws/operators/test_redshift_data.py @@ -88,7 +88,8 @@ def test_init(self): assert op.hook._config is None @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.execute_query") - def test_execute(self, mock_exec_query): + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") + def test_execute(self, mock_conn, mock_exec_query): cluster_identifier = "cluster_identifier" workgroup_name = None db_user = "db_user" @@ -96,8 +97,9 @@ def test_execute(self, mock_exec_query): statement_name = "statement_name" parameters = [{"name": "id", "value": "1"}] poll_interval = 5 - wait_for_completion = True + mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID} + mock_conn.describe_statement.return_value = {"Status": "FINISHED"} mock_exec_query.return_value = QueryExecutionOutput(statement_id=STATEMENT_ID, session_id=None) operator = RedshiftDataOperator( @@ -113,8 +115,11 @@ def test_execute(self, mock_exec_query): wait_for_completion=True, poll_interval=poll_interval, ) + + # Mock the TaskInstance, call the execute method mock_ti = mock.MagicMock(name="MockedTaskInstance") - operator.execute({"ti": mock_ti}) + actual_result = operator.execute({"ti": mock_ti}) + mock_exec_query.assert_called_once_with( sql=SQL, database=DATABASE, @@ -125,16 +130,19 @@ def test_execute(self, mock_exec_query): statement_name=statement_name, parameters=parameters, with_event=False, - wait_for_completion=wait_for_completion, + wait_for_completion=True, # Matches above poll_interval=poll_interval, session_id=None, session_keep_alive_seconds=None, ) + # Check that the result returned is a list of the statement_id's + assert actual_result == [STATEMENT_ID] mock_ti.xcom_push.assert_not_called() @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.execute_query") - def test_execute_with_workgroup_name(self, mock_exec_query): + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") + def test_execute_with_workgroup_name(self, mock_conn, mock_exec_query): cluster_identifier = None workgroup_name = "workgroup_name" db_user = "db_user" @@ -142,7 +150,11 @@ def test_execute_with_workgroup_name(self, mock_exec_query): statement_name = "statement_name" parameters = [{"name": "id", "value": "1"}] poll_interval = 5 - wait_for_completion = True + + # Like before, return a statement ID and a status + mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID} + mock_conn.describe_statement.return_value = {"Status": "FINISHED"} + mock_exec_query.return_value = QueryExecutionOutput(statement_id=STATEMENT_ID, session_id=None) operator = RedshiftDataOperator( aws_conn_id=CONN_ID, @@ -157,26 +169,32 @@ def test_execute_with_workgroup_name(self, mock_exec_query): wait_for_completion=True, poll_interval=poll_interval, ) + + # Mock the TaskInstance, call the execute method mock_ti = mock.MagicMock(name="MockedTaskInstance") - operator.execute({"ti": mock_ti}) + actual_result = operator.execute({"ti": mock_ti}) + + # Assertions + assert actual_result == [STATEMENT_ID] mock_exec_query.assert_called_once_with( sql=SQL, database=DATABASE, cluster_identifier=cluster_identifier, - workgroup_name=workgroup_name, + workgroup_name=workgroup_name, # Called with workgroup_name db_user=db_user, secret_arn=secret_arn, statement_name=statement_name, parameters=parameters, with_event=False, - wait_for_completion=wait_for_completion, + wait_for_completion=True, poll_interval=poll_interval, session_id=None, session_keep_alive_seconds=None, ) @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.execute_query") - def test_execute_new_session(self, mock_exec_query): + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") + def test_execute_new_session(self, mock_conn, mock_exec_query): cluster_identifier = "cluster_identifier" workgroup_name = None db_user = "db_user" @@ -186,6 +204,9 @@ def test_execute_new_session(self, mock_exec_query): poll_interval = 5 wait_for_completion = True + # Like before, return a statement ID and a status + mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID} + mock_conn.describe_statement.return_value = {"Status": "FINISHED"} mock_exec_query.return_value = QueryExecutionOutput(statement_id=STATEMENT_ID, session_id=SESSION_ID) operator = RedshiftDataOperator( @@ -203,8 +224,10 @@ def test_execute_new_session(self, mock_exec_query): session_keep_alive_seconds=123, ) + # Mock the TaskInstance and call the execute method mock_ti = mock.MagicMock(name="MockedTaskInstance") operator.execute({"ti": mock_ti}) + mock_exec_query.assert_called_once_with( sql=SQL, database=DATABASE, @@ -256,14 +279,17 @@ def test_on_kill_with_query(self, mock_conn): @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") def test_return_sql_result(self, mock_conn): - expected_result = {"Result": True} - mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID, "SessionId": SESSION_ID} - mock_conn.describe_statement.return_value = {"Status": "FINISHED"} - mock_conn.get_statement_result.return_value = expected_result + expected_result = [{"Result": True}] cluster_identifier = "cluster_identifier" db_user = "db_user" secret_arn = "secret_arn" statement_name = "statement_name" + + # Mock the conn object + mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID, "SessionId": SESSION_ID} + mock_conn.describe_statement.return_value = {"Status": "FINISHED"} + mock_conn.get_statement_result.return_value = {"Result": True} + operator = RedshiftDataOperator( task_id=TASK_ID, cluster_identifier=cluster_identifier, @@ -275,8 +301,11 @@ def test_return_sql_result(self, mock_conn): aws_conn_id=CONN_ID, return_sql_result=True, ) + + # Mock the TaskInstance, run the execute method mock_ti = mock.MagicMock(name="MockedTaskInstance") actual_result = operator.execute({"ti": mock_ti}) + assert actual_result == expected_result mock_conn.execute_statement.assert_called_once_with( Database=DATABASE, @@ -287,17 +316,18 @@ def test_return_sql_result(self, mock_conn): StatementName=statement_name, WithEvent=False, ) - mock_conn.get_statement_result.assert_called_once_with( - Id=STATEMENT_ID, - ) + mock_conn.get_statement_result.assert_called_once_with(Id=STATEMENT_ID) + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") @mock.patch("airflow.providers.amazon.aws.operators.redshift_data.RedshiftDataOperator.defer") @mock.patch( "airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.check_query_is_finished", return_value=True, ) @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.execute_query") - def test_execute_finished_before_defer(self, mock_exec_query, check_query_is_finished, mock_defer): + def test_execute_finished_before_defer( + self, mock_exec_query, check_query_is_finished, mock_defer, mock_conn + ): cluster_identifier = "cluster_identifier" workgroup_name = None db_user = "db_user" @@ -366,25 +396,26 @@ def test_execute_complete_exception(self, deferrable_operator): deferrable_operator.execute_complete(context=None, event=None) assert exc.value.args[0] == "Trigger error: event is None" - def test_execute_complete(self, deferrable_operator): + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") + def test_execute_complete(self, mock_conn, deferrable_operator): """Asserts that logging occurs as expected""" deferrable_operator.statement_id = "uuid" with mock.patch.object(deferrable_operator.log, "info") as mock_log_info: - assert ( - deferrable_operator.execute_complete( - context=None, - event={"status": "success", "message": "Job completed", "statement_id": "uuid"}, - ) - == "uuid" - ) + assert deferrable_operator.execute_complete( + context=None, + event={"status": "success", "message": "Job completed", "statement_id": "uuid"}, + ) == ["uuid"] mock_log_info.assert_called_with("%s completed successfully.", TASK_ID) @mock.patch("airflow.providers.amazon.aws.operators.redshift_data.RedshiftDataOperator.defer") @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.check_query_is_finished") + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.execute_query") - def test_no_wait_for_completion(self, mock_exec_query, mock_check_query_is_finished, mock_defer): + def test_no_wait_for_completion( + self, mock_exec_query, mock_conn, mock_check_query_is_finished, mock_defer + ): """Tests that the operator does not check for completion nor defers when wait_for_completion is False, no matter the value of deferrable""" cluster_identifier = "cluster_identifier" @@ -393,9 +424,12 @@ def test_no_wait_for_completion(self, mock_exec_query, mock_check_query_is_finis statement_name = "statement_name" parameters = [{"name": "id", "value": "1"}] poll_interval = 5 - wait_for_completion = False + # Mock the describe_statement call + mock_conn.describe_statement.return_value = {"Status": "FINISHED"} + mock_exec_query.return_value = QueryExecutionOutput(statement_id=STATEMENT_ID, session_id=SESSION_ID) + for deferrable in [True, False]: operator = RedshiftDataOperator( aws_conn_id=CONN_ID, @@ -411,11 +445,14 @@ def test_no_wait_for_completion(self, mock_exec_query, mock_check_query_is_finis poll_interval=poll_interval, deferrable=deferrable, ) + + # Mock the TaskInstance, call the execute method mock_ti = mock.MagicMock(name="MockedTaskInstance") - operator.execute({"ti": mock_ti}) + actual_results = operator.execute({"ti": mock_ti}) assert not mock_check_query_is_finished.called assert not mock_defer.called + assert actual_results == [STATEMENT_ID] def test_template_fields(self): operator = RedshiftDataOperator( diff --git a/providers/tests/system/amazon/aws/example_redshift.py b/providers/tests/system/amazon/aws/example_redshift.py index 146a20da451ba..1d8784e4176cd 100644 --- a/providers/tests/system/amazon/aws/example_redshift.py +++ b/providers/tests/system/amazon/aws/example_redshift.py @@ -146,13 +146,15 @@ cluster_identifier=redshift_cluster_identifier, database=DB_NAME, db_user=DB_LOGIN, - sql=""" + sql=[ + """ CREATE TABLE IF NOT EXISTS fruit ( fruit_id INTEGER, name VARCHAR NOT NULL, color VARCHAR NOT NULL ); - """, + """ + ], poll_interval=POLL_INTERVAL, wait_for_completion=True, ) @@ -163,14 +165,14 @@ cluster_identifier=redshift_cluster_identifier, database=DB_NAME, db_user=DB_LOGIN, - sql=""" - INSERT INTO fruit VALUES ( 1, 'Banana', 'Yellow'); - INSERT INTO fruit VALUES ( 2, 'Apple', 'Red'); - INSERT INTO fruit VALUES ( 3, 'Lemon', 'Yellow'); - INSERT INTO fruit VALUES ( 4, 'Grape', 'Purple'); - INSERT INTO fruit VALUES ( 5, 'Pear', 'Green'); - INSERT INTO fruit VALUES ( 6, 'Strawberry', 'Red'); - """, + sql=[ + "INSERT INTO fruit VALUES ( 1, 'Banana', 'Yellow');", + "INSERT INTO fruit VALUES ( 2, 'Apple', 'Red');", + "INSERT INTO fruit VALUES ( 3, 'Lemon', 'Yellow');", + "INSERT INTO fruit VALUES ( 4, 'Grape', 'Purple');", + "INSERT INTO fruit VALUES ( 5, 'Pear', 'Green');", + "INSERT INTO fruit VALUES ( 6, 'Strawberry', 'Red');", + ], poll_interval=POLL_INTERVAL, wait_for_completion=True, ) @@ -181,13 +183,15 @@ cluster_identifier=redshift_cluster_identifier, database=DB_NAME, db_user=DB_LOGIN, - sql=""" + sql=[ + """ CREATE TEMPORARY TABLE tmp_people ( id INTEGER, first_name VARCHAR(100), age INTEGER ); - """, + """ + ], poll_interval=POLL_INTERVAL, wait_for_completion=True, session_keep_alive_seconds=600, @@ -195,11 +199,11 @@ insert_data_reuse_session = RedshiftDataOperator( task_id="insert_data_reuse_session", - sql=""" - INSERT INTO tmp_people VALUES ( 1, 'Bob', 30); - INSERT INTO tmp_people VALUES ( 2, 'Alice', 35); - INSERT INTO tmp_people VALUES ( 3, 'Charlie', 40); - """, + sql=[ + "INSERT INTO tmp_people VALUES ( 1, 'Bob', 30);", + "INSERT INTO tmp_people VALUES ( 2, 'Alice', 35);", + "INSERT INTO tmp_people VALUES ( 3, 'Charlie', 40);", + ], poll_interval=POLL_INTERVAL, wait_for_completion=True, session_id="{{ task_instance.xcom_pull(task_ids='create_tmp_table_data_api', key='session_id') }}", From c3227005af201aab4944933f98d3a5c2358440d8 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Tue, 15 Oct 2024 17:26:27 +0100 Subject: [PATCH 125/199] Bump `uv` to `0.4.21` in other places (#43033) Follow-up of #43032: Bumps uv to 0.4.21 (https://pypi.org/project/uv/0.4.21/) --- dev/breeze/doc/ci/02_images.md | 58 +++++++++---------- .../commands/release_management_commands.py | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 8c699e43b5dbb..96b61920d837f 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -421,36 +421,36 @@ DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ The following build arguments (`--build-arg` in docker build command) can be used for CI images: -| Build argument | Default value | Description | -|-----------------------------------|-------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python image | -| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor version of Python (should match base image) | -| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | -| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | -| `PIP_NO_CACHE_DIR` | `true` | if true, then no pip cache will be stored | -| `UV_NO_CACHE` | `true` | if true, then no uv cache will be stored | -| `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | -| `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | -| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | -| `AIRFLOW_REPO` | `apache/airflow` | the repository from which PIP dependencies are pre-installed | -| `AIRFLOW_BRANCH` | `main` | the branch from which PIP dependencies are pre-installed | -| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing this value will reinstall PIP dependencies from the repository from scratch | -| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not empty, it will override the source of the constraints with the specified URL or file. | +| Build argument | Default value | Description | +|-----------------------------------|-------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python image | +| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor version of Python (should match base image) | +| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | +| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | +| `PIP_NO_CACHE_DIR` | `true` | if true, then no pip cache will be stored | +| `UV_NO_CACHE` | `true` | if true, then no uv cache will be stored | +| `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | +| `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | +| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | +| `AIRFLOW_REPO` | `apache/airflow` | the repository from which PIP dependencies are pre-installed | +| `AIRFLOW_BRANCH` | `main` | the branch from which PIP dependencies are pre-installed | +| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing this value will reinstall PIP dependencies from the repository from scratch | +| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not empty, it will override the source of the constraints with the specified URL or file. | | `AIRFLOW_CONSTRAINTS_REFERENCE` | | reference (branch or tag) from GitHub repository from which constraints are used. By default it is set to `constraints-main` but can be `constraints-2-X`. | -| `AIRFLOW_EXTRAS` | `all` | extras to install | -| `UPGRADE_INVALIDATION_STRING` | | If set to any random value the dependencies are upgraded to newer versions. In CI it is set to build id. | -| `AIRFLOW_PRE_CACHED_PIP_PACKAGES` | `true` | Allows to pre-cache airflow PIP packages from the GitHub of Apache Airflow This allows to optimize iterations for Image builds and speeds up CI jobs. | -| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional extras to install | -| `ADDITIONAL_PYTHON_DEPS` | | additional Python dependencies to install | -| `DEV_APT_COMMAND` | | Dev apt command executed before dev deps are installed in the first part of image | -| `ADDITIONAL_DEV_APT_COMMAND` | | Additional Dev apt command executed before dev dep are installed in the first part of the image | -| `DEV_APT_DEPS` | Empty - install default dependencies (see `install_os_dependencies.sh`) | Dev APT dependencies installed in the first part of the image | -| `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | -| `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | -| `AIRFLOW_PIP_VERSION` | `24.0` | PIP version used. | -| `AIRFLOW_UV_VERSION` | `0.1.10` | UV version used. | -| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | -| `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | +| `AIRFLOW_EXTRAS` | `all` | extras to install | +| `UPGRADE_INVALIDATION_STRING` | | If set to any random value the dependencies are upgraded to newer versions. In CI it is set to build id. | +| `AIRFLOW_PRE_CACHED_PIP_PACKAGES` | `true` | Allows to pre-cache airflow PIP packages from the GitHub of Apache Airflow This allows to optimize iterations for Image builds and speeds up CI jobs. | +| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional extras to install | +| `ADDITIONAL_PYTHON_DEPS` | | additional Python dependencies to install | +| `DEV_APT_COMMAND` | | Dev apt command executed before dev deps are installed in the first part of image | +| `ADDITIONAL_DEV_APT_COMMAND` | | Additional Dev apt command executed before dev dep are installed in the first part of the image | +| `DEV_APT_DEPS` | Empty - install default dependencies (see `install_os_dependencies.sh`) | Dev APT dependencies installed in the first part of the image | +| `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | +| `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | +| `AIRFLOW_PIP_VERSION` | `24.0` | PIP version used. | +| `AIRFLOW_UV_VERSION` | `0.4.21` | UV version used. | +| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | +| `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | Here are some examples of how CI images can built manually. CI is always built from local sources. diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index a56947f026f04..cbc83fb043ad3 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -230,7 +230,7 @@ class VersionedFile(NamedTuple): AIRFLOW_PIP_VERSION = "24.0" -AIRFLOW_UV_VERSION = "0.1.10" +AIRFLOW_UV_VERSION = "0.4.21" AIRFLOW_USE_UV = False WHEEL_VERSION = "0.36.2" GITPYTHON_VERSION = "3.1.40" From d8c1edc9bb4f4c83199a76e2e7cfdf9111fd21e3 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Tue, 15 Oct 2024 18:48:17 +0200 Subject: [PATCH 126/199] require 1.2.1 common.compat for openlineage provider (#43039) Signed-off-by: Maciej Obuchowski --- generated/provider_dependencies.json | 2 +- providers/src/airflow/providers/openlineage/provider.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index e025266eebf81..571cd29f1a71b 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -965,7 +965,7 @@ }, "openlineage": { "deps": [ - "apache-airflow-providers-common-compat>=1.2.0", + "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow-providers-common-sql>=1.6.0", "apache-airflow>=2.8.0", "attrs>=22.2", diff --git a/providers/src/airflow/providers/openlineage/provider.yaml b/providers/src/airflow/providers/openlineage/provider.yaml index fadaf9c1e07a4..92783f6309155 100644 --- a/providers/src/airflow/providers/openlineage/provider.yaml +++ b/providers/src/airflow/providers/openlineage/provider.yaml @@ -51,7 +51,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - apache-airflow-providers-common-sql>=1.6.0 - - apache-airflow-providers-common-compat>=1.2.0 + - apache-airflow-providers-common-compat>=1.2.1 - attrs>=22.2 - openlineage-integration-common>=1.22.0 - openlineage-python>=1.22.0 From ae5985eaa70d6a898d22c5f370efe6ee2b1dc0a4 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Tue, 15 Oct 2024 20:05:19 +0100 Subject: [PATCH 127/199] Detect system color theme and add active state to nav button (#43041) * Detect system color theme and add active state to nav button * Clean up navbutton props --- airflow/ui/src/layouts/Nav/Nav.tsx | 13 +++++++++++-- airflow/ui/src/layouts/Nav/NavButton.tsx | 18 ++++++++++-------- airflow/ui/src/theme.ts | 1 + 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/airflow/ui/src/layouts/Nav/Nav.tsx b/airflow/ui/src/layouts/Nav/Nav.tsx index 3b47595e8c173..84ab23497cf17 100644 --- a/airflow/ui/src/layouts/Nav/Nav.tsx +++ b/airflow/ui/src/layouts/Nav/Nav.tsx @@ -68,7 +68,12 @@ export const Nav = () => { > - } isDisabled title="Home" /> + } + isDisabled + title="Home" + to="/" + /> } title="DAGs" @@ -78,29 +83,33 @@ export const Nav = () => { icon={} isDisabled title="Assets" + to="assets" /> } isDisabled title="DAG Runs" + to="dag_runs" /> } isDisabled title="Browse" + to="browse" /> } isDisabled title="Admin" + to="admin" /> } title="Return to legacy UI" + to={import.meta.env.VITE_LEGACY_API_URL} /> diff --git a/airflow/ui/src/layouts/Nav/NavButton.tsx b/airflow/ui/src/layouts/Nav/NavButton.tsx index 1839049b4f7f0..6fc46bf886a21 100644 --- a/airflow/ui/src/layouts/Nav/NavButton.tsx +++ b/airflow/ui/src/layouts/Nav/NavButton.tsx @@ -18,21 +18,23 @@ */ import { Box, Button, type ButtonProps } from "@chakra-ui/react"; import type { ReactElement } from "react"; -import { Link as RouterLink } from "react-router-dom"; +import { NavLink } from "react-router-dom"; import { navButtonProps } from "./navButtonProps"; type NavButtonProps = { - readonly href?: string; readonly icon: ReactElement; - readonly target?: string; readonly title?: string; - readonly to?: string; + readonly to: string; } & ButtonProps; export const NavButton = ({ icon, title, to, ...rest }: NavButtonProps) => ( - + + {({ isActive }: { readonly isActive: boolean }) => ( + + )} + ); diff --git a/airflow/ui/src/theme.ts b/airflow/ui/src/theme.ts index 06a3b10cc7fcf..0c7146643d55c 100644 --- a/airflow/ui/src/theme.ts +++ b/airflow/ui/src/theme.ts @@ -64,6 +64,7 @@ const theme = extendTheme({ }, }, config: { + initialColorMode: "system", useSystemColorMode: true, }, semanticTokens: { From 2640fe2d818f4093b63633025bf13d45e8bc4c86 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Tue, 15 Oct 2024 20:05:39 +0100 Subject: [PATCH 128/199] Migrate health info to fastapi (#42938) * Add health info to public fastapi * Add test and fix TaskInstanceState name --- .../endpoints/health_endpoint.py | 2 + airflow/api_fastapi/openapi/v1-generated.yaml | 103 ++++++++++++- airflow/api_fastapi/serializers/dashboard.py | 4 +- airflow/api_fastapi/serializers/monitor.py | 52 +++++++ airflow/api_fastapi/views/public/__init__.py | 2 + airflow/api_fastapi/views/public/monitor.py | 30 ++++ airflow/ui/openapi-gen/queries/common.ts | 13 ++ airflow/ui/openapi-gen/queries/prefetch.ts | 11 ++ airflow/ui/openapi-gen/queries/queries.ts | 19 +++ airflow/ui/openapi-gen/queries/suspense.ts | 19 +++ .../ui/openapi-gen/requests/schemas.gen.ts | 140 +++++++++++++++++- .../ui/openapi-gen/requests/services.gen.ts | 15 ++ airflow/ui/openapi-gen/requests/types.gen.ts | 57 ++++++- .../api_fastapi/views/public/test_monitor.py | 104 +++++++++++++ 14 files changed, 561 insertions(+), 10 deletions(-) create mode 100644 airflow/api_fastapi/serializers/monitor.py create mode 100644 airflow/api_fastapi/views/public/monitor.py create mode 100644 tests/api_fastapi/views/public/test_monitor.py diff --git a/airflow/api_connexion/endpoints/health_endpoint.py b/airflow/api_connexion/endpoints/health_endpoint.py index bdcf3bd93a1e1..12f2548c0e128 100644 --- a/airflow/api_connexion/endpoints/health_endpoint.py +++ b/airflow/api_connexion/endpoints/health_endpoint.py @@ -20,11 +20,13 @@ from airflow.api.common.airflow_health import get_airflow_health from airflow.api_connexion.schemas.health_schema import health_schema +from airflow.utils.api_migration import mark_fastapi_migration_done if TYPE_CHECKING: from airflow.api_connexion.types import APIResponse +@mark_fastapi_migration_done def get_health() -> APIResponse: """Return the health of the airflow scheduler, metadatabase and triggerer.""" airflow_health_status = get_airflow_health() diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/openapi/v1-generated.yaml index d9339fdb56cff..4b028c315f32e 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/openapi/v1-generated.yaml @@ -944,8 +944,33 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/monitor/health: + get: + tags: + - Monitor + summary: Get Health + operationId: get_health + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/HealthInfoSchema' components: schemas: + BaseInfoSchema: + properties: + status: + anyOf: + - type: string + - type: 'null' + title: Status + type: object + required: + - status + title: BaseInfoSchema + description: Base status field for metadatabase and scheduler. ConnectionCollectionResponse: properties: connections: @@ -1553,6 +1578,24 @@ components: - dataset_triggered title: DAGRunTypes description: DAG Run Types for responses. + DagProcessorInfoSchema: + properties: + status: + anyOf: + - type: string + - type: 'null' + title: Status + latest_dag_processor_heartbeat: + anyOf: + - type: string + - type: 'null' + title: Latest Dag Processor Heartbeat + type: object + required: + - status + - latest_dag_processor_heartbeat + title: DagProcessorInfoSchema + description: Schema for DagProcessor info. DagRunState: type: string enum: @@ -1627,6 +1670,24 @@ components: title: Detail type: object title: HTTPValidationError + HealthInfoSchema: + properties: + metadatabase: + $ref: '#/components/schemas/BaseInfoSchema' + scheduler: + $ref: '#/components/schemas/SchedulerInfoSchema' + triggerer: + $ref: '#/components/schemas/TriggererInfoSchema' + dag_processor: + $ref: '#/components/schemas/DagProcessorInfoSchema' + type: object + required: + - metadatabase + - scheduler + - triggerer + - dag_processor + title: HealthInfoSchema + description: Schema for the Health endpoint. HistoricalMetricDataResponse: properties: dag_run_types: @@ -1634,7 +1695,7 @@ components: dag_run_states: $ref: '#/components/schemas/DAGRunStates' task_instance_states: - $ref: '#/components/schemas/TaskInstantState' + $ref: '#/components/schemas/TaskInstanceState' type: object required: - dag_run_types @@ -1642,7 +1703,25 @@ components: - task_instance_states title: HistoricalMetricDataResponse description: Historical Metric Data serializer for responses. - TaskInstantState: + SchedulerInfoSchema: + properties: + status: + anyOf: + - type: string + - type: 'null' + title: Status + latest_scheduler_heartbeat: + anyOf: + - type: string + - type: 'null' + title: Latest Scheduler Heartbeat + type: object + required: + - status + - latest_scheduler_heartbeat + title: SchedulerInfoSchema + description: Schema for Scheduler info. + TaskInstanceState: properties: no_status: type: integer @@ -1698,8 +1777,26 @@ components: - upstream_failed - skipped - deferred - title: TaskInstantState + title: TaskInstanceState description: TaskInstance serializer for responses. + TriggererInfoSchema: + properties: + status: + anyOf: + - type: string + - type: 'null' + title: Status + latest_triggerer_heartbeat: + anyOf: + - type: string + - type: 'null' + title: Latest Triggerer Heartbeat + type: object + required: + - status + - latest_triggerer_heartbeat + title: TriggererInfoSchema + description: Schema for Triggerer info. ValidationError: properties: loc: diff --git a/airflow/api_fastapi/serializers/dashboard.py b/airflow/api_fastapi/serializers/dashboard.py index f5a38fa22ea87..ee31a812945ec 100644 --- a/airflow/api_fastapi/serializers/dashboard.py +++ b/airflow/api_fastapi/serializers/dashboard.py @@ -37,7 +37,7 @@ class DAGRunStates(BaseModel): failed: int -class TaskInstantState(BaseModel): +class TaskInstanceState(BaseModel): """TaskInstance serializer for responses.""" no_status: int @@ -60,4 +60,4 @@ class HistoricalMetricDataResponse(BaseModel): dag_run_types: DAGRunTypes dag_run_states: DAGRunStates - task_instance_states: TaskInstantState + task_instance_states: TaskInstanceState diff --git a/airflow/api_fastapi/serializers/monitor.py b/airflow/api_fastapi/serializers/monitor.py new file mode 100644 index 0000000000000..0734321a45fd5 --- /dev/null +++ b/airflow/api_fastapi/serializers/monitor.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pydantic import BaseModel + + +class BaseInfoSchema(BaseModel): + """Base status field for metadatabase and scheduler.""" + + status: str | None + + +class SchedulerInfoSchema(BaseInfoSchema): + """Schema for Scheduler info.""" + + latest_scheduler_heartbeat: str | None + + +class TriggererInfoSchema(BaseInfoSchema): + """Schema for Triggerer info.""" + + latest_triggerer_heartbeat: str | None + + +class DagProcessorInfoSchema(BaseInfoSchema): + """Schema for DagProcessor info.""" + + latest_dag_processor_heartbeat: str | None + + +class HealthInfoSchema(BaseModel): + """Schema for the Health endpoint.""" + + metadatabase: BaseInfoSchema + scheduler: SchedulerInfoSchema + triggerer: TriggererInfoSchema + dag_processor: DagProcessorInfoSchema diff --git a/airflow/api_fastapi/views/public/__init__.py b/airflow/api_fastapi/views/public/__init__.py index 9d90a0966802c..de0a0d0bb4a55 100644 --- a/airflow/api_fastapi/views/public/__init__.py +++ b/airflow/api_fastapi/views/public/__init__.py @@ -20,6 +20,7 @@ from airflow.api_fastapi.views.public.connections import connections_router from airflow.api_fastapi.views.public.dag_run import dag_run_router from airflow.api_fastapi.views.public.dags import dags_router +from airflow.api_fastapi.views.public.monitor import monitor_router from airflow.api_fastapi.views.public.variables import variables_router from airflow.api_fastapi.views.router import AirflowRouter @@ -30,3 +31,4 @@ public_router.include_router(connections_router) public_router.include_router(variables_router) public_router.include_router(dag_run_router) +public_router.include_router(monitor_router) diff --git a/airflow/api_fastapi/views/public/monitor.py b/airflow/api_fastapi/views/public/monitor.py new file mode 100644 index 0000000000000..07916f6e06d53 --- /dev/null +++ b/airflow/api_fastapi/views/public/monitor.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from airflow.api.common.airflow_health import get_airflow_health +from airflow.api_fastapi.serializers.monitor import HealthInfoSchema +from airflow.api_fastapi.views.router import AirflowRouter + +monitor_router = AirflowRouter(tags=["Monitor"], prefix="/monitor") + + +@monitor_router.get("/health") +async def get_health() -> HealthInfoSchema: + airflow_health_status = get_airflow_health() + return HealthInfoSchema.model_validate(airflow_health_status) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 30a20826109c6..2a05104c8fa70 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -7,6 +7,7 @@ import { DagRunService, DagService, DashboardService, + MonitorService, VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; @@ -209,6 +210,18 @@ export const UseDagRunServiceGetDagRunKeyFn = ( }, queryKey?: Array, ) => [useDagRunServiceGetDagRunKey, ...(queryKey ?? [{ dagId, dagRunId }])]; +export type MonitorServiceGetHealthDefaultResponse = Awaited< + ReturnType +>; +export type MonitorServiceGetHealthQueryResult< + TData = MonitorServiceGetHealthDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useMonitorServiceGetHealthKey = "MonitorServiceGetHealth"; +export const UseMonitorServiceGetHealthKeyFn = (queryKey?: Array) => [ + useMonitorServiceGetHealthKey, + ...(queryKey ?? []), +]; export type VariableServicePostVariableMutationResult = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index 79b310107d27c..28145dc536470 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -7,6 +7,7 @@ import { DagRunService, DagService, DashboardService, + MonitorService, VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; @@ -259,3 +260,13 @@ export const prefetchUseDagRunServiceGetDagRun = ( queryKey: Common.UseDagRunServiceGetDagRunKeyFn({ dagId, dagRunId }), queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }), }); +/** + * Get Health + * @returns HealthInfoSchema Successful Response + * @throws ApiError + */ +export const prefetchUseMonitorServiceGetHealth = (queryClient: QueryClient) => + queryClient.prefetchQuery({ + queryKey: Common.UseMonitorServiceGetHealthKeyFn(), + queryFn: () => MonitorService.getHealth(), + }); diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 1e9586e436018..7009a7dee7afe 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -12,6 +12,7 @@ import { DagRunService, DagService, DashboardService, + MonitorService, VariableService, } from "../requests/services.gen"; import { DAGPatchBody, DagRunState, VariableBody } from "../requests/types.gen"; @@ -331,6 +332,24 @@ export const useDagRunServiceGetDagRun = < queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, ...options, }); +/** + * Get Health + * @returns HealthInfoSchema Successful Response + * @throws ApiError + */ +export const useMonitorServiceGetHealth = < + TData = Common.MonitorServiceGetHealthDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseMonitorServiceGetHealthKeyFn(queryKey), + queryFn: () => MonitorService.getHealth() as TData, + ...options, + }); /** * Post Variable * Create a variable. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index af2a571871eb1..51e35f321a81c 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -7,6 +7,7 @@ import { DagRunService, DagService, DashboardService, + MonitorService, VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; @@ -326,3 +327,21 @@ export const useDagRunServiceGetDagRunSuspense = < queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, ...options, }); +/** + * Get Health + * @returns HealthInfoSchema Successful Response + * @throws ApiError + */ +export const useMonitorServiceGetHealthSuspense = < + TData = Common.MonitorServiceGetHealthDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseMonitorServiceGetHealthKeyFn(queryKey), + queryFn: () => MonitorService.getHealth() as TData, + ...options, + }); diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 44a29d3899d2a..981c8f659a3dd 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -1,5 +1,25 @@ // This file is auto-generated by @hey-api/openapi-ts +export const $BaseInfoSchema = { + properties: { + status: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Status", + }, + }, + type: "object", + required: ["status"], + title: "BaseInfoSchema", + description: "Base status field for metadatabase and scheduler.", +} as const; + export const $ConnectionCollectionResponse = { properties: { connections: { @@ -993,6 +1013,37 @@ export const $DAGRunTypes = { description: "DAG Run Types for responses.", } as const; +export const $DagProcessorInfoSchema = { + properties: { + status: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Status", + }, + latest_dag_processor_heartbeat: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Latest Dag Processor Heartbeat", + }, + }, + type: "object", + required: ["status", "latest_dag_processor_heartbeat"], + title: "DagProcessorInfoSchema", + description: "Schema for DagProcessor info.", +} as const; + export const $DagRunState = { type: "string", enum: ["queued", "running", "success", "failed"], @@ -1079,6 +1130,27 @@ export const $HTTPValidationError = { title: "HTTPValidationError", } as const; +export const $HealthInfoSchema = { + properties: { + metadatabase: { + $ref: "#/components/schemas/BaseInfoSchema", + }, + scheduler: { + $ref: "#/components/schemas/SchedulerInfoSchema", + }, + triggerer: { + $ref: "#/components/schemas/TriggererInfoSchema", + }, + dag_processor: { + $ref: "#/components/schemas/DagProcessorInfoSchema", + }, + }, + type: "object", + required: ["metadatabase", "scheduler", "triggerer", "dag_processor"], + title: "HealthInfoSchema", + description: "Schema for the Health endpoint.", +} as const; + export const $HistoricalMetricDataResponse = { properties: { dag_run_types: { @@ -1088,7 +1160,7 @@ export const $HistoricalMetricDataResponse = { $ref: "#/components/schemas/DAGRunStates", }, task_instance_states: { - $ref: "#/components/schemas/TaskInstantState", + $ref: "#/components/schemas/TaskInstanceState", }, }, type: "object", @@ -1097,7 +1169,38 @@ export const $HistoricalMetricDataResponse = { description: "Historical Metric Data serializer for responses.", } as const; -export const $TaskInstantState = { +export const $SchedulerInfoSchema = { + properties: { + status: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Status", + }, + latest_scheduler_heartbeat: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Latest Scheduler Heartbeat", + }, + }, + type: "object", + required: ["status", "latest_scheduler_heartbeat"], + title: "SchedulerInfoSchema", + description: "Schema for Scheduler info.", +} as const; + +export const $TaskInstanceState = { properties: { no_status: { type: "integer", @@ -1168,10 +1271,41 @@ export const $TaskInstantState = { "skipped", "deferred", ], - title: "TaskInstantState", + title: "TaskInstanceState", description: "TaskInstance serializer for responses.", } as const; +export const $TriggererInfoSchema = { + properties: { + status: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Status", + }, + latest_triggerer_heartbeat: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Latest Triggerer Heartbeat", + }, + }, + type: "object", + required: ["status", "latest_triggerer_heartbeat"], + title: "TriggererInfoSchema", + description: "Schema for Triggerer info.", +} as const; + export const $ValidationError = { properties: { loc: { diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 6f9b0aa5cb8a1..7516a625d71b0 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -37,6 +37,7 @@ import type { GetDagRunResponse, DeleteDagRunData, DeleteDagRunResponse, + GetHealthResponse, } from "./types.gen"; export class AssetService { @@ -546,3 +547,17 @@ export class DagRunService { }); } } + +export class MonitorService { + /** + * Get Health + * @returns HealthInfoSchema Successful Response + * @throws ApiError + */ + public static getHealth(): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/monitor/health", + }); + } +} diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 2fae25c7660c4..5f5ce6da71e3c 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1,5 +1,12 @@ // This file is auto-generated by @hey-api/openapi-ts +/** + * Base status field for metadatabase and scheduler. + */ +export type BaseInfoSchema = { + status: string | null; +}; + /** * DAG Collection serializer for responses. */ @@ -170,6 +177,14 @@ export type DAGRunTypes = { dataset_triggered: number; }; +/** + * Schema for DagProcessor info. + */ +export type DagProcessorInfoSchema = { + status: string | null; + latest_dag_processor_heartbeat: string | null; +}; + /** * All possible states that a DagRun can be in. * @@ -224,19 +239,37 @@ export type HTTPValidationError = { detail?: Array; }; +/** + * Schema for the Health endpoint. + */ +export type HealthInfoSchema = { + metadatabase: BaseInfoSchema; + scheduler: SchedulerInfoSchema; + triggerer: TriggererInfoSchema; + dag_processor: DagProcessorInfoSchema; +}; + /** * Historical Metric Data serializer for responses. */ export type HistoricalMetricDataResponse = { dag_run_types: DAGRunTypes; dag_run_states: DAGRunStates; - task_instance_states: TaskInstantState; + task_instance_states: TaskInstanceState; +}; + +/** + * Schema for Scheduler info. + */ +export type SchedulerInfoSchema = { + status: string | null; + latest_scheduler_heartbeat: string | null; }; /** * TaskInstance serializer for responses. */ -export type TaskInstantState = { +export type TaskInstanceState = { no_status: number; removed: number; scheduled: number; @@ -252,6 +285,14 @@ export type TaskInstantState = { deferred: number; }; +/** + * Schema for Triggerer info. + */ +export type TriggererInfoSchema = { + status: string | null; + latest_triggerer_heartbeat: string | null; +}; + export type ValidationError = { loc: Array; msg: string; @@ -407,6 +448,8 @@ export type DeleteDagRunData = { export type DeleteDagRunResponse = void; +export type GetHealthResponse = HealthInfoSchema; + export type $OpenApiTs = { "/ui/next_run_assets/{dag_id}": { get: { @@ -847,4 +890,14 @@ export type $OpenApiTs = { }; }; }; + "/public/monitor/health": { + get: { + res: { + /** + * Successful Response + */ + 200: HealthInfoSchema; + }; + }; + }; }; diff --git a/tests/api_fastapi/views/public/test_monitor.py b/tests/api_fastapi/views/public/test_monitor.py new file mode 100644 index 0000000000000..e37047dc906c6 --- /dev/null +++ b/tests/api_fastapi/views/public/test_monitor.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import timedelta +from unittest import mock + +import pytest + +from airflow.jobs.job import Job +from airflow.jobs.scheduler_job_runner import SchedulerJobRunner +from airflow.utils import timezone +from airflow.utils.session import provide_session +from airflow.utils.state import State +from tests_common.test_utils.db import clear_db_jobs + +HEALTHY = "healthy" +UNHEALTHY = "unhealthy" + +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] + + +class TestMonitorEndpoint: + @pytest.fixture(autouse=True) + def setup(self) -> None: + clear_db_jobs() + + def teardown_method(self): + clear_db_jobs() + + +class TestGetHealth(TestMonitorEndpoint): + @provide_session + def test_healthy_scheduler_status(self, test_client, session): + last_scheduler_heartbeat_for_testing_1 = timezone.utcnow() + job = Job(state=State.RUNNING, latest_heartbeat=last_scheduler_heartbeat_for_testing_1) + SchedulerJobRunner(job=job) + session.add(job) + session.commit() + response = test_client.get("/public/monitor/health") + + assert response.status_code == 200 + body = response.json() + + assert "healthy" == body["metadatabase"]["status"] + assert "healthy" == body["scheduler"]["status"] + assert ( + last_scheduler_heartbeat_for_testing_1.isoformat() + == body["scheduler"]["latest_scheduler_heartbeat"] + ) + + @provide_session + def test_unhealthy_scheduler_is_slow(self, test_client, session): + last_scheduler_heartbeat_for_testing_2 = timezone.utcnow() - timedelta(minutes=1) + job = Job(state=State.RUNNING, latest_heartbeat=last_scheduler_heartbeat_for_testing_2) + SchedulerJobRunner(job=job) + session.add(job) + session.commit() + response = test_client.get("/public/monitor/health") + + assert response.status_code == 200 + body = response.json() + + assert "healthy" == body["metadatabase"]["status"] + assert "unhealthy" == body["scheduler"]["status"] + assert ( + last_scheduler_heartbeat_for_testing_2.isoformat() + == body["scheduler"]["latest_scheduler_heartbeat"] + ) + + def test_unhealthy_scheduler_no_job(self, test_client): + response = test_client.get("/public/monitor/health") + + assert response.status_code == 200 + body = response.json() + + assert "healthy" == body["metadatabase"]["status"] + assert "unhealthy" == body["scheduler"]["status"] + assert body["scheduler"]["latest_scheduler_heartbeat"] is None + + @mock.patch.object(SchedulerJobRunner, "most_recent_job") + def test_unhealthy_metadatabase_status(self, most_recent_job_mock, test_client): + most_recent_job_mock.side_effect = Exception + response = test_client.get("/public/monitor/health") + + assert response.status_code == 200 + body = response.json() + + assert "unhealthy" == body["metadatabase"]["status"] + assert body["scheduler"]["latest_scheduler_heartbeat"] is None From 915169d501955caff7b900ac15ac179f6214ddf4 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Wed, 16 Oct 2024 00:36:11 +0530 Subject: [PATCH 129/199] Add upperbound to microsoft-kiota-abstractions (#43021) * add upperbound to microsoft-kiota-abstractions * add * add comment about breaking change --- generated/provider_dependencies.json | 1 + providers/src/airflow/providers/microsoft/azure/provider.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 571cd29f1a71b..02323df136bda 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -828,6 +828,7 @@ "azure-storage-file-share>=12.7.0", "azure-synapse-artifacts>=0.17.0", "azure-synapse-spark>=0.2.0", + "microsoft-kiota-abstractions<1.4.0", "microsoft-kiota-http>=1.3.0,!=1.3.4", "microsoft-kiota-serialization-json==1.0.0", "microsoft-kiota-serialization-text==1.0.0", diff --git a/providers/src/airflow/providers/microsoft/azure/provider.yaml b/providers/src/airflow/providers/microsoft/azure/provider.yaml index c4831a641b333..b1839b10fd0ce 100644 --- a/providers/src/airflow/providers/microsoft/azure/provider.yaml +++ b/providers/src/airflow/providers/microsoft/azure/provider.yaml @@ -113,6 +113,9 @@ dependencies: - microsoft-kiota-http>=1.3.0,!=1.3.4 - microsoft-kiota-serialization-json==1.0.0 - microsoft-kiota-serialization-text==1.0.0 + # microsoft-kiota-abstractions 1.4.0 breaks MyPy static checks on main + # see https://github.com/apache/airflow/issues/43036 + - microsoft-kiota-abstractions<1.4.0 devel-dependencies: - pywinrm From ac9aa621326aa14a0750e61e23f60f195c4fedf3 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Tue, 15 Oct 2024 21:39:22 +0200 Subject: [PATCH 130/199] fix path to providers dir for mount sources breeze flag (#43042) --- scripts/ci/docker-compose/providers-and-tests-sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/docker-compose/providers-and-tests-sources.yml b/scripts/ci/docker-compose/providers-and-tests-sources.yml index e792d783dce15..29498ae27eb94 100644 --- a/scripts/ci/docker-compose/providers-and-tests-sources.yml +++ b/scripts/ci/docker-compose/providers-and-tests-sources.yml @@ -31,7 +31,7 @@ services: # but keep tests - ../../../tests/:/opt/airflow/tests:cached # and providers - - ../../../airflow/providers:/opt/airflow/airflow/providers:cached + - ../../../providers/src/airflow/providers:/opt/airflow/airflow/providers:cached # and entrypoint and in_container scripts for testing - ../../../scripts/docker/entrypoint_ci.sh:/entrypoint - ../../../scripts/in_container/:/opt/airflow/scripts/in_container From 17e4e353f59e25e99d4a3f1bde68294ef4540eb7 Mon Sep 17 00:00:00 2001 From: Christian Yarros Date: Tue, 15 Oct 2024 15:42:02 -0400 Subject: [PATCH 131/199] Create Operators for Google Cloud Vertex AI Context Caching (#43008) * Fix merge conflicts * Fix documentation. * Update return variables. --- .../operators/cloud/vertex_ai.rst | 20 +++ generated/provider_dependencies.json | 2 +- .../cloud/hooks/vertex_ai/generative_model.py | 79 +++++++++ .../operators/vertex_ai/generative_model.py | 156 +++++++++++++++++- .../airflow/providers/google/provider.yaml | 2 +- .../hooks/vertex_ai/test_generative_model.py | 60 ++++++- .../vertex_ai/test_generative_model.py | 84 +++++++++- .../example_vertex_ai_generative_model.py | 48 +++++- 8 files changed, 442 insertions(+), 9 deletions(-) diff --git a/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst b/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst index b6bc94620ff2d..f24ab3b06f719 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst @@ -645,6 +645,26 @@ The operator returns the evaluation summary metrics in :ref:`XCom ` under ``return_value`` key. + +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py + :language: python + :dedent: 4 + :start-after: [START how_to_cloud_vertex_ai_create_cached_content_operator] + :end-before: [END how_to_cloud_vertex_ai_create_cached_content_operator] + +To generate a response from cached content you can use +:class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.GenerateFromCachedContentOperator`. +The operator returns the cached content response in :ref:`XCom ` under ``return_value`` key. + +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py + :language: python + :dedent: 4 + :start-after: [START how_to_cloud_vertex_ai_generate_from_cached_content_operator] + :end-before: [END how_to_cloud_vertex_ai_generate_from_cached_content_operator] + Reference ^^^^^^^^^ diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 02323df136bda..ac587bad765be 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -640,7 +640,7 @@ "google-api-python-client>=2.0.2", "google-auth-httplib2>=0.0.1", "google-auth>=2.29.0", - "google-cloud-aiplatform>=1.63.0", + "google-cloud-aiplatform>=1.70.0", "google-cloud-automl>=2.12.0", "google-cloud-batch>=0.13.0", "google-cloud-bigquery-datatransfer>=3.13.0", diff --git a/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py index 04242306f8739..27037baaafdad 100644 --- a/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +++ b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py @@ -20,12 +20,15 @@ from __future__ import annotations import time +from datetime import timedelta from typing import TYPE_CHECKING, Sequence import vertexai from vertexai.generative_models import GenerativeModel, Part from vertexai.language_models import TextEmbeddingModel, TextGenerationModel +from vertexai.preview.caching import CachedContent from vertexai.preview.evaluation import EvalResult, EvalTask +from vertexai.preview.generative_models import GenerativeModel as preview_generative_model from vertexai.preview.tuning import sft from airflow.exceptions import AirflowProviderDeprecationWarning @@ -95,6 +98,16 @@ def get_eval_task( ) return eval_task + def get_cached_context_model( + self, + cached_content_name: str, + ) -> preview_generative_model: + """Return a Generative Model with Cached Context.""" + cached_content = CachedContent(cached_content_name=cached_content_name) + + cached_context_model = preview_generative_model.from_cached_content(cached_content) + return cached_context_model + @deprecated( planned_removal_date="January 01, 2025", use_instead="Part objects included in contents parameter of " @@ -528,3 +541,69 @@ def run_evaluation( ) return eval_result + + def create_cached_content( + self, + model_name: str, + location: str, + ttl_hours: float = 1, + system_instruction: str | None = None, + contents: list | None = None, + display_name: str | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> str: + """ + Create CachedContent to reduce the cost of requests that contain repeat content with high input token counts. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param model_name: Required. The name of the publisher model to use for cached content. + :param system_instruction: Developer set system instruction. + :param contents: The content to cache. + :param ttl_hours: The TTL for this resource in hours. The expiration time is computed: now + TTL. + Defaults to one hour. + :param display_name: The user-generated meaningful display name of the cached content + """ + vertexai.init(project=project_id, location=location, credentials=self.get_credentials()) + + response = CachedContent.create( + model_name=model_name, + system_instruction=system_instruction, + contents=contents, + ttl=timedelta(hours=ttl_hours), + display_name=display_name, + ) + + return response.name + + def generate_from_cached_content( + self, + location: str, + cached_content_name: str, + contents: list, + generation_config: dict | None = None, + safety_settings: dict | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> str: + """ + Generate a response from CachedContent. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param cached_content_name: Required. The name of the cached content resource. + :param contents: Required. The multi-part content of a message that a user or a program + gives to the generative model, in order to elicit a specific response. + :param generation_config: Optional. Generation configuration settings. + :param safety_settings: Optional. Per request settings for blocking unsafe content. + """ + vertexai.init(project=project_id, location=location, credentials=self.get_credentials()) + + cached_context_model = self.get_cached_context_model(cached_content_name=cached_content_name) + + response = cached_context_model.generate_content( + contents=contents, + generation_config=generation_config, + safety_settings=safety_settings, + ) + + return response.text diff --git a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/generative_model.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/generative_model.py index fddd5dcf72861..05a4abf138073 100644 --- a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +++ b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/generative_model.py @@ -21,8 +21,6 @@ from typing import TYPE_CHECKING, Sequence -from google.cloud.aiplatform_v1beta1 import types as types_v1beta1 - from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.vertex_ai.generative_model import GenerativeModelHook from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator @@ -742,8 +740,6 @@ def execute(self, context: Context): self.xcom_push(context, key="total_tokens", value=response.total_tokens) self.xcom_push(context, key="total_billable_characters", value=response.total_billable_characters) - return types_v1beta1.CountTokensResponse.to_dict(response) - class RunEvaluationOperator(GoogleCloudBaseOperator): """ @@ -842,3 +838,155 @@ def execute(self, context: Context): ) return response.summary_metrics + + +class CreateCachedContentOperator(GoogleCloudBaseOperator): + """ + Create CachedContent to reduce the cost of requests that contain repeat content with high input token counts. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param model_name: Required. The name of the publisher model to use for cached content. + :param system_instruction: Developer set system instruction. + :param contents: The content to cache. + :param ttl_hours: The TTL for this resource in hours. The expiration time is computed: now + TTL. + Defaults to one hour. + :param display_name: The user-generated meaningful display name of the cached content + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account (templated). + """ + + template_fields = ( + "location", + "project_id", + "impersonation_chain", + "model_name", + "contents", + "system_instruction", + ) + + def __init__( + self, + *, + project_id: str, + location: str, + model_name: str, + system_instruction: str | None = None, + contents: list | None = None, + ttl_hours: float = 1, + display_name: str | None = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + + self.project_id = project_id + self.location = location + self.model_name = model_name + self.system_instruction = system_instruction + self.contents = contents + self.ttl_hours = ttl_hours + self.display_name = display_name + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + def execute(self, context: Context): + self.hook = GenerativeModelHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + cached_content_name = self.hook.create_cached_content( + project_id=self.project_id, + location=self.location, + model_name=self.model_name, + system_instruction=self.system_instruction, + contents=self.contents, + ttl_hours=self.ttl_hours, + display_name=self.display_name, + ) + + self.log.info("Cached Content Name: %s", cached_content_name) + + return cached_content_name + + +class GenerateFromCachedContentOperator(GoogleCloudBaseOperator): + """ + Generate a response from CachedContent. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param cached_content_name: Required. The name of the cached content resource. + :param contents: Required. The multi-part content of a message that a user or a program + gives to the generative model, in order to elicit a specific response. + :param generation_config: Optional. Generation configuration settings. + :param safety_settings: Optional. Per request settings for blocking unsafe content. + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account (templated). + """ + + template_fields = ( + "location", + "project_id", + "impersonation_chain", + "cached_content_name", + "contents", + ) + + def __init__( + self, + *, + project_id: str, + location: str, + cached_content_name: str, + contents: list, + generation_config: dict | None = None, + safety_settings: dict | None = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + + self.project_id = project_id + self.location = location + self.cached_content_name = cached_content_name + self.contents = contents + self.generation_config = generation_config + self.safety_settings = safety_settings + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + def execute(self, context: Context): + self.hook = GenerativeModelHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + cached_content_text = self.hook.generate_from_cached_content( + project_id=self.project_id, + location=self.location, + cached_content_name=self.cached_content_name, + contents=self.contents, + generation_config=self.generation_config, + safety_settings=self.safety_settings, + ) + + self.log.info("Cached Content Response: %s", cached_content_text) + + return cached_content_text diff --git a/providers/src/airflow/providers/google/provider.yaml b/providers/src/airflow/providers/google/provider.yaml index 86a98a6962757..0d616a53e12f5 100644 --- a/providers/src/airflow/providers/google/provider.yaml +++ b/providers/src/airflow/providers/google/provider.yaml @@ -114,7 +114,7 @@ dependencies: - google-api-python-client>=2.0.2 - google-auth>=2.29.0 - google-auth-httplib2>=0.0.1 - - google-cloud-aiplatform>=1.63.0 + - google-cloud-aiplatform>=1.70.0 - google-cloud-automl>=2.12.0 # Excluded versions contain bug https://github.com/apache/airflow/issues/39541 which is resolved in 3.24.0 - google-cloud-bigquery>=3.4.0,!=3.21.*,!=3.22.0,!=3.23.* diff --git a/providers/tests/google/cloud/hooks/vertex_ai/test_generative_model.py b/providers/tests/google/cloud/hooks/vertex_ai/test_generative_model.py index 52a8c417c6ac2..35d3fc9256e6c 100644 --- a/providers/tests/google/cloud/hooks/vertex_ai/test_generative_model.py +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_generative_model.py @@ -27,7 +27,9 @@ # For no Pydantic environment, we need to skip the tests pytest.importorskip("google.cloud.aiplatform_v1") -from vertexai.generative_models import HarmBlockThreshold, HarmCategory, Tool, grounding +from datetime import timedelta + +from vertexai.generative_models import HarmBlockThreshold, HarmCategory, Part, Tool, grounding from vertexai.preview.evaluation import MetricPromptTemplateExamples from airflow.providers.google.cloud.hooks.vertex_ai.generative_model import ( @@ -106,6 +108,27 @@ TEST_EXPERIMENT_RUN_NAME = "eval-experiment-airflow-operator-run" TEST_PROMPT_TEMPLATE = "{instruction}. Article: {context}. Summary:" +TEST_CACHED_CONTENT_NAME = "test-example-cache" +TEST_CACHED_CONTENT_PROMPT = ["What are these papers about?"] +TEST_CACHED_MODEL = "gemini-1.5-pro-002" +TEST_CACHED_SYSTEM_INSTRUCTION = """ +You are an expert researcher. You always stick to the facts in the sources provided, and never make up new facts. +Now look at these research papers, and answer the following questions. +""" + +TEST_CACHED_CONTENTS = [ + Part.from_uri( + "gs://cloud-samples-data/generative-ai/pdf/2312.11805v3.pdf", + mime_type="application/pdf", + ), + Part.from_uri( + "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf", + mime_type="application/pdf", + ), +] +TEST_CACHED_TTL = 1 +TEST_CACHED_DISPLAY_NAME = "test-example-cache" + BASE_STRING = "airflow.providers.google.common.hooks.base_google.{}" GENERATIVE_MODEL_STRING = "airflow.providers.google.cloud.hooks.vertex_ai.generative_model.{}" @@ -299,3 +322,38 @@ def test_run_evaluation(self, mock_eval_task, mock_model) -> None: prompt_template=TEST_PROMPT_TEMPLATE, experiment_run_name=TEST_EXPERIMENT_RUN_NAME, ) + + @mock.patch("vertexai.preview.caching.CachedContent.create") + def test_create_cached_content(self, mock_cached_content_create) -> None: + self.hook.create_cached_content( + project_id=GCP_PROJECT, + location=GCP_LOCATION, + model_name=TEST_CACHED_MODEL, + system_instruction=TEST_CACHED_SYSTEM_INSTRUCTION, + contents=TEST_CACHED_CONTENTS, + ttl_hours=TEST_CACHED_TTL, + display_name=TEST_CACHED_DISPLAY_NAME, + ) + + mock_cached_content_create.assert_called_once_with( + model_name=TEST_CACHED_MODEL, + system_instruction=TEST_CACHED_SYSTEM_INSTRUCTION, + contents=TEST_CACHED_CONTENTS, + ttl=timedelta(hours=TEST_CACHED_TTL), + display_name=TEST_CACHED_DISPLAY_NAME, + ) + + @mock.patch(GENERATIVE_MODEL_STRING.format("GenerativeModelHook.get_cached_context_model")) + def test_generate_from_cached_content(self, mock_cached_context_model) -> None: + self.hook.generate_from_cached_content( + project_id=GCP_PROJECT, + location=GCP_LOCATION, + cached_content_name=TEST_CACHED_CONTENT_NAME, + contents=TEST_CACHED_CONTENT_PROMPT, + ) + + mock_cached_context_model.return_value.generate_content.assert_called_once_with( + contents=TEST_CACHED_CONTENT_PROMPT, + generation_config=None, + safety_settings=None, + ) diff --git a/providers/tests/google/cloud/operators/vertex_ai/test_generative_model.py b/providers/tests/google/cloud/operators/vertex_ai/test_generative_model.py index e8efb9601fce0..5bdb04cb3edb3 100644 --- a/providers/tests/google/cloud/operators/vertex_ai/test_generative_model.py +++ b/providers/tests/google/cloud/operators/vertex_ai/test_generative_model.py @@ -28,11 +28,13 @@ pytest.importorskip("google.cloud.aiplatform_v1") pytest.importorskip("google.cloud.aiplatform_v1beta1") vertexai = pytest.importorskip("vertexai.generative_models") -from vertexai.generative_models import HarmBlockThreshold, HarmCategory, Tool, grounding +from vertexai.generative_models import HarmBlockThreshold, HarmCategory, Part, Tool, grounding from vertexai.preview.evaluation import MetricPromptTemplateExamples from airflow.providers.google.cloud.operators.vertex_ai.generative_model import ( CountTokensOperator, + CreateCachedContentOperator, + GenerateFromCachedContentOperator, GenerateTextEmbeddingsOperator, GenerativeModelGenerateContentOperator, PromptLanguageModelOperator, @@ -540,3 +542,83 @@ def test_execute( safety_settings=safety_settings, tools=tools, ) + + +class TestVertexAICreateCachedContentOperator: + @mock.patch(VERTEX_AI_PATH.format("generative_model.GenerativeModelHook")) + def test_execute(self, mock_hook): + model_name = "gemini-1.5-pro-002" + system_instruction = """ + You are an expert researcher. You always stick to the facts in the sources provided, and never make up new facts. + Now look at these research papers, and answer the following questions. + """ + + contents = [ + Part.from_uri( + "gs://cloud-samples-data/generative-ai/pdf/2312.11805v3.pdf", + mime_type="application/pdf", + ), + Part.from_uri( + "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf", + mime_type="application/pdf", + ), + ] + ttl_hours = 1 + display_name = "test-example-cache" + + op = CreateCachedContentOperator( + task_id=TASK_ID, + project_id=GCP_PROJECT, + location=GCP_LOCATION, + model_name=model_name, + system_instruction=system_instruction, + contents=contents, + ttl_hours=ttl_hours, + display_name=display_name, + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + op.execute(context={"ti": mock.MagicMock()}) + mock_hook.assert_called_once_with( + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + mock_hook.return_value.create_cached_content.assert_called_once_with( + project_id=GCP_PROJECT, + location=GCP_LOCATION, + model_name=model_name, + system_instruction=system_instruction, + contents=contents, + ttl_hours=ttl_hours, + display_name=display_name, + ) + + +class TestVertexAIGenerateFromCachedContentOperator: + @mock.patch(VERTEX_AI_PATH.format("generative_model.GenerativeModelHook")) + def test_execute(self, mock_hook): + cached_content_name = "test" + contents = ["what are in these papers"] + + op = GenerateFromCachedContentOperator( + task_id=TASK_ID, + project_id=GCP_PROJECT, + location=GCP_LOCATION, + cached_content_name=cached_content_name, + contents=contents, + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + op.execute(context={"ti": mock.MagicMock()}) + mock_hook.assert_called_once_with( + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + mock_hook.return_value.generate_from_cached_content.assert_called_once_with( + project_id=GCP_PROJECT, + location=GCP_LOCATION, + cached_content_name=cached_content_name, + contents=contents, + generation_config=None, + safety_settings=None, + ) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py index a1140afc1983f..4384626999d0a 100644 --- a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_generative_model.py @@ -25,12 +25,14 @@ import os from datetime import datetime -from vertexai.generative_models import HarmBlockThreshold, HarmCategory, Tool, grounding +from vertexai.generative_models import HarmBlockThreshold, HarmCategory, Part, Tool, grounding from vertexai.preview.evaluation import MetricPromptTemplateExamples from airflow.models.dag import DAG from airflow.providers.google.cloud.operators.vertex_ai.generative_model import ( CountTokensOperator, + CreateCachedContentOperator, + GenerateFromCachedContentOperator, GenerativeModelGenerateContentOperator, RunEvaluationOperator, TextEmbeddingModelGetEmbeddingsOperator, @@ -90,6 +92,23 @@ EXPERIMENT_RUN_NAME = "eval-experiment-airflow-operator-run" PROMPT_TEMPLATE = "{instruction}. Article: {context}. Summary:" +CACHED_MODEL = "gemini-1.5-pro-002" +CACHED_SYSTEM_INSTRUCTION = """ +You are an expert researcher. You always stick to the facts in the sources provided, and never make up new facts. +Now look at these research papers, and answer the following questions. +""" + +CACHED_CONTENTS = [ + Part.from_uri( + "gs://cloud-samples-data/generative-ai/pdf/2312.11805v3.pdf", + mime_type="application/pdf", + ), + Part.from_uri( + "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf", + mime_type="application/pdf", + ), +] + with DAG( dag_id=DAG_ID, description="Sample DAG with generative models.", @@ -155,6 +174,33 @@ ) # [END how_to_cloud_vertex_ai_run_evaluation_operator] + # [START how_to_cloud_vertex_ai_create_cached_content_operator] + create_cached_content_task = CreateCachedContentOperator( + task_id="create_cached_content_task", + project_id=PROJECT_ID, + location=REGION, + model_name=CACHED_MODEL, + system_instruction=CACHED_SYSTEM_INSTRUCTION, + contents=CACHED_CONTENTS, + ttl_hours=1, + display_name="example-cache", + ) + # [END how_to_cloud_vertex_ai_create_cached_content_operator] + + # [START how_to_cloud_vertex_ai_generate_from_cached_content_operator] + generate_from_cached_content_task = GenerateFromCachedContentOperator( + task_id="generate_from_cached_content_task", + project_id=PROJECT_ID, + location=REGION, + cached_content_name="{{ task_instance.xcom_pull(task_ids='create_cached_content_task', key='return_value') }}", + contents=["What are the papers about?"], + generation_config=GENERATION_CONFIG, + safety_settings=SAFETY_SETTINGS, + ) + # [END how_to_cloud_vertex_ai_generate_from_cached_content_operator] + + create_cached_content_task >> generate_from_cached_content_task + from tests_common.test_utils.watcher import watcher # This test needs watcher in order to properly mark success/failure From 988250638387781acd53b59b350d011ad7936c7a Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Wed, 16 Oct 2024 02:34:24 +0530 Subject: [PATCH 132/199] add lowerbount to requests-toolbelt and replace requests_toolbelt with requests-toolbelt (#43020) --- docs/apache-airflow-providers-http/index.rst | 2 +- generated/provider_dependencies.json | 4 ++-- providers/src/airflow/providers/http/provider.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/apache-airflow-providers-http/index.rst b/docs/apache-airflow-providers-http/index.rst index 2e7bdf6b78d88..1e3b271b29ef5 100644 --- a/docs/apache-airflow-providers-http/index.rst +++ b/docs/apache-airflow-providers-http/index.rst @@ -102,7 +102,7 @@ PIP package Version required ===================== ================== ``apache-airflow`` ``>=2.8.0`` ``requests`` ``>=2.27.0,<3`` -``requests_toolbelt`` +``requests-toolbelt`` ``aiohttp`` ``>=3.9.2`` ``asgiref`` ===================== ================== diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index ac587bad765be..fb50939ac7fb8 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -748,8 +748,8 @@ "aiohttp>=3.9.2", "apache-airflow>=2.8.0", "asgiref>=2.3.0", - "requests>=2.27.0,<3", - "requests_toolbelt" + "requests-toolbelt>=0.4.0", + "requests>=2.27.0,<3" ], "devel-deps": [], "plugins": [], diff --git a/providers/src/airflow/providers/http/provider.yaml b/providers/src/airflow/providers/http/provider.yaml index e7a9b9ef94f84..4f56f74ab460d 100644 --- a/providers/src/airflow/providers/http/provider.yaml +++ b/providers/src/airflow/providers/http/provider.yaml @@ -65,7 +65,7 @@ dependencies: # The 2.26.0 release of requests got rid of the chardet LGPL mandatory dependency, allowing us to # release it as a requirement for airflow - requests>=2.27.0,<3 - - requests_toolbelt + - requests-toolbelt>=0.4.0 - aiohttp>=3.9.2 - asgiref>=2.3.0 From 07b96c5ed5fa30329e697b5d2d01f753151e5baa Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Wed, 16 Oct 2024 00:52:34 +0300 Subject: [PATCH 133/199] =?UTF-8?q?=E2=9C=A8=20Allow=20node=5Fselector=20t?= =?UTF-8?q?emplating=20in=20KPO=20(#43051)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/airflow/providers/cncf/kubernetes/operators/pod.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py b/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py index 62af050a50cdd..5f0462904447d 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py @@ -170,7 +170,7 @@ class KubernetesPodOperator(BaseOperator): :param affinity: affinity scheduling rules for the launched pod. :param config_file: The path to the Kubernetes config file. (templated) If not specified, default value is ``~/.kube/config`` - :param node_selector: A dict containing a group of scheduling rules. + :param node_selector: A dict containing a group of scheduling rules. (templated) :param image_pull_secrets: Any image pull secrets to be given to the pod. If more than one secret is required, provide a comma separated list: secret_a,secret_b @@ -258,6 +258,7 @@ class KubernetesPodOperator(BaseOperator): "volume_mounts", "cluster_context", "env_from", + "node_selector", ) template_fields_renderers = {"env_vars": "py"} From efc54ff20ad8b512471bc020310f0175bbfd5e99 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Tue, 15 Oct 2024 23:15:18 +0100 Subject: [PATCH 134/199] Rename views to routes for FastAPI apps (#43057) This is part of the change I want to do for https://github.com/apache/airflow/pull/43015 but going to break it down to keep it manageable. Since these are actual routes for APIs, renaming them to routes instead of views. --- airflow/api_fastapi/app.py | 4 ++-- airflow/api_fastapi/{views => routes}/__init__.py | 0 .../api_fastapi/{views => routes}/public/__init__.py | 12 ++++++------ .../{views => routes}/public/connections.py | 2 +- .../api_fastapi/{views => routes}/public/dag_run.py | 2 +- airflow/api_fastapi/{views => routes}/public/dags.py | 2 +- .../api_fastapi/{views => routes}/public/monitor.py | 2 +- .../{views => routes}/public/variables.py | 2 +- airflow/api_fastapi/{views => routes}/router.py | 0 airflow/api_fastapi/{views => routes}/ui/__init__.py | 6 +++--- airflow/api_fastapi/{views => routes}/ui/assets.py | 2 +- .../api_fastapi/{views => routes}/ui/dashboard.py | 2 +- tests/api_fastapi/{views => routes}/__init__.py | 0 .../api_fastapi/{views => routes}/public/__init__.py | 0 .../{views => routes}/public/test_connections.py | 0 .../{views => routes}/public/test_dag_run.py | 0 .../{views => routes}/public/test_dags.py | 4 ++-- .../{views => routes}/public/test_monitor.py | 0 .../{views => routes}/public/test_variables.py | 0 tests/api_fastapi/{views => routes}/ui/__init__.py | 0 .../api_fastapi/{views => routes}/ui/test_assets.py | 0 .../{views => routes}/ui/test_dashboard.py | 0 22 files changed, 20 insertions(+), 20 deletions(-) rename airflow/api_fastapi/{views => routes}/__init__.py (100%) rename airflow/api_fastapi/{views => routes}/public/__init__.py (72%) rename airflow/api_fastapi/{views => routes}/public/connections.py (98%) rename airflow/api_fastapi/{views => routes}/public/dag_run.py (97%) rename airflow/api_fastapi/{views => routes}/public/dags.py (99%) rename airflow/api_fastapi/{views => routes}/public/monitor.py (95%) rename airflow/api_fastapi/{views => routes}/public/variables.py (98%) rename airflow/api_fastapi/{views => routes}/router.py (100%) rename airflow/api_fastapi/{views => routes}/ui/__init__.py (83%) rename airflow/api_fastapi/{views => routes}/ui/assets.py (98%) rename airflow/api_fastapi/{views => routes}/ui/dashboard.py (98%) rename tests/api_fastapi/{views => routes}/__init__.py (100%) rename tests/api_fastapi/{views => routes}/public/__init__.py (100%) rename tests/api_fastapi/{views => routes}/public/test_connections.py (100%) rename tests/api_fastapi/{views => routes}/public/test_dag_run.py (100%) rename tests/api_fastapi/{views => routes}/public/test_dags.py (98%) rename tests/api_fastapi/{views => routes}/public/test_monitor.py (100%) rename tests/api_fastapi/{views => routes}/public/test_variables.py (100%) rename tests/api_fastapi/{views => routes}/ui/__init__.py (100%) rename tests/api_fastapi/{views => routes}/ui/test_assets.py (100%) rename tests/api_fastapi/{views => routes}/ui/test_dashboard.py (100%) diff --git a/airflow/api_fastapi/app.py b/airflow/api_fastapi/app.py index 42720847638a3..912b8496dd6e8 100644 --- a/airflow/api_fastapi/app.py +++ b/airflow/api_fastapi/app.py @@ -77,8 +77,8 @@ def create_app() -> FastAPI: def init_views(app: FastAPI) -> None: """Init views by registering the different routers.""" - from airflow.api_fastapi.views.public import public_router - from airflow.api_fastapi.views.ui import ui_router + from airflow.api_fastapi.routes.public import public_router + from airflow.api_fastapi.routes.ui import ui_router app.include_router(ui_router) app.include_router(public_router) diff --git a/airflow/api_fastapi/views/__init__.py b/airflow/api_fastapi/routes/__init__.py similarity index 100% rename from airflow/api_fastapi/views/__init__.py rename to airflow/api_fastapi/routes/__init__.py diff --git a/airflow/api_fastapi/views/public/__init__.py b/airflow/api_fastapi/routes/public/__init__.py similarity index 72% rename from airflow/api_fastapi/views/public/__init__.py rename to airflow/api_fastapi/routes/public/__init__.py index de0a0d0bb4a55..01a9b7f239621 100644 --- a/airflow/api_fastapi/views/public/__init__.py +++ b/airflow/api_fastapi/routes/public/__init__.py @@ -17,12 +17,12 @@ from __future__ import annotations -from airflow.api_fastapi.views.public.connections import connections_router -from airflow.api_fastapi.views.public.dag_run import dag_run_router -from airflow.api_fastapi.views.public.dags import dags_router -from airflow.api_fastapi.views.public.monitor import monitor_router -from airflow.api_fastapi.views.public.variables import variables_router -from airflow.api_fastapi.views.router import AirflowRouter +from airflow.api_fastapi.routes.public.connections import connections_router +from airflow.api_fastapi.routes.public.dag_run import dag_run_router +from airflow.api_fastapi.routes.public.dags import dags_router +from airflow.api_fastapi.routes.public.monitor import monitor_router +from airflow.api_fastapi.routes.public.variables import variables_router +from airflow.api_fastapi.routes.router import AirflowRouter public_router = AirflowRouter(prefix="/public") diff --git a/airflow/api_fastapi/views/public/connections.py b/airflow/api_fastapi/routes/public/connections.py similarity index 98% rename from airflow/api_fastapi/views/public/connections.py rename to airflow/api_fastapi/routes/public/connections.py index 6fca43aca26c7..76684835cf481 100644 --- a/airflow/api_fastapi/views/public/connections.py +++ b/airflow/api_fastapi/routes/public/connections.py @@ -24,8 +24,8 @@ from airflow.api_fastapi.db.common import get_session, paginated_select from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc from airflow.api_fastapi.parameters import QueryLimit, QueryOffset, SortParam +from airflow.api_fastapi.routes.router import AirflowRouter from airflow.api_fastapi.serializers.connections import ConnectionCollectionResponse, ConnectionResponse -from airflow.api_fastapi.views.router import AirflowRouter from airflow.models import Connection connections_router = AirflowRouter(tags=["Connection"], prefix="/connections") diff --git a/airflow/api_fastapi/views/public/dag_run.py b/airflow/api_fastapi/routes/public/dag_run.py similarity index 97% rename from airflow/api_fastapi/views/public/dag_run.py rename to airflow/api_fastapi/routes/public/dag_run.py index 2f44f03ea6103..62e4d97a829bf 100644 --- a/airflow/api_fastapi/views/public/dag_run.py +++ b/airflow/api_fastapi/routes/public/dag_run.py @@ -24,8 +24,8 @@ from airflow.api_fastapi.db.common import get_session from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.routes.router import AirflowRouter from airflow.api_fastapi.serializers.dag_run import DAGRunResponse -from airflow.api_fastapi.views.router import AirflowRouter from airflow.models import DagRun dag_run_router = AirflowRouter(tags=["DagRun"], prefix="/dags/{dag_id}/dagRuns") diff --git a/airflow/api_fastapi/views/public/dags.py b/airflow/api_fastapi/routes/public/dags.py similarity index 99% rename from airflow/api_fastapi/views/public/dags.py rename to airflow/api_fastapi/routes/public/dags.py index 46d4d8b3540c0..d2b608e05176f 100644 --- a/airflow/api_fastapi/views/public/dags.py +++ b/airflow/api_fastapi/routes/public/dags.py @@ -42,13 +42,13 @@ QueryTagsFilter, SortParam, ) +from airflow.api_fastapi.routes.router import AirflowRouter from airflow.api_fastapi.serializers.dags import ( DAGCollectionResponse, DAGDetailsResponse, DAGPatchBody, DAGResponse, ) -from airflow.api_fastapi.views.router import AirflowRouter from airflow.exceptions import AirflowException, DagNotFound from airflow.models import DAG, DagModel diff --git a/airflow/api_fastapi/views/public/monitor.py b/airflow/api_fastapi/routes/public/monitor.py similarity index 95% rename from airflow/api_fastapi/views/public/monitor.py rename to airflow/api_fastapi/routes/public/monitor.py index 07916f6e06d53..30e8f1330a58b 100644 --- a/airflow/api_fastapi/views/public/monitor.py +++ b/airflow/api_fastapi/routes/public/monitor.py @@ -18,8 +18,8 @@ from __future__ import annotations from airflow.api.common.airflow_health import get_airflow_health +from airflow.api_fastapi.routes.router import AirflowRouter from airflow.api_fastapi.serializers.monitor import HealthInfoSchema -from airflow.api_fastapi.views.router import AirflowRouter monitor_router = AirflowRouter(tags=["Monitor"], prefix="/monitor") diff --git a/airflow/api_fastapi/views/public/variables.py b/airflow/api_fastapi/routes/public/variables.py similarity index 98% rename from airflow/api_fastapi/views/public/variables.py rename to airflow/api_fastapi/routes/public/variables.py index a61b9bb930023..61cd84c700453 100644 --- a/airflow/api_fastapi/views/public/variables.py +++ b/airflow/api_fastapi/routes/public/variables.py @@ -23,8 +23,8 @@ from airflow.api_fastapi.db.common import get_session from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.routes.router import AirflowRouter from airflow.api_fastapi.serializers.variables import VariableBody, VariableResponse -from airflow.api_fastapi.views.router import AirflowRouter from airflow.models.variable import Variable variables_router = AirflowRouter(tags=["Variable"], prefix="/variables") diff --git a/airflow/api_fastapi/views/router.py b/airflow/api_fastapi/routes/router.py similarity index 100% rename from airflow/api_fastapi/views/router.py rename to airflow/api_fastapi/routes/router.py diff --git a/airflow/api_fastapi/views/ui/__init__.py b/airflow/api_fastapi/routes/ui/__init__.py similarity index 83% rename from airflow/api_fastapi/views/ui/__init__.py rename to airflow/api_fastapi/routes/ui/__init__.py index f01686cc99ee2..2087549c98ea0 100644 --- a/airflow/api_fastapi/views/ui/__init__.py +++ b/airflow/api_fastapi/routes/ui/__init__.py @@ -16,9 +16,9 @@ # under the License. from __future__ import annotations -from airflow.api_fastapi.views.router import AirflowRouter -from airflow.api_fastapi.views.ui.assets import assets_router -from airflow.api_fastapi.views.ui.dashboard import dashboard_router +from airflow.api_fastapi.routes.router import AirflowRouter +from airflow.api_fastapi.routes.ui.assets import assets_router +from airflow.api_fastapi.routes.ui.dashboard import dashboard_router ui_router = AirflowRouter(prefix="/ui") diff --git a/airflow/api_fastapi/views/ui/assets.py b/airflow/api_fastapi/routes/ui/assets.py similarity index 98% rename from airflow/api_fastapi/views/ui/assets.py rename to airflow/api_fastapi/routes/ui/assets.py index 1a198745f11f3..31406b1b94db1 100644 --- a/airflow/api_fastapi/views/ui/assets.py +++ b/airflow/api_fastapi/routes/ui/assets.py @@ -23,7 +23,7 @@ from typing_extensions import Annotated from airflow.api_fastapi.db.common import get_session -from airflow.api_fastapi.views.router import AirflowRouter +from airflow.api_fastapi.routes.router import AirflowRouter from airflow.models import DagModel from airflow.models.asset import AssetDagRunQueue, AssetEvent, AssetModel, DagScheduleAssetReference diff --git a/airflow/api_fastapi/views/ui/dashboard.py b/airflow/api_fastapi/routes/ui/dashboard.py similarity index 98% rename from airflow/api_fastapi/views/ui/dashboard.py rename to airflow/api_fastapi/routes/ui/dashboard.py index 0d6b69a1ced5b..e7cec5fbf0ee1 100644 --- a/airflow/api_fastapi/views/ui/dashboard.py +++ b/airflow/api_fastapi/routes/ui/dashboard.py @@ -33,7 +33,7 @@ if TYPE_CHECKING: from sqlalchemy.orm import Session from airflow.api_fastapi.db.common import get_session -from airflow.api_fastapi.views.router import AirflowRouter +from airflow.api_fastapi.routes.router import AirflowRouter from airflow.utils import timezone dashboard_router = AirflowRouter(tags=["Dashboard"]) diff --git a/tests/api_fastapi/views/__init__.py b/tests/api_fastapi/routes/__init__.py similarity index 100% rename from tests/api_fastapi/views/__init__.py rename to tests/api_fastapi/routes/__init__.py diff --git a/tests/api_fastapi/views/public/__init__.py b/tests/api_fastapi/routes/public/__init__.py similarity index 100% rename from tests/api_fastapi/views/public/__init__.py rename to tests/api_fastapi/routes/public/__init__.py diff --git a/tests/api_fastapi/views/public/test_connections.py b/tests/api_fastapi/routes/public/test_connections.py similarity index 100% rename from tests/api_fastapi/views/public/test_connections.py rename to tests/api_fastapi/routes/public/test_connections.py diff --git a/tests/api_fastapi/views/public/test_dag_run.py b/tests/api_fastapi/routes/public/test_dag_run.py similarity index 100% rename from tests/api_fastapi/views/public/test_dag_run.py rename to tests/api_fastapi/routes/public/test_dag_run.py diff --git a/tests/api_fastapi/views/public/test_dags.py b/tests/api_fastapi/routes/public/test_dags.py similarity index 98% rename from tests/api_fastapi/views/public/test_dags.py rename to tests/api_fastapi/routes/public/test_dags.py index ab0c54f51764f..99ac939406f78 100644 --- a/tests/api_fastapi/views/public/test_dags.py +++ b/tests/api_fastapi/routes/public/test_dags.py @@ -293,7 +293,7 @@ def test_dag_details( "description": None, "doc_md": "details", "end_date": None, - "fileloc": "/opt/airflow/tests/api_fastapi/views/public/test_dags.py", + "fileloc": "/opt/airflow/tests/api_fastapi/routes/public/test_dags.py", "file_token": file_token, "has_import_errors": False, "has_task_concurrency_limits": True, @@ -358,7 +358,7 @@ def test_get_dag(self, test_client, query_params, dag_id, expected_status_code, "dag_id": dag_id, "dag_display_name": dag_display_name, "description": None, - "fileloc": "/opt/airflow/tests/api_fastapi/views/public/test_dags.py", + "fileloc": "/opt/airflow/tests/api_fastapi/routes/public/test_dags.py", "file_token": file_token, "is_paused": False, "is_active": True, diff --git a/tests/api_fastapi/views/public/test_monitor.py b/tests/api_fastapi/routes/public/test_monitor.py similarity index 100% rename from tests/api_fastapi/views/public/test_monitor.py rename to tests/api_fastapi/routes/public/test_monitor.py diff --git a/tests/api_fastapi/views/public/test_variables.py b/tests/api_fastapi/routes/public/test_variables.py similarity index 100% rename from tests/api_fastapi/views/public/test_variables.py rename to tests/api_fastapi/routes/public/test_variables.py diff --git a/tests/api_fastapi/views/ui/__init__.py b/tests/api_fastapi/routes/ui/__init__.py similarity index 100% rename from tests/api_fastapi/views/ui/__init__.py rename to tests/api_fastapi/routes/ui/__init__.py diff --git a/tests/api_fastapi/views/ui/test_assets.py b/tests/api_fastapi/routes/ui/test_assets.py similarity index 100% rename from tests/api_fastapi/views/ui/test_assets.py rename to tests/api_fastapi/routes/ui/test_assets.py diff --git a/tests/api_fastapi/views/ui/test_dashboard.py b/tests/api_fastapi/routes/ui/test_dashboard.py similarity index 100% rename from tests/api_fastapi/views/ui/test_dashboard.py rename to tests/api_fastapi/routes/ui/test_dashboard.py From bf50fb8129e0b8b214305ab80276db0a515e6df1 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Tue, 15 Oct 2024 16:53:11 -0700 Subject: [PATCH 135/199] Follow-up SLA purge (#42808) Co-authored-by: dstandish --- .../endpoints/task_instance_endpoint.py | 96 ++------------- .../api_connexion/schemas/sla_miss_schema.py | 38 ------ .../schemas/task_instance_schema.py | 21 +--- airflow/example_dags/tutorial.py | 1 - airflow/models/__init__.py | 3 - airflow/models/slamiss.py | 46 ------- airflow/notifications/basenotifier.py | 8 +- airflow/utils/context.py | 3 + airflow/www/auth.py | 4 +- airflow/www/extensions/init_views.py | 3 - airflow/www/views.py | 115 +----------------- .../test_task_instance_endpoint.py | 3 +- .../tests/smtp/notifications/test_smtp.py | 33 ----- .../test_mapped_task_instance_endpoint.py | 4 +- .../endpoints/test_task_instance_endpoint.py | 29 +---- .../schemas/test_task_instance_schema.py | 64 +--------- tests/jobs/test_scheduler_job.py | 3 - .../test_dag_import_error_listener.py | 2 - tests/notifications/test_basenotifier.py | 1 - tests_common/test_utils/db.py | 7 -- 20 files changed, 34 insertions(+), 450 deletions(-) delete mode 100644 airflow/api_connexion/schemas/sla_miss_schema.py delete mode 100644 airflow/models/slamiss.py diff --git a/airflow/api_connexion/endpoints/task_instance_endpoint.py b/airflow/api_connexion/endpoints/task_instance_endpoint.py index 23e5f5a6d16f4..0e98173f68a5f 100644 --- a/airflow/api_connexion/endpoints/task_instance_endpoint.py +++ b/airflow/api_connexion/endpoints/task_instance_endpoint.py @@ -20,7 +20,7 @@ from flask import g from marshmallow import ValidationError -from sqlalchemy import and_, or_, select +from sqlalchemy import or_, select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import joinedload @@ -48,7 +48,6 @@ from airflow.api_connexion.security import get_readable_dags from airflow.auth.managers.models.resource_details import DagAccessEntity, DagDetails from airflow.exceptions import TaskNotFound -from airflow.models import SlaMiss from airflow.models.dagrun import DagRun as DR from airflow.models.taskinstance import TaskInstance as TI, clear_task_instances from airflow.models.taskinstancehistory import TaskInstanceHistory as TIH @@ -84,27 +83,18 @@ def get_task_instance( select(TI) .where(TI.dag_id == dag_id, TI.run_id == dag_run_id, TI.task_id == task_id) .join(TI.dag_run) - .outerjoin( - SlaMiss, - and_( - SlaMiss.dag_id == TI.dag_id, - SlaMiss.execution_date == DR.execution_date, - SlaMiss.task_id == TI.task_id, - ), - ) - .add_columns(SlaMiss) .options(joinedload(TI.rendered_task_instance_fields)) ) try: - task_instance = session.execute(query).one_or_none() + task_instance = session.scalar(query) except MultipleResultsFound: raise NotFound( "Task instance not found", detail="Task instance is mapped, add the map_index value to the URL" ) if task_instance is None: raise NotFound("Task instance not found") - if task_instance[0].map_index != -1: + if task_instance.map_index != -1: raise NotFound( "Task instance not found", detail="Task instance is mapped, add the map_index value to the URL" ) @@ -127,18 +117,9 @@ def get_mapped_task_instance( select(TI) .where(TI.dag_id == dag_id, TI.run_id == dag_run_id, TI.task_id == task_id, TI.map_index == map_index) .join(TI.dag_run) - .outerjoin( - SlaMiss, - and_( - SlaMiss.dag_id == TI.dag_id, - SlaMiss.execution_date == DR.execution_date, - SlaMiss.task_id == TI.task_id, - ), - ) - .add_columns(SlaMiss) .options(joinedload(TI.rendered_task_instance_fields)) ) - task_instance = session.execute(query).one_or_none() + task_instance = session.scalar(query) if task_instance is None: raise NotFound("Task instance not found") @@ -232,28 +213,13 @@ def get_mapped_task_instances( # Count elements before joining extra columns total_entries = get_query_count(base_query, session=session) - # Add SLA miss - entry_query = ( - base_query.outerjoin( - SlaMiss, - and_( - SlaMiss.dag_id == TI.dag_id, - SlaMiss.task_id == TI.task_id, - SlaMiss.execution_date == DR.execution_date, - ), - ) - .add_columns(SlaMiss) - .options(joinedload(TI.rendered_task_instance_fields)) - ) - try: order_by_params = _get_order_by_params(order_by) - entry_query = entry_query.order_by(*order_by_params) + entry_query = base_query.order_by(*order_by_params) except _UnsupportedOrderBy as e: raise BadRequest(detail=f"Ordering with {e.order_by!r} is not supported") - # using execute because we want the SlaMiss entity. Scalars don't return None for missing entities - task_instances = session.execute(entry_query.offset(offset).limit(limit)).all() + task_instances = session.scalars(entry_query.offset(offset).limit(limit)) return task_instance_collection_schema.dump( TaskInstanceCollection(task_instances=task_instances, total_entries=total_entries) ) @@ -384,28 +350,13 @@ def get_task_instances( # Count elements before joining extra columns total_entries = get_query_count(base_query, session=session) - # Add join - entry_query = ( - base_query.outerjoin( - SlaMiss, - and_( - SlaMiss.dag_id == TI.dag_id, - SlaMiss.task_id == TI.task_id, - SlaMiss.execution_date == DR.execution_date, - ), - ) - .add_columns(SlaMiss) - .options(joinedload(TI.rendered_task_instance_fields)) - ) - try: order_by_params = _get_order_by_params(order_by) - entry_query = entry_query.order_by(*order_by_params) + entry_query = base_query.order_by(*order_by_params) except _UnsupportedOrderBy as e: raise BadRequest(detail=f"Ordering with {e.order_by!r} is not supported") - # using execute because we want the SlaMiss entity. Scalars don't return None for missing entities - task_instances = session.execute(entry_query.offset(offset).limit(limit)).all() + task_instances = session.scalars(entry_query.offset(offset).limit(limit)) return task_instance_collection_schema.dump( TaskInstanceCollection(task_instances=task_instances, total_entries=total_entries) ) @@ -463,16 +414,7 @@ def get_task_instances_batch(session: Session = NEW_SESSION) -> APIResponse: # Count elements before joining extra columns total_entries = get_query_count(base_query, session=session) - # Add join - base_query = base_query.join( - SlaMiss, - and_( - SlaMiss.dag_id == TI.dag_id, - SlaMiss.task_id == TI.task_id, - SlaMiss.execution_date == DR.execution_date, - ), - isouter=True, - ).add_columns(SlaMiss) + ti_query = base_query.options( joinedload(TI.rendered_task_instance_fields), joinedload(TI.task_instance_note) ) @@ -483,8 +425,7 @@ def get_task_instances_batch(session: Session = NEW_SESSION) -> APIResponse: except _UnsupportedOrderBy as e: raise BadRequest(detail=f"Ordering with {e.order_by!r} is not supported") - # using execute because we want the SlaMiss entity. Scalars don't return None for missing entities - task_instances = session.execute(ti_query).all() + task_instances = session.scalars(ti_query) return task_instance_collection_schema.dump( TaskInstanceCollection(task_instances=task_instances, total_entries=total_entries) @@ -690,15 +631,6 @@ def set_task_instance_note( select(TI) .where(TI.dag_id == dag_id, TI.run_id == dag_run_id, TI.task_id == task_id) .join(TI.dag_run) - .outerjoin( - SlaMiss, - and_( - SlaMiss.dag_id == TI.dag_id, - SlaMiss.execution_date == DR.execution_date, - SlaMiss.task_id == TI.task_id, - ), - ) - .add_columns(SlaMiss) .options(joinedload(TI.rendered_task_instance_fields)) ) if map_index == -1: @@ -707,17 +639,15 @@ def set_task_instance_note( query = query.where(TI.map_index == map_index) try: - result = session.execute(query).one_or_none() + ti = session.scalar(query) except MultipleResultsFound: raise NotFound( "Task instance not found", detail="Task instance is mapped, add the map_index value to the URL" ) - if result is None: + if ti is None: error_message = f"Task Instance not found for dag_id={dag_id}, run_id={dag_run_id}, task_id={task_id}" raise NotFound(error_message) - ti, sla_miss = result - current_user_id = get_auth_manager().get_user_id() if ti.task_instance_note is None: ti.note = (new_note, current_user_id) @@ -725,7 +655,7 @@ def set_task_instance_note( ti.task_instance_note.content = new_note ti.task_instance_note.user_id = current_user_id session.commit() - return task_instance_schema.dump((ti, sla_miss)) + return task_instance_schema.dump(ti) @security.requires_access_dag("GET", DagAccessEntity.TASK_INSTANCE) diff --git a/airflow/api_connexion/schemas/sla_miss_schema.py b/airflow/api_connexion/schemas/sla_miss_schema.py deleted file mode 100644 index 97a462e186d59..0000000000000 --- a/airflow/api_connexion/schemas/sla_miss_schema.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field - -from airflow.models import SlaMiss - - -class SlaMissSchema(SQLAlchemySchema): - """Sla Miss Schema.""" - - class Meta: - """Meta.""" - - model = SlaMiss - - task_id = auto_field(dump_only=True) - dag_id = auto_field(dump_only=True) - execution_date = auto_field(dump_only=True) - email_sent = auto_field(dump_only=True) - timestamp = auto_field(dump_only=True) - description = auto_field(dump_only=True) - notification_sent = auto_field(dump_only=True) diff --git a/airflow/api_connexion/schemas/task_instance_schema.py b/airflow/api_connexion/schemas/task_instance_schema.py index 0c1daf6ce2c7d..f0b8285fdfa0e 100644 --- a/airflow/api_connexion/schemas/task_instance_schema.py +++ b/airflow/api_connexion/schemas/task_instance_schema.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from typing import TYPE_CHECKING, NamedTuple +from typing import NamedTuple from marshmallow import Schema, ValidationError, fields, validate, validates_schema from marshmallow.utils import get_value @@ -26,16 +26,12 @@ from airflow.api_connexion.schemas.common_schema import JsonObjectField from airflow.api_connexion.schemas.enum_schemas import TaskInstanceStateField from airflow.api_connexion.schemas.job_schema import JobSchema -from airflow.api_connexion.schemas.sla_miss_schema import SlaMissSchema from airflow.api_connexion.schemas.trigger_schema import TriggerSchema from airflow.models import TaskInstance from airflow.models.taskinstancehistory import TaskInstanceHistory from airflow.utils.helpers import exactly_one from airflow.utils.state import TaskInstanceState -if TYPE_CHECKING: - from airflow.models import SlaMiss - class TaskInstanceSchema(SQLAlchemySchema): """Task instance schema.""" @@ -69,22 +65,15 @@ class Meta: executor = auto_field() executor_config = auto_field() note = auto_field() - sla_miss = fields.Nested(SlaMissSchema, dump_default=None) rendered_map_index = auto_field() rendered_fields = JsonObjectField(dump_default={}) trigger = fields.Nested(TriggerSchema) triggerer_job = fields.Nested(JobSchema) def get_attribute(self, obj, attr, default): - if attr == "sla_miss": - # Object is a tuple of task_instance and slamiss - # and the get_value expects a dict with key, value - # corresponding to the attr. - slamiss_instance = {"sla_miss": obj[1]} - return get_value(slamiss_instance, attr, default) - elif attr == "rendered_fields": - return get_value(obj[0], "rendered_task_instance_fields.rendered_fields", default) - return get_value(obj[0], attr, default) + if attr == "rendered_fields": + return get_value(obj, "rendered_task_instance_fields.rendered_fields", default) + return get_value(obj, attr, default) class TaskInstanceHistorySchema(SQLAlchemySchema): @@ -122,7 +111,7 @@ class Meta: class TaskInstanceCollection(NamedTuple): """List of task instances with metadata.""" - task_instances: list[tuple[TaskInstance, SlaMiss | None]] + task_instances: list[TaskInstance | None] total_entries: int diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py index 6e27bbcd2e5fe..4e55ab7ff15a0 100644 --- a/airflow/example_dags/tutorial.py +++ b/airflow/example_dags/tutorial.py @@ -55,7 +55,6 @@ # 'priority_weight': 10, # 'end_date': datetime(2016, 1, 1), # 'wait_for_downstream': False, - # 'sla': timedelta(hours=2), # 'execution_timeout': timedelta(seconds=300), # 'on_failure_callback': some_function, # or list of functions # 'on_success_callback': some_other_function, # or list of functions diff --git a/airflow/models/__init__.py b/airflow/models/__init__.py index 375761bc20f52..1a998d60c5c92 100644 --- a/airflow/models/__init__.py +++ b/airflow/models/__init__.py @@ -41,7 +41,6 @@ "Pool", "RenderedTaskInstanceFields", "SkipMixin", - "SlaMiss", "TaskFail", "TaskInstance", "TaskReschedule", @@ -104,7 +103,6 @@ def __getattr__(name): "Pool": "airflow.models.pool", "RenderedTaskInstanceFields": "airflow.models.renderedtifields", "SkipMixin": "airflow.models.skipmixin", - "SlaMiss": "airflow.models.slamiss", "TaskFail": "airflow.models.taskfail", "TaskInstance": "airflow.models.taskinstance", "TaskReschedule": "airflow.models.taskreschedule", @@ -134,7 +132,6 @@ def __getattr__(name): from airflow.models.pool import Pool from airflow.models.renderedtifields import RenderedTaskInstanceFields from airflow.models.skipmixin import SkipMixin - from airflow.models.slamiss import SlaMiss from airflow.models.taskfail import TaskFail from airflow.models.taskinstance import TaskInstance, clear_task_instances from airflow.models.taskinstancehistory import TaskInstanceHistory diff --git a/airflow/models/slamiss.py b/airflow/models/slamiss.py deleted file mode 100644 index 4fb7e53a17bab..0000000000000 --- a/airflow/models/slamiss.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -from sqlalchemy import Boolean, Column, Index, String, Text - -from airflow.models.base import COLLATION_ARGS, ID_LEN, Base -from airflow.utils.sqlalchemy import UtcDateTime - - -class SlaMiss(Base): - """ - Model that stores a history of the SLA that have been missed. - - It is used to keep track of SLA failures over time and to avoid double triggering alert emails. - """ - - __tablename__ = "sla_miss" - - task_id = Column(String(ID_LEN, **COLLATION_ARGS), primary_key=True) - dag_id = Column(String(ID_LEN, **COLLATION_ARGS), primary_key=True) - execution_date = Column(UtcDateTime, primary_key=True) - email_sent = Column(Boolean, default=False) - timestamp = Column(UtcDateTime) - description = Column(Text) - notification_sent = Column(Boolean, default=False) - - __table_args__ = (Index("sm_dag", dag_id, unique=False),) - - def __repr__(self): - return str((self.dag_id, self.task_id, self.execution_date.isoformat())) diff --git a/airflow/notifications/basenotifier.py b/airflow/notifications/basenotifier.py index 91fef87167ccb..11d7ee17fe939 100644 --- a/airflow/notifications/basenotifier.py +++ b/airflow/notifications/basenotifier.py @@ -87,10 +87,6 @@ def __call__(self, *args) -> None: :param context: The airflow context """ - # Currently, there are two ways a callback is invoked - # 1. callback(context) - for on_*_callbacks - # 2. callback(dag, task_list, blocking_task_list, slas, blocking_tis) - for sla_miss_callback - # we have to distinguish between the two calls so that we can prepare the correct context, if len(args) == 1: context = args[0] else: @@ -98,9 +94,9 @@ def __call__(self, *args) -> None: "dag": args[0], "task_list": args[1], "blocking_task_list": args[2], - "slas": args[3], - "blocking_tis": args[4], + "blocking_tis": args[3], } + self._update_context(context) self.render_template_fields(context) try: diff --git a/airflow/utils/context.py b/airflow/utils/context.py index e5d30b1e2d7d2..4f308a6f9e060 100644 --- a/airflow/utils/context.py +++ b/airflow/utils/context.py @@ -418,6 +418,9 @@ def context_merge(context: Context, *args: Any, **kwargs: Any) -> None: :meta private: """ + if not context: + context = Context() + context.update(*args, **kwargs) diff --git a/airflow/www/auth.py b/airflow/www/auth.py index 74f31d135c1aa..d4b8ad619e6e5 100644 --- a/airflow/www/auth.py +++ b/airflow/www/auth.py @@ -49,7 +49,7 @@ IsAuthorizedPoolRequest, IsAuthorizedVariableRequest, ) - from airflow.models import DagRun, Pool, SlaMiss, TaskInstance, Variable + from airflow.models import DagRun, Pool, TaskInstance, Variable from airflow.models.connection import Connection from airflow.models.xcom import BaseXCom @@ -239,7 +239,7 @@ def has_access_dag_entities(method: ResourceMethod, access_entity: DagAccessEnti def has_access_decorator(func: T): @wraps(func) def decorated(*args, **kwargs): - items: set[SlaMiss | BaseXCom | DagRun | TaskInstance] = set(args[1]) + items: set[BaseXCom | DagRun | TaskInstance] = set(args[1]) requests: Sequence[IsAuthorizedDagRequest] = [ { "method": method, diff --git a/airflow/www/extensions/init_views.py b/airflow/www/extensions/init_views.py index a04116a7c7d4d..1b0eddf55420a 100644 --- a/airflow/www/extensions/init_views.py +++ b/airflow/www/extensions/init_views.py @@ -103,9 +103,6 @@ def init_appbuilder_views(app): appbuilder.add_view( views.ConnectionModelView, permissions.RESOURCE_CONNECTION, category=permissions.RESOURCE_ADMIN_MENU ) - appbuilder.add_view( - views.SlaMissModelView, permissions.RESOURCE_SLA_MISS, category=permissions.RESOURCE_BROWSE_MENU - ) appbuilder.add_view( views.PluginView, permissions.RESOURCE_PLUGIN, category=permissions.RESOURCE_ADMIN_MENU ) diff --git a/airflow/www/views.py b/airflow/www/views.py index 23ccae6224960..1751bb33de3a4 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -102,7 +102,7 @@ from airflow.jobs.job import Job from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.jobs.triggerer_job_runner import TriggererJobRunner -from airflow.models import Connection, DagModel, DagTag, Log, SlaMiss, Trigger, XCom +from airflow.models import Connection, DagModel, DagTag, Log, Trigger, XCom from airflow.models.asset import AssetDagRunQueue, AssetEvent, AssetModel, DagScheduleAssetReference from airflow.models.dag import get_asset_triggered_next_run_info from airflow.models.dagrun import RUN_ID_REGEX, DagRun, DagRunType @@ -3876,119 +3876,6 @@ def action_post(self): return action.func(items) -class SlaMissModelView(AirflowModelView): - """View to show SlaMiss table.""" - - route_base = "/slamiss" - - datamodel = AirflowModelView.CustomSQLAInterface(SlaMiss) # type: ignore - - class_permission_name = permissions.RESOURCE_SLA_MISS - method_permission_name = { - "list": "read", - "action_muldelete": "delete", - "action_mulnotificationsent": "edit", - "action_mulnotificationsentfalse": "edit", - "action_mulemailsent": "edit", - "action_mulemailsentfalse": "edit", - } - - base_permissions = [ - permissions.ACTION_CAN_READ, - permissions.ACTION_CAN_ACCESS_MENU, - ] - - list_columns = ["dag_id", "task_id", "execution_date", "email_sent", "notification_sent", "timestamp"] - - label_columns = { - "execution_date": "Logical Date", - } - - add_columns = ["dag_id", "task_id", "execution_date", "email_sent", "notification_sent", "timestamp"] - edit_columns = ["dag_id", "task_id", "execution_date", "email_sent", "notification_sent", "timestamp"] - search_columns = ["dag_id", "task_id", "email_sent", "notification_sent", "timestamp", "execution_date"] - base_order = ("execution_date", "desc") - base_filters = [["dag_id", DagFilter, list]] - - formatters_columns = { - "task_id": wwwutils.task_instance_link, - "execution_date": wwwutils.datetime_f("execution_date"), - "timestamp": wwwutils.datetime_f("timestamp"), - "dag_id": wwwutils.dag_link, - "map_index": wwwutils.format_map_index, - } - - @action("muldelete", "Delete", "Are you sure you want to delete selected records?", single=False) - @auth.has_access_dag_entities("DELETE", DagAccessEntity.SLA_MISS) - def action_muldelete(self, items): - """Multiple delete action.""" - self.datamodel.delete_all(items) - self.update_redirect() - return redirect(self.get_redirect()) - - @action( - "mulnotificationsent", - "Set notification sent to true", - "Are you sure you want to set all these notifications to sent?", - single=False, - ) - @auth.has_access_dag_entities("PUT", DagAccessEntity.SLA_MISS) - def action_mulnotificationsent(self, items: list[SlaMiss]): - return self._set_notification_property(items, "notification_sent", True) - - @action( - "mulnotificationsentfalse", - "Set notification sent to false", - "Are you sure you want to mark these SLA alerts as notification not sent yet?", - single=False, - ) - @auth.has_access_dag_entities("PUT", DagAccessEntity.SLA_MISS) - def action_mulnotificationsentfalse(self, items: list[SlaMiss]): - return self._set_notification_property(items, "notification_sent", False) - - @action( - "mulemailsent", - "Set email sent to true", - "Are you sure you want to mark these SLA alerts as emails were sent?", - single=False, - ) - @auth.has_access_dag_entities("PUT", DagAccessEntity.SLA_MISS) - def action_mulemailsent(self, items: list[SlaMiss]): - return self._set_notification_property(items, "email_sent", True) - - @action( - "mulemailsentfalse", - "Set email sent to false", - "Are you sure you want to mark these SLA alerts as emails not sent yet?", - single=False, - ) - @auth.has_access_dag_entities("PUT", DagAccessEntity.SLA_MISS) - def action_mulemailsentfalse(self, items: list[SlaMiss]): - return self._set_notification_property(items, "email_sent", False) - - @provide_session - def _set_notification_property( - self, - items: list[SlaMiss], - attr: str, - new_value: bool, - session: Session = NEW_SESSION, - ): - try: - count = 0 - for sla in items: - count += 1 - setattr(sla, attr, new_value) - session.merge(sla) - session.commit() - flash(f"{count} SLAMisses had {attr} set to {new_value}.") - except Exception as ex: - flash(str(ex), "error") - flash("Failed to set state", "error") - self.update_redirect() - return redirect(self.get_default_url()) - - class XComModelView(AirflowModelView): """View to show records from XCom table.""" diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py index 6aea6b86f7b50..f0063212c697c 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py @@ -21,7 +21,7 @@ import pytest from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import DagRun, TaskInstance @@ -124,7 +124,6 @@ def setup_attrs(self, configured_app, dagbag) -> None: self.app = configured_app self.client = self.app.test_client() # type:ignore clear_db_runs() - clear_db_sla_miss() clear_rendered_ti_fields() self.dagbag = dagbag diff --git a/providers/tests/smtp/notifications/test_smtp.py b/providers/tests/smtp/notifications/test_smtp.py index 594fa19dfee34..95b1618aa4fb3 100644 --- a/providers/tests/smtp/notifications/test_smtp.py +++ b/providers/tests/smtp/notifications/test_smtp.py @@ -25,7 +25,6 @@ from tests_common.test_utils.config import conf_vars from airflow.configuration import conf -from airflow.models import SlaMiss from airflow.operators.empty import EmptyOperator from airflow.providers.smtp.hooks.smtp import SmtpHook from airflow.providers.smtp.notifications.smtp import ( @@ -145,38 +144,6 @@ def test_notifier_with_defaults(self, mock_smtphook_hook, create_task_instance): content = mock_smtphook_hook.return_value.__enter__().send_email_smtp.call_args.kwargs["html_content"] assert f"{NUM_TRY} of 1" in content - @mock.patch("airflow.providers.smtp.notifications.smtp.SmtpHook") - def test_notifier_with_defaults_sla(self, mock_smtphook_hook, dag_maker): - with dag_maker("test_notifier") as dag: - EmptyOperator(task_id="task1") - context = { - "dag": dag, - "slas": [SlaMiss(task_id="op", dag_id=dag.dag_id, execution_date=timezone.datetime(2018, 1, 1))], - "task_list": [], - "blocking_task_list": [], - "blocking_tis": [], - } - notifier = SmtpNotifier( - from_email=conf.get("smtp", "smtp_mail_from"), - to="test_reciver@test.com", - ) - notifier(context) - mock_smtphook_hook.return_value.__enter__().send_email_smtp.assert_called_once_with( - from_email=conf.get("smtp", "smtp_mail_from"), - to="test_reciver@test.com", - subject="SLA Missed for DAG test_notifier - Task op", - html_content=mock.ANY, - smtp_conn_id="smtp_default", - files=None, - cc=None, - bcc=None, - mime_subtype="mixed", - mime_charset="utf-8", - custom_headers=None, - ) - content = mock_smtphook_hook.return_value.__enter__().send_email_smtp.call_args.kwargs["html_content"] - assert "Task List:" in content - @mock.patch("airflow.providers.smtp.notifications.smtp.SmtpHook") def test_notifier_with_nondefault_conf_vars(self, mock_smtphook_hook, create_task_instance): ti = create_task_instance(dag_id="dag", task_id="op", execution_date=timezone.datetime(2018, 1, 1)) diff --git a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py index 843de41f1b401..80fdd72726f17 100644 --- a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py @@ -33,7 +33,7 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields from tests_common.test_utils.mock_operators import MockOperator pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] @@ -81,7 +81,6 @@ def setup_attrs(self, configured_app) -> None: self.app = configured_app self.client = self.app.test_client() # type:ignore clear_db_runs() - clear_db_sla_miss() clear_rendered_ti_fields() def create_dag_runs_with_mapped_tasks(self, dag_maker, session, dags=None): @@ -227,7 +226,6 @@ def test_mapped_task_instances(self, one_task_with_mapped_tis, session): "queued_when": None, "rendered_fields": {}, "rendered_map_index": None, - "sla_miss": None, "start_date": "2020-01-01T00:00:00+00:00", "state": "success", "task_id": "task_2", diff --git a/tests/api_connexion/endpoints/test_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_task_instance_endpoint.py index bec52292fbe46..5746740205570 100644 --- a/tests/api_connexion/endpoints/test_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_instance_endpoint.py @@ -27,7 +27,7 @@ from airflow.jobs.job import Job from airflow.jobs.triggerer_job_runner import TriggererJobRunner -from airflow.models import DagRun, SlaMiss, TaskInstance, Trigger +from airflow.models import DagRun, TaskInstance, Trigger from airflow.models.renderedtifields import RenderedTaskInstanceFields as RTIF from airflow.models.taskinstancehistory import TaskInstanceHistory from airflow.utils.platform import getuser @@ -36,7 +36,7 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user -from tests_common.test_utils.db import clear_db_runs, clear_db_sla_miss, clear_rendered_ti_fields +from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields from tests_common.test_utils.www import _check_last_log pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] @@ -85,7 +85,6 @@ def setup_attrs(self, configured_app, dagbag) -> None: self.app = configured_app self.client = self.app.test_client() # type:ignore clear_db_runs() - clear_db_sla_miss() clear_rendered_ti_fields() self.dagbag = dagbag @@ -197,7 +196,6 @@ def test_should_respond_200(self, session): "priority_weight": 9, "queue": "default_queue", "queued_when": None, - "sla_miss": None, "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", @@ -256,7 +254,6 @@ def test_should_respond_200_with_task_state_in_deferred(self, session): "priority_weight": 9, "queue": "default_queue", "queued_when": None, - "sla_miss": None, "start_date": "2020-01-02T00:00:00+00:00", "state": "deferred", "task_id": "print_the_context", @@ -304,7 +301,6 @@ def test_should_respond_200_with_task_state_in_removed(self, session): "priority_weight": 9, "queue": "default_queue", "queued_when": None, - "sla_miss": None, "start_date": "2020-01-02T00:00:00+00:00", "state": "removed", "task_id": "print_the_context", @@ -318,16 +314,9 @@ def test_should_respond_200_with_task_state_in_removed(self, session): "triggerer_job": None, } - def test_should_respond_200_task_instance_with_sla_and_rendered(self, session): + def test_should_respond_200_task_instance_with_rendered(self, session): tis = self.create_task_instances(session) session.query() - sla_miss = SlaMiss( - task_id="print_the_context", - dag_id="example_python_operator", - execution_date=self.default_time, - timestamp=self.default_time, - ) - session.add(sla_miss) rendered_fields = RTIF(tis[0], render_templates=False) session.add(rendered_fields) session.commit() @@ -355,15 +344,6 @@ def test_should_respond_200_task_instance_with_sla_and_rendered(self, session): "priority_weight": 9, "queue": "default_queue", "queued_when": None, - "sla_miss": { - "dag_id": "example_python_operator", - "description": None, - "email_sent": False, - "execution_date": "2020-01-01T00:00:00+00:00", - "notification_sent": False, - "task_id": "print_the_context", - "timestamp": "2020-01-01T00:00:00+00:00", - }, "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", @@ -416,7 +396,6 @@ def test_should_respond_200_mapped_task_instance_with_rtif(self, session): "priority_weight": 9, "queue": "default_queue", "queued_when": None, - "sla_miss": None, "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", @@ -2320,7 +2299,6 @@ def test_should_respond_200(self, session): "priority_weight": 9, "queue": "default_queue", "queued_when": None, - "sla_miss": None, "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", @@ -2380,7 +2358,6 @@ def test_should_respond_200_mapped_task_instance_with_rtif(self, session): "priority_weight": 9, "queue": "default_queue", "queued_when": None, - "sla_miss": None, "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", diff --git a/tests/api_connexion/schemas/test_task_instance_schema.py b/tests/api_connexion/schemas/test_task_instance_schema.py index 181e8351d2f75..ff18ad75b5448 100644 --- a/tests/api_connexion/schemas/test_task_instance_schema.py +++ b/tests/api_connexion/schemas/test_task_instance_schema.py @@ -26,7 +26,7 @@ set_task_instance_state_form, task_instance_schema, ) -from airflow.models import RenderedTaskInstanceFields as RTIF, SlaMiss, TaskInstance as TI +from airflow.models import TaskInstance as TI from airflow.operators.empty import EmptyOperator from airflow.utils.platform import getuser from airflow.utils.state import State @@ -64,12 +64,12 @@ def set_attrs(self, session, dag_maker): session.rollback() - def test_task_instance_schema_without_sla_and_rendered(self, session): + def test_task_instance_schema_without_rendered(self, session): ti = TI(task=self.task, **self.default_ti_init) session.add(ti) for key, value in self.default_ti_extras.items(): setattr(ti, key, value) - serialized_ti = task_instance_schema.dump((ti, None, None)) + serialized_ti = task_instance_schema.dump(ti) expected_json = { "dag_id": "TEST_DAG_ID", "duration": 10000.0, @@ -88,7 +88,6 @@ def test_task_instance_schema_without_sla_and_rendered(self, session): "priority_weight": 1, "queue": "default_queue", "queued_when": None, - "sla_miss": None, "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "TEST_TASK_ID", @@ -103,63 +102,6 @@ def test_task_instance_schema_without_sla_and_rendered(self, session): } assert serialized_ti == expected_json - def test_task_instance_schema_with_sla_and_rendered(self, session): - sla_miss = SlaMiss( - task_id="TEST_TASK_ID", - dag_id="TEST_DAG_ID", - execution_date=self.default_time, - ) - session.add(sla_miss) - session.flush() - ti = TI(task=self.task, **self.default_ti_init) - session.add(ti) - for key, value in self.default_ti_extras.items(): - setattr(ti, key, value) - self.task.template_fields = ["partitions"] - setattr(self.task, "partitions", "data/ds=2022-02-17") - ti.rendered_task_instance_fields = RTIF(ti, render_templates=False) - serialized_ti = task_instance_schema.dump((ti, sla_miss)) - expected_json = { - "dag_id": "TEST_DAG_ID", - "duration": 10000.0, - "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", - "executor": None, - "executor_config": "{}", - "hostname": "", - "map_index": -1, - "max_tries": 0, - "note": "added some notes", - "operator": "EmptyOperator", - "pid": 100, - "pool": "default_pool", - "pool_slots": 1, - "priority_weight": 1, - "queue": "default_queue", - "queued_when": None, - "sla_miss": { - "dag_id": "TEST_DAG_ID", - "description": None, - "email_sent": False, - "execution_date": "2020-01-01T00:00:00+00:00", - "notification_sent": False, - "task_id": "TEST_TASK_ID", - "timestamp": None, - }, - "start_date": "2020-01-02T00:00:00+00:00", - "state": "running", - "task_id": "TEST_TASK_ID", - "task_display_name": "TEST_TASK_ID", - "try_number": 0, - "unixname": getuser(), - "dag_run_id": None, - "rendered_fields": {"partitions": "data/ds=2022-02-17"}, - "rendered_map_index": None, - "trigger": None, - "triggerer_job": None, - } - assert serialized_ti == expected_json - class TestClearTaskInstanceFormSchema: @pytest.mark.parametrize( diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 5244e96490984..f35ed29e7b1da 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -81,7 +81,6 @@ clear_db_pools, clear_db_runs, clear_db_serialized_dags, - clear_db_sla_miss, set_default_pool_slots, ) from tests_common.test_utils.mock_executor import MockExecutor @@ -143,7 +142,6 @@ def clean_db(): clear_db_backfills() clear_db_pools() clear_db_dags() - clear_db_sla_miss() clear_db_import_errors() clear_db_jobs() clear_db_assets() @@ -6236,7 +6234,6 @@ def clean_db(): clear_db_pools() clear_db_backfills() clear_db_dags() - clear_db_sla_miss() clear_db_import_errors() clear_db_jobs() clear_db_serialized_dags() diff --git a/tests/listeners/test_dag_import_error_listener.py b/tests/listeners/test_dag_import_error_listener.py index 52a6103dc5bbd..e636de4218c53 100644 --- a/tests/listeners/test_dag_import_error_listener.py +++ b/tests/listeners/test_dag_import_error_listener.py @@ -39,7 +39,6 @@ clear_db_pools, clear_db_runs, clear_db_serialized_dags, - clear_db_sla_miss, ) from tests_common.test_utils.mock_executor import MockExecutor @@ -76,7 +75,6 @@ def clean_db(): clear_db_runs() clear_db_pools() clear_db_dags() - clear_db_sla_miss() clear_db_import_errors() clear_db_jobs() clear_db_serialized_dags() diff --git a/tests/notifications/test_basenotifier.py b/tests/notifications/test_basenotifier.py index 65bda6ade8ade..b2e3d751a6f30 100644 --- a/tests/notifications/test_basenotifier.py +++ b/tests/notifications/test_basenotifier.py @@ -94,7 +94,6 @@ def test_notifier_call_with_prepared_context(self, dag_maker, caplog): "dag": None, "task_list": ["some_task"], "blocking_task_list": None, - "slas": None, "blocking_tis": None, "message": "task: {{ task_list[0] }}", } diff --git a/tests_common/test_utils/db.py b/tests_common/test_utils/db.py index bf92c3a2582d8..c9df1f3970026 100644 --- a/tests_common/test_utils/db.py +++ b/tests_common/test_utils/db.py @@ -29,7 +29,6 @@ Log, Pool, RenderedTaskInstanceFields, - SlaMiss, TaskFail, TaskInstance, TaskReschedule, @@ -129,11 +128,6 @@ def clear_db_serialized_dags(): session.query(SerializedDagModel).delete() -def clear_db_sla_miss(): - with create_session() as session: - session.query(SlaMiss).delete() - - def clear_db_pools(): with create_session() as session: session.query(Pool).delete() @@ -256,7 +250,6 @@ def clear_all(): clear_db_assets() clear_db_dags() clear_db_serialized_dags() - clear_db_sla_miss() clear_db_dag_code() clear_db_callbacks() clear_rendered_ti_fields() From 538368f215935b1e949b380f35d3c5f39869803b Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 16 Oct 2024 01:04:56 +0100 Subject: [PATCH 136/199] Bump `uv` to `0.4.22` (#43056) https://pypi.org/project/uv/0.4.22 I added `--no-sources` to our `uv pip install` command, without it, it fails with following error ``` error: Failed to build: `apache-airflow @ file:///dist/apache_airflow-3.0.0.dev0.tar.gz` Caused by: Failed to parse entry for: `local-providers` Caused by: Package is not included as workspace package in `tool.uv.workspace` ``` Example: https://github.com/apache/airflow/actions/runs/11354720505/job/31582868760#step:11:2896 --- Dockerfile | 2 +- Dockerfile.ci | 4 ++-- dev/breeze/doc/ci/02_images.md | 2 +- .../airflow_breeze/commands/release_management_commands.py | 2 +- scripts/in_container/install_airflow_and_providers.py | 2 ++ 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e0261b8f69d20..129de7a3f61ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.10.2" ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.21 +ARG AIRFLOW_UV_VERSION=0.4.22 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index bbdf044158089..cab5c91e2016d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1278,7 +1278,7 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.21 +ARG AIRFLOW_UV_VERSION=0.4.22 ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller @@ -1302,7 +1302,7 @@ ARG AIRFLOW_VERSION="" ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.21 +ARG AIRFLOW_UV_VERSION=0.4.22 ARG AIRFLOW_USE_UV="true" ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 96b61920d837f..c83adcdc03e14 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -448,7 +448,7 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.0` | PIP version used. | -| `AIRFLOW_UV_VERSION` | `0.4.21` | UV version used. | +| `AIRFLOW_UV_VERSION` | `0.4.22` | UV version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index cbc83fb043ad3..d42b3b7e95cb9 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -230,7 +230,7 @@ class VersionedFile(NamedTuple): AIRFLOW_PIP_VERSION = "24.0" -AIRFLOW_UV_VERSION = "0.4.21" +AIRFLOW_UV_VERSION = "0.4.22" AIRFLOW_USE_UV = False WHEEL_VERSION = "0.36.2" GITPYTHON_VERSION = "3.1.40" diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index bc28155875848..909eb98c2af96 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -484,6 +484,7 @@ def install_airflow_and_providers( "/usr/local/bin/uv", "pip", "install", + "--no-sources", "--python", "/usr/local/bin/python", installation_spec.airflow_package, @@ -506,6 +507,7 @@ def install_airflow_and_providers( "/usr/local/bin/uv", "pip", "install", + "--no-sources", "--python", "/usr/local/bin/python", ] From 71eaaeeab2428451e5b6be2aeecb30cbda26ed37 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 16 Oct 2024 10:23:29 +0800 Subject: [PATCH 137/199] Add 'name' and 'group' to public Asset class (#42812) --- airflow/assets/__init__.py | 73 +++++++++++++++---- airflow/models/asset.py | 20 +++-- .../assets/{tests_asset.py => test_asset.py} | 41 ++++++++++- tests/models/test_dag.py | 6 +- .../serialization/test_serialized_objects.py | 2 +- 5 files changed, 115 insertions(+), 27 deletions(-) rename tests/assets/{tests_asset.py => test_asset.py} (95%) diff --git a/airflow/assets/__init__.py b/airflow/assets/__init__.py index e11b9c49df36d..1580541847272 100644 --- a/airflow/assets/__init__.py +++ b/airflow/assets/__init__.py @@ -21,7 +21,7 @@ import os import urllib.parse import warnings -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Iterable, Iterator, cast +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Iterable, Iterator, cast, overload import attr from sqlalchemy import select @@ -74,12 +74,6 @@ def _sanitize_uri(uri: str) -> str: This checks for URI validity, and normalizes the URI if needed. A fully normalized URI is returned. """ - if not uri: - raise ValueError("Asset URI cannot be empty") - if uri.isspace(): - raise ValueError("Asset URI cannot be just whitespace") - if not uri.isascii(): - raise ValueError("Asset URI must only consist of ASCII characters") parsed = urllib.parse.urlsplit(uri) if not parsed.scheme and not parsed.netloc: # Does not look like a URI. return uri @@ -126,6 +120,24 @@ def _sanitize_uri(uri: str) -> str: return urllib.parse.urlunsplit(parsed) +def _validate_identifier(instance, attribute, value): + if not isinstance(value, str): + raise ValueError(f"{type(instance).__name__} {attribute.name} must be a string") + if len(value) > 1500: + raise ValueError(f"{type(instance).__name__} {attribute.name} cannot exceed 1500 characters") + if value.isspace(): + raise ValueError(f"{type(instance).__name__} {attribute.name} cannot be just whitespace") + if not value.isascii(): + raise ValueError(f"{type(instance).__name__} {attribute.name} must only consist of ASCII characters") + return value + + +def _validate_non_empty_identifier(instance, attribute, value): + if not _validate_identifier(instance, attribute, value): + raise ValueError(f"{type(instance).__name__} {attribute.name} cannot be empty") + return value + + def extract_event_key(value: str | Asset | AssetAlias) -> str: """ Extract the key of an inlet or an outlet event. @@ -157,7 +169,7 @@ def expand_alias_to_assets(alias: str | AssetAlias, *, session: Session = NEW_SE select(AssetAliasModel).where(AssetAliasModel.name == alias_name).limit(1) ) if asset_alias_obj: - return [Asset(uri=asset.uri, extra=asset.extra) for asset in asset_alias_obj.datasets] + return [asset.to_public() for asset in asset_alias_obj.datasets] return [] @@ -214,7 +226,7 @@ def iter_dag_dependencies(self, *, source: str, target: str) -> Iterator[DagDepe class AssetAlias(BaseAsset): """A represeation of asset alias which is used to create asset during the runtime.""" - name: str + name: str = attr.field(validator=_validate_non_empty_identifier) def iter_assets(self) -> Iterator[tuple[str, Asset]]: return iter(()) @@ -256,18 +268,49 @@ def _set_extra_default(extra: dict | None) -> dict: return extra -@attr.define(unsafe_hash=False) +@attr.define(init=False, unsafe_hash=False) class Asset(os.PathLike, BaseAsset): """A representation of data dependencies between workflows.""" - uri: str = attr.field( - converter=_sanitize_uri, - validator=[attr.validators.min_len(1), attr.validators.max_len(1500)], - ) - extra: dict[str, Any] = attr.field(factory=dict, converter=_set_extra_default) + name: str = attr.field() + uri: str = attr.field() + group: str = attr.field() + extra: dict[str, Any] = attr.field() __version__: ClassVar[int] = 1 + @overload + def __init__(self, name: str, uri: str, *, group: str = "", extra: dict | None = None) -> None: + """Canonical; both name and uri are provided.""" + + @overload + def __init__(self, name: str, *, group: str = "", extra: dict | None = None) -> None: + """It's possible to only provide the name, either by keyword or as the only positional argument.""" + + @overload + def __init__(self, *, uri: str, group: str = "", extra: dict | None = None) -> None: + """It's possible to only provide the URI as a keyword argument.""" + + def __init__( + self, + name: str | None = None, + uri: str | None = None, + *, + group: str = "", + extra: dict | None = None, + ) -> None: + if name is None and uri is None: + raise TypeError("Asset() requires either 'name' or 'uri'") + elif name is None: + name = uri + elif uri is None: + uri = name + fields = attr.fields_dict(Asset) + self.name = _validate_non_empty_identifier(self, fields["name"], name) + self.uri = _sanitize_uri(_validate_non_empty_identifier(self, fields["uri"], uri)) + self.group = _validate_identifier(self, fields["group"], group) + self.extra = _set_extra_default(extra) + def __fspath__(self) -> str: return self.uri diff --git a/airflow/models/asset.py b/airflow/models/asset.py index d5ca0ea513f12..b565c9a100ef7 100644 --- a/airflow/models/asset.py +++ b/airflow/models/asset.py @@ -205,17 +205,23 @@ class AssetModel(Base): @classmethod def from_public(cls, obj: Asset) -> AssetModel: - return cls(uri=obj.uri, extra=obj.extra) - - def __init__(self, uri: str, **kwargs): + return cls(name=obj.name, uri=obj.uri, group=obj.group, extra=obj.extra) + + def __init__(self, name: str = "", uri: str = "", **kwargs): + if not name and not uri: + raise TypeError("must provide either 'name' or 'uri'") + elif not name: + name = uri + elif not uri: + uri = name try: uri.encode("ascii") except UnicodeEncodeError: - raise ValueError("URI must be ascii") + raise ValueError("URI must be ascii") from None parsed = urlsplit(uri) if parsed.scheme and parsed.scheme.lower() == "airflow": - raise ValueError("Scheme `airflow` is reserved.") - super().__init__(name=uri, uri=uri, **kwargs) + raise ValueError("Scheme 'airflow' is reserved.") + super().__init__(name=name, uri=uri, **kwargs) def __eq__(self, other): if isinstance(other, (self.__class__, Asset)): @@ -229,7 +235,7 @@ def __repr__(self): return f"{self.__class__.__name__}(uri={self.uri!r}, extra={self.extra!r})" def to_public(self) -> Asset: - return Asset(uri=self.uri, extra=self.extra) + return Asset(name=self.name, uri=self.uri, group=self.group, extra=self.extra) class AssetActive(Base): diff --git a/tests/assets/tests_asset.py b/tests/assets/test_asset.py similarity index 95% rename from tests/assets/tests_asset.py rename to tests/assets/test_asset.py index 0bcfb83e88a79..4d3466b90c13d 100644 --- a/tests/assets/tests_asset.py +++ b/tests/assets/test_asset.py @@ -50,12 +50,26 @@ def clear_assets(): clear_db_assets() +@pytest.mark.parametrize( + ["name"], + [ + pytest.param("", id="empty"), + pytest.param("\n\t", id="whitespace"), + pytest.param("a" * 1501, id="too_long"), + pytest.param("😊", id="non-ascii"), + ], +) +def test_invalid_names(name): + with pytest.raises(ValueError): + Asset(name=name) + + @pytest.mark.parametrize( ["uri"], [ pytest.param("", id="empty"), pytest.param("\n\t", id="whitespace"), - pytest.param("a" * 3001, id="too_long"), + pytest.param("a" * 1501, id="too_long"), pytest.param("airflow://xcom/dag/task", id="reserved_scheme"), pytest.param("😊", id="non-ascii"), ], @@ -65,6 +79,31 @@ def test_invalid_uris(uri): Asset(uri=uri) +def test_only_name(): + asset = Asset(name="foobar") + assert asset.name == "foobar" + assert asset.uri == "foobar" + + +def test_only_uri(): + asset = Asset(uri="s3://bucket/key/path") + assert asset.name == "s3://bucket/key/path" + assert asset.uri == "s3://bucket/key/path" + + +@pytest.mark.parametrize("arg", ["foobar", "s3://bucket/key/path"]) +def test_only_posarg(arg): + asset = Asset(arg) + assert asset.name == arg + assert asset.uri == arg + + +def test_both_name_and_uri(): + asset = Asset("foobar", "s3://bucket/key/path") + assert asset.name == "foobar" + assert asset.uri == "s3://bucket/key/path" + + @pytest.mark.parametrize( "uri, normalized", [ diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index b439487b0161e..e6f8042253ec0 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -2715,10 +2715,10 @@ def test_asset_expression(self, session: Session) -> None: dag = DAG( dag_id="test_dag_asset_expression", schedule=AssetAny( - Asset("s3://dag1/output_1.txt", {"hi": "bye"}), + Asset("s3://dag1/output_1.txt", extra={"hi": "bye"}), AssetAll( - Asset("s3://dag2/output_1.txt", {"hi": "bye"}), - Asset("s3://dag3/output_3.txt", {"hi": "bye"}), + Asset("s3://dag2/output_1.txt", extra={"hi": "bye"}), + Asset("s3://dag3/output_3.txt", extra={"hi": "bye"}), ), AssetAlias(name="test_name"), ), diff --git a/tests/serialization/test_serialized_objects.py b/tests/serialization/test_serialized_objects.py index 5d35278d89bfc..56a31d4d38b78 100644 --- a/tests/serialization/test_serialized_objects.py +++ b/tests/serialization/test_serialized_objects.py @@ -327,7 +327,7 @@ def test_backcompat_deserialize_connection(conn_uri): id=1, filename="test_file", elasticsearch_id="test_id", created_at=datetime.now() ), DagTagPydantic: DagTag(), - AssetPydantic: Asset("uri", {}), + AssetPydantic: Asset("uri", extra={}), AssetEventPydantic: AssetEvent(), } From 90fa32f0bc2eff9a83f7bf68109d5ce9746f1dde Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 16 Oct 2024 10:32:57 +0800 Subject: [PATCH 138/199] Remove zombie ti from its executor (#42932) --- airflow/jobs/scheduler_job_runner.py | 119 +++++++++++--------- airflow/models/taskinstance.py | 7 +- docs/apache-airflow/core-concepts/tasks.rst | 4 +- tests/jobs/test_scheduler_job.py | 68 ++++++----- tests_common/test_utils/mock_executor.py | 5 +- 5 files changed, 105 insertions(+), 98 deletions(-) diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index 30f58885a9aa2..951602e14e648 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -1077,7 +1077,7 @@ def _run_scheduler_loop(self) -> None: timers.call_regular_interval( conf.getfloat("scheduler", "zombie_detection_interval", fallback=10.0), - self._find_zombies, + self._find_and_purge_zombies, ) timers.call_regular_interval(60.0, self._update_dag_run_state_for_paused_dags) @@ -1953,73 +1953,80 @@ def check_trigger_timeouts( if num_timed_out_tasks: self.log.info("Timed out %i deferred tasks without fired triggers", num_timed_out_tasks) - # [START find_zombies] - def _find_zombies(self) -> None: + # [START find_and_purge_zombies] + def _find_and_purge_zombies(self) -> None: """ - Find zombie task instances and create a TaskCallbackRequest to be handled by the DAG processor. + Find and purge zombie task instances. - Zombie instances are tasks haven't heartbeated for too long or have a no-longer-running LocalTaskJob. + Zombie instances are tasks that failed to heartbeat for too long, or + have a no-longer-running LocalTaskJob. + + A TaskCallbackRequest is also created for the killed zombie to be + handled by the DAG processor, and the executor is informed to no longer + count the zombie as running when it calculates parallelism. """ + with create_session() as session: + if zombies := self._find_zombies(session=session): + self._purge_zombies(zombies, session=session) + + def _find_zombies(self, *, session: Session) -> list[tuple[TI, str, str]]: from airflow.jobs.job import Job self.log.debug("Finding 'running' jobs without a recent heartbeat") limit_dttm = timezone.utcnow() - timedelta(seconds=self._zombie_threshold_secs) - - with create_session() as session: - zombies: list[tuple[TI, str, str]] = ( - session.execute( - select(TI, DM.fileloc, DM.processor_subdir) - .with_hint(TI, "USE INDEX (ti_state)", dialect_name="mysql") - .join(Job, TI.job_id == Job.id) - .join(DM, TI.dag_id == DM.dag_id) - .where(TI.state == TaskInstanceState.RUNNING) - .where( - or_( - Job.state != JobState.RUNNING, - Job.latest_heartbeat < limit_dttm, - ) - ) - .where(Job.job_type == "LocalTaskJob") - .where(TI.queued_by_job_id == self.job.id) - ) - .unique() - .all() + zombies = ( + session.execute( + select(TI, DM.fileloc, DM.processor_subdir) + .with_hint(TI, "USE INDEX (ti_state)", dialect_name="mysql") + .join(Job, TI.job_id == Job.id) + .join(DM, TI.dag_id == DM.dag_id) + .where(TI.state == TaskInstanceState.RUNNING) + .where(or_(Job.state != JobState.RUNNING, Job.latest_heartbeat < limit_dttm)) + .where(Job.job_type == "LocalTaskJob") + .where(TI.queued_by_job_id == self.job.id) ) - + .unique() + .all() + ) if zombies: self.log.warning("Failing (%s) jobs without heartbeat after %s", len(zombies), limit_dttm) - - with create_session() as session: - for ti, file_loc, processor_subdir in zombies: - zombie_message_details = self._generate_zombie_message_details(ti) - request = TaskCallbackRequest( - full_filepath=file_loc, - processor_subdir=processor_subdir, - simple_task_instance=SimpleTaskInstance.from_ti(ti), - msg=str(zombie_message_details), - ) - session.add( - Log( - event="heartbeat timeout", - task_instance=ti.key, - extra=( - f"Task did not emit heartbeat within time limit ({self._zombie_threshold_secs} " - "seconds) and will be terminated. " - "See https://airflow.apache.org/docs/apache-airflow/" - "stable/core-concepts/tasks.html#zombie-undead-tasks" - ), - ) - ) - self.log.error( - "Detected zombie job: %s " - "(See https://airflow.apache.org/docs/apache-airflow/" - "stable/core-concepts/tasks.html#zombie-undead-tasks)", - request, + return zombies + + def _purge_zombies(self, zombies: list[tuple[TI, str, str]], *, session: Session) -> None: + for ti, file_loc, processor_subdir in zombies: + zombie_message_details = self._generate_zombie_message_details(ti) + request = TaskCallbackRequest( + full_filepath=file_loc, + processor_subdir=processor_subdir, + simple_task_instance=SimpleTaskInstance.from_ti(ti), + msg=str(zombie_message_details), + ) + session.add( + Log( + event="heartbeat timeout", + task_instance=ti.key, + extra=( + f"Task did not emit heartbeat within time limit ({self._zombie_threshold_secs} " + "seconds) and will be terminated. " + "See https://airflow.apache.org/docs/apache-airflow/" + "stable/core-concepts/tasks.html#zombie-undead-tasks" + ), ) - self.job.executor.send_callback(request) - Stats.incr("zombies_killed", tags={"dag_id": ti.dag_id, "task_id": ti.task_id}) + ) + self.log.error( + "Detected zombie job: %s " + "(See https://airflow.apache.org/docs/apache-airflow/" + "stable/core-concepts/tasks.html#zombie-undead-tasks)", + request, + ) + self.job.executor.send_callback(request) + if (executor := self._try_to_load_executor(ti.executor)) is None: + self.log.warning("Cannot clean up zombie %r with non-existent executor %s", ti, ti.executor) + continue + executor.change_state(ti.key, TaskInstanceState.FAILED, remove_running=True) + Stats.incr("zombies_killed", tags={"dag_id": ti.dag_id, "task_id": ti.task_id}) - # [END find_zombies] + # [END find_and_purge_zombies] @staticmethod def _generate_zombie_message_details(ti: TI) -> dict[str, Any]: diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index c1373e5d6a12d..1dbe299f25b39 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -154,7 +154,6 @@ from sqlalchemy.sql.expression import ColumnOperators from airflow.models.abstractoperator import TaskStateChangeCallback - from airflow.models.asset import AssetEvent from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG, DagModel from airflow.models.dagrun import DagRun @@ -3925,7 +3924,11 @@ def __init__( self.queue = queue self.key = key - def __eq__(self, other): + def __repr__(self) -> str: + attrs = ", ".join(f"{k}={v!r}" for k, v in self.__dict__.items()) + return f"SimpleTaskInstance({attrs})" + + def __eq__(self, other) -> bool: if isinstance(other, self.__class__): return self.__dict__ == other.__dict__ return NotImplemented diff --git a/docs/apache-airflow/core-concepts/tasks.rst b/docs/apache-airflow/core-concepts/tasks.rst index 5adfe8be46024..3f5a1a21b778b 100644 --- a/docs/apache-airflow/core-concepts/tasks.rst +++ b/docs/apache-airflow/core-concepts/tasks.rst @@ -189,8 +189,8 @@ Below is the code snippet from the Airflow scheduler that runs periodically to d .. exampleinclude:: /../../airflow/jobs/scheduler_job_runner.py :language: python - :start-after: [START find_zombies] - :end-before: [END find_zombies] + :start-after: [START find_and_purge_zombies] + :end-before: [END find_and_purge_zombies] The explanation of the criteria used in the above snippet to detect zombie tasks is as below: diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index f35ed29e7b1da..01d1c5fe7a331 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -2976,8 +2976,8 @@ def test_scheduler_task_start_date(self, configs): ti2s = tiq.filter(TaskInstance.task_id == "dummy2").all() assert len(ti1s) == 0 assert len(ti2s) >= 2 - for task in ti2s: - assert task.state == State.SUCCESS + for ti in ti2s: + assert ti.state == State.SUCCESS @pytest.mark.parametrize( "configs", @@ -5539,36 +5539,37 @@ def side_effect(*args, **kwargs): with pytest.raises(OperationalError): check_if_trigger_timeout(1) - def test_find_zombies_nothing(self): + def test_find_and_purge_zombies_nothing(self): executor = MockExecutor(do_update=False) scheduler_job = Job(executor=executor) - self.job_runner = SchedulerJobRunner(scheduler_job) - self.job_runner.processor_agent = mock.MagicMock() - - self.job_runner._find_zombies() - - scheduler_job.executor.callback_sink.send.assert_not_called() + with mock.patch("airflow.executors.executor_loader.ExecutorLoader.load_executor") as loader_mock: + loader_mock.return_value = executor + self.job_runner = SchedulerJobRunner(scheduler_job) + self.job_runner.processor_agent = mock.MagicMock() + self.job_runner._find_and_purge_zombies() + executor.callback_sink.send.assert_not_called() - def test_find_zombies(self, load_examples): + def test_find_and_purge_zombies(self, load_examples, session): dagbag = DagBag(TEST_DAG_FOLDER, read_dags_from_db=False) - with create_session() as session: - session.query(Job).delete() - dag = dagbag.get_dag("example_branch_operator") - dag.sync_to_db() - data_interval = dag.infer_automated_data_interval(DEFAULT_LOGICAL_DATE) - triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dag_run = dag.create_dagrun( - state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, - run_type=DagRunType.SCHEDULED, - session=session, - data_interval=data_interval, - **triggered_by_kwargs, - ) - scheduler_job = Job() + dag = dagbag.get_dag("example_branch_operator") + dag.sync_to_db() + data_interval = dag.infer_automated_data_interval(DEFAULT_LOGICAL_DATE) + triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} + dag_run = dag.create_dagrun( + state=DagRunState.RUNNING, + execution_date=DEFAULT_DATE, + run_type=DagRunType.SCHEDULED, + session=session, + data_interval=data_interval, + **triggered_by_kwargs, + ) + + executor = MockExecutor() + scheduler_job = Job(executor=executor) + with mock.patch("airflow.executors.executor_loader.ExecutorLoader.load_executor") as loader_mock: + loader_mock.return_value = executor self.job_runner = SchedulerJobRunner(job=scheduler_job, subdir=os.devnull) - scheduler_job.executor = MockExecutor() self.job_runner.processor_agent = mock.MagicMock() # We will provision 2 tasks so we can check we only find zombies from this scheduler @@ -5594,11 +5595,12 @@ def test_find_zombies(self, load_examples): ti.queued_by_job_id = scheduler_job.id session.flush() + executor.running.add(ti.key) # The executor normally does this during heartbeat. + self.job_runner._find_and_purge_zombies() + assert ti.key not in executor.running - self.job_runner._find_zombies() - - scheduler_job.executor.callback_sink.send.assert_called_once() - callback_requests = scheduler_job.executor.callback_sink.send.call_args.args + executor.callback_sink.send.assert_called_once() + callback_requests = executor.callback_sink.send.call_args.args assert len(callback_requests) == 1 callback_request = callback_requests[0] assert isinstance(callback_request.simple_task_instance, SimpleTaskInstance) @@ -5610,10 +5612,6 @@ def test_find_zombies(self, load_examples): assert callback_request.simple_task_instance.run_id == ti.run_id assert callback_request.simple_task_instance.map_index == ti.map_index - with create_session() as session: - session.query(TaskInstance).delete() - session.query(Job).delete() - def test_zombie_message(self, load_examples): """ Check that the zombie message comes out as expected @@ -5729,7 +5727,7 @@ def test_find_zombies_handle_failure_callbacks_are_correctly_passed_to_dag_proce scheduler_job.executor = MockExecutor() self.job_runner.processor_agent = mock.MagicMock() - self.job_runner._find_zombies() + self.job_runner._find_and_purge_zombies() scheduler_job.executor.callback_sink.send.assert_called_once() diff --git a/tests_common/test_utils/mock_executor.py b/tests_common/test_utils/mock_executor.py index 506c0447589ac..6d4791e889150 100644 --- a/tests_common/test_utils/mock_executor.py +++ b/tests_common/test_utils/mock_executor.py @@ -36,7 +36,6 @@ class MockExecutor(BaseExecutor): def __init__(self, do_update=True, *args, **kwargs): self.do_update = do_update - self._running = [] self.callback_sink = MagicMock() # A list of "batches" of tasks @@ -88,8 +87,8 @@ def terminate(self): def end(self): self.sync() - def change_state(self, key, state, info=None): - super().change_state(key, state, info=info) + def change_state(self, key, state, info=None, remove_running=False): + super().change_state(key, state, info=info, remove_running=remove_running) # The normal event buffer is cleared after reading, we want to keep # a list of all events for testing self.sorted_tasks.append((key, (state, info))) From 771188d500198e1f6e9a242317b8c0a90ad333bb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 16 Oct 2024 12:08:50 +0800 Subject: [PATCH 139/199] Drop unneeded unique() call on SQL (#43064) --- airflow/jobs/scheduler_job_runner.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index 951602e14e648..7958adb64cf96 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -1974,20 +1974,16 @@ def _find_zombies(self, *, session: Session) -> list[tuple[TI, str, str]]: self.log.debug("Finding 'running' jobs without a recent heartbeat") limit_dttm = timezone.utcnow() - timedelta(seconds=self._zombie_threshold_secs) - zombies = ( - session.execute( - select(TI, DM.fileloc, DM.processor_subdir) - .with_hint(TI, "USE INDEX (ti_state)", dialect_name="mysql") - .join(Job, TI.job_id == Job.id) - .join(DM, TI.dag_id == DM.dag_id) - .where(TI.state == TaskInstanceState.RUNNING) - .where(or_(Job.state != JobState.RUNNING, Job.latest_heartbeat < limit_dttm)) - .where(Job.job_type == "LocalTaskJob") - .where(TI.queued_by_job_id == self.job.id) - ) - .unique() - .all() - ) + zombies = session.execute( + select(TI, DM.fileloc, DM.processor_subdir) + .with_hint(TI, "USE INDEX (ti_state)", dialect_name="mysql") + .join(Job, TI.job_id == Job.id) + .join(DM, TI.dag_id == DM.dag_id) + .where(TI.state == TaskInstanceState.RUNNING) + .where(or_(Job.state != JobState.RUNNING, Job.latest_heartbeat < limit_dttm)) + .where(Job.job_type == "LocalTaskJob") + .where(TI.queued_by_job_id == self.job.id) + ).all() if zombies: self.log.warning("Failing (%s) jobs without heartbeat after %s", len(zombies), limit_dttm) return zombies From ba9c07557480f0505922dd29b975dc956ff29540 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 16 Oct 2024 12:19:41 +0100 Subject: [PATCH 140/199] Allow building docs for not-ready providers (#43071) The `not-ready` providers were skipped in https://github.com/apache/airflow/pull/42873 as we were releasing a batch of providers. I have manually gone ahead and uploaded inventories for edge & standard providers so this is no longer a problem. --- dev/README_AIRFLOW3_DEV.md | 2 +- docs/conf.py | 1 + docs/exts/provider_yaml_utils.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/README_AIRFLOW3_DEV.md b/dev/README_AIRFLOW3_DEV.md index fb626843bdc85..42ac848da84e4 100644 --- a/dev/README_AIRFLOW3_DEV.md +++ b/dev/README_AIRFLOW3_DEV.md @@ -147,7 +147,7 @@ TBD Milestone will be added only to the original PR. -1. PR targeting `v2-10-test` directly - mlinestone will be on that PR. +1. PR targeting `v2-10-test` directly - milestone will be on that PR. 2. PR targeting `main` with backport PR targeting `v2-10-test`. Milestone will be added only on the PR targeting `v2-10-main`. ## Set 2.11 milestone diff --git a/docs/conf.py b/docs/conf.py index a09e54db63d20..e95796db7c780 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -765,6 +765,7 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "") # These sub-folders aren't really providers, but we need __init__.py files else various tools (ruff, mypy) # get confused by providers/tests/systems/cncf/kubernetes and think that folder is the top level # kubernetes module! + "*/providers/src/airflow/providers/__init__.py", "*/providers/tests/__init__.py", "*/providers/tests/cncf/__init__.py", "*/providers/tests/common/__init__.py", diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index ad99da3c10eaf..6e5d3a835e1d0 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -70,7 +70,7 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: except jsonschema.ValidationError as ex: msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" raise RuntimeError(msg) - if provider["state"] in ["suspended", "not-ready"] and not include_suspended: + if provider["state"] == "suspended" and not include_suspended: continue provider_yaml_dir = os.path.dirname(provider_yaml_path) provider["python-module"] = _filepath_to_module(provider_yaml_dir) From 9fdee6403a54e676ce3626cda3ad870326d7f8f0 Mon Sep 17 00:00:00 2001 From: Biswamitra Biswas Date: Wed, 16 Oct 2024 13:33:49 +0200 Subject: [PATCH 141/199] Typo correction in 07_local_virtualenv.rst - "way" (#43072) --- contributing-docs/07_local_virtualenv.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing-docs/07_local_virtualenv.rst b/contributing-docs/07_local_virtualenv.rst index 2c92edee97a7b..a5238bfe829b9 100644 --- a/contributing-docs/07_local_virtualenv.rst +++ b/contributing-docs/07_local_virtualenv.rst @@ -19,7 +19,7 @@ Local Virtual Environment (virtualenv) ====================================== -The easiest wey to run tests for Airflow is to use local virtualenv. While Breeze is the recommended +The easiest way to run tests for Airflow is to use local virtualenv. While Breeze is the recommended way to run tests - because it provides a reproducible environment and is easy to set up, it is not always the best option as you need to run your tests inside a docker container. This might make it harder to debug the tests and to use your IDE to run them. From b55756512e6620af3e7afd7d77987efdf02721f1 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Wed, 16 Oct 2024 04:38:27 -0700 Subject: [PATCH 142/199] Enable explicit namespaces (#42951) * The recent PR to reorganize the package requires a change to explicit namespaces for at least some IDEs (PyChasrm for sure, possibly others). * static fixes * Update providers/src/airflow/providers/__init__.py Co-authored-by: Ash Berlin-Taylor * Remove duplicate line * Rephrase relevant comments * Add providers tests init file to autoapi_ignore * typo Co-authored-by: Shahar Epstein <60007259+shahar1@users.noreply.github.com> --------- Co-authored-by: Ash Berlin-Taylor Co-authored-by: Shahar Epstein <60007259+shahar1@users.noreply.github.com> --- airflow/__init__.py | 11 ++++----- providers/src/airflow/providers/__init__.py | 26 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 providers/src/airflow/providers/__init__.py diff --git a/airflow/__init__.py b/airflow/__init__.py index 18f4cc3e3c28c..f6c40b5091bbc 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -17,6 +17,11 @@ # under the License. from __future__ import annotations +# Make `airflow` a namespace package, supporting installing +# airflow.providers.* in different locations (i.e. one in site, and one in user +# lib.) This is required by some IDEs to resolve the import paths. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore + __version__ = "3.0.0.dev0" import os @@ -59,12 +64,6 @@ "XComArg", ] -# Make `airflow` a namespace package, supporting installing -# airflow.providers.* in different locations (i.e. one in site, and one in user -# lib.) -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore - - # Perform side-effects unless someone has explicitly opted out before import # WARNING: DO NOT USE THIS UNLESS YOU REALLY KNOW WHAT YOU'RE DOING. # This environment variable prevents proper initialization, and things like diff --git a/providers/src/airflow/providers/__init__.py b/providers/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..66fbd04b36e4d --- /dev/null +++ b/providers/src/airflow/providers/__init__.py @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +# Make `airflow` a namespace package, supporting installing +# airflow.providers.* in different locations (i.e. one in site, and one in user +# lib.) This is required by some IDEs to resolve the import paths. +# +# Note: this file is not installed or distributed in any distribution! +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore From bfc0ac38738c1be8199c7b664394ff5fef795713 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 16 Oct 2024 14:15:19 +0200 Subject: [PATCH 143/199] Replace Mateusz with Kalyan in the triage team (#43074) --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 250208c9dcf2e..7685192c06351 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -123,7 +123,7 @@ github: # https://github.com/apache/infrastructure-asfyaml/blob/main/README.md#assigning-the-github-triage-role-to-external-collaborators - aritra24 - dirrao - - mhenc + - rawwar - nathadfield - sunank200 - vatsrahul1001 From 456c2f6bf0fb27336e021408d2badb2aca77d3b6 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 16 Oct 2024 13:37:36 +0100 Subject: [PATCH 144/199] Reorganize ``api_fastapi`` folder into apps (#43062) --- .pre-commit-config.yaml | 2 +- airflow/api_fastapi/app.py | 79 +-------------- .../api_fastapi/{db => common}/__init__.py | 0 .../{openapi => common/db}/__init__.py | 0 airflow/api_fastapi/{ => common}/db/common.py | 2 +- airflow/api_fastapi/{ => common}/db/dags.py | 0 .../api_fastapi/{ => common}/parameters.py | 0 .../api_fastapi/{routes => common}/router.py | 0 .../{routes => core_api}/__init__.py | 0 airflow/api_fastapi/core_api/app.py | 96 +++++++++++++++++++ .../openapi}/__init__.py | 0 .../{ => core_api}/openapi/exceptions.py | 0 .../{ => core_api}/openapi/v1-generated.yaml | 2 +- .../api_fastapi/core_api}/routes/__init__.py | 0 .../{ => core_api}/routes/public/__init__.py | 12 +-- .../routes/public/connections.py | 13 ++- .../{ => core_api}/routes/public/dag_run.py | 8 +- .../{ => core_api}/routes/public/dags.py | 12 +-- .../{ => core_api}/routes/public/monitor.py | 4 +- .../{ => core_api}/routes/public/variables.py | 8 +- .../{ => core_api}/routes/ui/__init__.py | 6 +- .../{ => core_api}/routes/ui/assets.py | 4 +- .../{ => core_api}/routes/ui/dashboard.py | 10 +- .../core_api/serializers}/__init__.py | 0 .../{ => core_api}/serializers/connections.py | 0 .../{ => core_api}/serializers/dag_run.py | 0 .../{ => core_api}/serializers/dags.py | 0 .../{ => core_api}/serializers/dashboard.py | 0 .../{ => core_api}/serializers/monitor.py | 0 .../{ => core_api}/serializers/variables.py | 0 airflow/ui/package.json | 2 +- contributing-docs/17_adding_api_endpoints.rst | 6 +- .../run_update_fastapi_api_spec.py | 2 +- .../{routes/ui => core_api}/__init__.py | 0 tests/api_fastapi/core_api/routes/__init__.py | 16 ++++ .../core_api/routes/public/__init__.py | 16 ++++ .../routes/public/test_connections.py | 0 .../routes/public/test_dag_run.py | 0 .../{ => core_api}/routes/public/test_dags.py | 4 +- .../routes/public/test_monitor.py | 0 .../routes/public/test_variables.py | 0 .../core_api/routes/ui/__init__.py | 16 ++++ .../{ => core_api}/routes/ui/test_assets.py | 0 .../routes/ui/test_dashboard.py | 0 44 files changed, 199 insertions(+), 121 deletions(-) rename airflow/api_fastapi/{db => common}/__init__.py (100%) rename airflow/api_fastapi/{openapi => common/db}/__init__.py (100%) rename airflow/api_fastapi/{ => common}/db/common.py (97%) rename airflow/api_fastapi/{ => common}/db/dags.py (100%) rename airflow/api_fastapi/{ => common}/parameters.py (100%) rename airflow/api_fastapi/{routes => common}/router.py (100%) rename airflow/api_fastapi/{routes => core_api}/__init__.py (100%) create mode 100644 airflow/api_fastapi/core_api/app.py rename airflow/api_fastapi/{serializers => core_api/openapi}/__init__.py (100%) rename airflow/api_fastapi/{ => core_api}/openapi/exceptions.py (100%) rename airflow/api_fastapi/{ => core_api}/openapi/v1-generated.yaml (99%) rename {tests/api_fastapi => airflow/api_fastapi/core_api}/routes/__init__.py (100%) rename airflow/api_fastapi/{ => core_api}/routes/public/__init__.py (70%) rename airflow/api_fastapi/{ => core_api}/routes/public/connections.py (87%) rename airflow/api_fastapi/{ => core_api}/routes/public/dag_run.py (88%) rename airflow/api_fastapi/{ => core_api}/routes/public/dags.py (94%) rename airflow/api_fastapi/{ => core_api}/routes/public/monitor.py (89%) rename airflow/api_fastapi/{ => core_api}/routes/public/variables.py (92%) rename airflow/api_fastapi/{ => core_api}/routes/ui/__init__.py (81%) rename airflow/api_fastapi/{ => core_api}/routes/ui/assets.py (96%) rename airflow/api_fastapi/{ => core_api}/routes/ui/dashboard.py (90%) rename {tests/api_fastapi/routes/public => airflow/api_fastapi/core_api/serializers}/__init__.py (100%) rename airflow/api_fastapi/{ => core_api}/serializers/connections.py (100%) rename airflow/api_fastapi/{ => core_api}/serializers/dag_run.py (100%) rename airflow/api_fastapi/{ => core_api}/serializers/dags.py (100%) rename airflow/api_fastapi/{ => core_api}/serializers/dashboard.py (100%) rename airflow/api_fastapi/{ => core_api}/serializers/monitor.py (100%) rename airflow/api_fastapi/{ => core_api}/serializers/variables.py (100%) rename tests/api_fastapi/{routes/ui => core_api}/__init__.py (100%) create mode 100644 tests/api_fastapi/core_api/routes/__init__.py create mode 100644 tests/api_fastapi/core_api/routes/public/__init__.py rename tests/api_fastapi/{ => core_api}/routes/public/test_connections.py (100%) rename tests/api_fastapi/{ => core_api}/routes/public/test_dag_run.py (100%) rename tests/api_fastapi/{ => core_api}/routes/public/test_dags.py (98%) rename tests/api_fastapi/{ => core_api}/routes/public/test_monitor.py (100%) rename tests/api_fastapi/{ => core_api}/routes/public/test_variables.py (100%) create mode 100644 tests/api_fastapi/core_api/routes/ui/__init__.py rename tests/api_fastapi/{ => core_api}/routes/ui/test_assets.py (100%) rename tests/api_fastapi/{ => core_api}/routes/ui/test_dashboard.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe4315fafc212..467f0e44d806b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all YAML files except Helm templates - exclude: ^\.github/.*$|^.*/.*_vendor/|^chart/templates/.*|.*/reproducible_build.yaml$|^airflow/api_fastapi/openapi/v1-generated.yaml$|^.*/pnpm-lock.yaml$ + exclude: ^\.github/.*$|^.*/.*_vendor/|^chart/templates/.*|.*/reproducible_build.yaml$|^airflow/api_fastapi/core_api/openapi/v1-generated.yaml$|^.*/pnpm-lock.yaml$ types: [yaml] files: \.ya?ml$ args: diff --git a/airflow/api_fastapi/app.py b/airflow/api_fastapi/app.py index 912b8496dd6e8..ffe13bec0fa76 100644 --- a/airflow/api_fastapi/app.py +++ b/airflow/api_fastapi/app.py @@ -17,40 +17,25 @@ from __future__ import annotations import logging -import os -from pathlib import Path -from typing import cast -from fastapi import FastAPI, Request +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse -from fastapi.staticfiles import StaticFiles -from fastapi.templating import Jinja2Templates -from airflow.settings import AIRFLOW_PATH -from airflow.www.extensions.init_dagbag import get_dag_bag +from airflow.api_fastapi.core_api.app import init_dag_bag, init_plugins, init_views log = logging.getLogger(__name__) app: FastAPI | None = None -def init_dag_bag(app: FastAPI) -> None: - """ - Create global DagBag for the FastAPI application. - - To access it use ``request.app.state.dag_bag``. - """ - app.state.dag_bag = get_dag_bag() - - -def create_app() -> FastAPI: +def create_app(apps: list[str] | None = None) -> FastAPI: from airflow.configuration import conf app = FastAPI( + title="Airflow API", description="Airflow API. All endpoints located under ``/public`` can be used safely, are stable and backward compatible. " "Endpoints located under ``/ui`` are dedicated to the UI and are subject to breaking change " - "depending on the need of the frontend. Users should not rely on those but use the public ones instead." + "depending on the need of the frontend. Users should not rely on those but use the public ones instead.", ) init_dag_bag(app) @@ -75,60 +60,6 @@ def create_app() -> FastAPI: return app -def init_views(app: FastAPI) -> None: - """Init views by registering the different routers.""" - from airflow.api_fastapi.routes.public import public_router - from airflow.api_fastapi.routes.ui import ui_router - - app.include_router(ui_router) - app.include_router(public_router) - - dev_mode = os.environ.get("DEV_MODE", False) == "true" - - directory = Path(AIRFLOW_PATH) / ("airflow/ui/dev" if dev_mode else "airflow/ui/dist") - - # During python tests or when the backend is run without having the frontend build - # those directories might not exist. App should not fail initializing in those scenarios. - Path(directory).mkdir(exist_ok=True) - - templates = Jinja2Templates(directory=directory) - - app.mount( - "/static", - StaticFiles( - directory=directory, - html=True, - ), - name="webapp_static_folder", - ) - - @app.get("/webapp/{rest_of_path:path}", response_class=HTMLResponse, include_in_schema=False) - def webapp(request: Request, rest_of_path: str): - return templates.TemplateResponse("/index.html", {"request": request}, media_type="text/html") - - -def init_plugins(app: FastAPI) -> None: - """Integrate FastAPI app plugins.""" - from airflow import plugins_manager - - plugins_manager.initialize_fastapi_plugins() - - # After calling initialize_fastapi_plugins, fastapi_apps cannot be None anymore. - for subapp_dict in cast(list, plugins_manager.fastapi_apps): - name = subapp_dict.get("name") - subapp = subapp_dict.get("app") - if subapp is None: - log.error("'app' key is missing for the fastapi app: %s", name) - continue - url_prefix = subapp_dict.get("url_prefix") - if url_prefix is None: - log.error("'url_prefix' key is missing for the fastapi app: %s", name) - continue - - log.debug("Adding subapplication %s under prefix %s", name, url_prefix) - app.mount(url_prefix, subapp) - - def cached_app(config=None, testing=False) -> FastAPI: """Return cached instance of Airflow UI app.""" global app diff --git a/airflow/api_fastapi/db/__init__.py b/airflow/api_fastapi/common/__init__.py similarity index 100% rename from airflow/api_fastapi/db/__init__.py rename to airflow/api_fastapi/common/__init__.py diff --git a/airflow/api_fastapi/openapi/__init__.py b/airflow/api_fastapi/common/db/__init__.py similarity index 100% rename from airflow/api_fastapi/openapi/__init__.py rename to airflow/api_fastapi/common/db/__init__.py diff --git a/airflow/api_fastapi/db/common.py b/airflow/api_fastapi/common/db/common.py similarity index 97% rename from airflow/api_fastapi/db/common.py rename to airflow/api_fastapi/common/db/common.py index f611eaa64f07d..3feb3ba599504 100644 --- a/airflow/api_fastapi/db/common.py +++ b/airflow/api_fastapi/common/db/common.py @@ -26,7 +26,7 @@ from sqlalchemy.orm import Session from sqlalchemy.sql import Select - from airflow.api_fastapi.parameters import BaseParam + from airflow.api_fastapi.common.parameters import BaseParam async def get_session() -> Session: diff --git a/airflow/api_fastapi/db/dags.py b/airflow/api_fastapi/common/db/dags.py similarity index 100% rename from airflow/api_fastapi/db/dags.py rename to airflow/api_fastapi/common/db/dags.py diff --git a/airflow/api_fastapi/parameters.py b/airflow/api_fastapi/common/parameters.py similarity index 100% rename from airflow/api_fastapi/parameters.py rename to airflow/api_fastapi/common/parameters.py diff --git a/airflow/api_fastapi/routes/router.py b/airflow/api_fastapi/common/router.py similarity index 100% rename from airflow/api_fastapi/routes/router.py rename to airflow/api_fastapi/common/router.py diff --git a/airflow/api_fastapi/routes/__init__.py b/airflow/api_fastapi/core_api/__init__.py similarity index 100% rename from airflow/api_fastapi/routes/__init__.py rename to airflow/api_fastapi/core_api/__init__.py diff --git a/airflow/api_fastapi/core_api/app.py b/airflow/api_fastapi/core_api/app.py new file mode 100644 index 0000000000000..f47a33227f366 --- /dev/null +++ b/airflow/api_fastapi/core_api/app.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import cast + +from fastapi import FastAPI +from starlette.requests import Request +from starlette.responses import HTMLResponse +from starlette.staticfiles import StaticFiles +from starlette.templating import Jinja2Templates + +from airflow.settings import AIRFLOW_PATH +from airflow.www.extensions.init_dagbag import get_dag_bag + +log = logging.getLogger(__name__) + + +def init_dag_bag(app: FastAPI) -> None: + """ + Create global DagBag for the FastAPI application. + + To access it use ``request.app.state.dag_bag``. + """ + app.state.dag_bag = get_dag_bag() + + +def init_views(app: FastAPI) -> None: + """Init views by registering the different routers.""" + from airflow.api_fastapi.core_api.routes.public import public_router + from airflow.api_fastapi.core_api.routes.ui import ui_router + + app.include_router(ui_router) + app.include_router(public_router) + + dev_mode = os.environ.get("DEV_MODE", False) == "true" + + directory = Path(AIRFLOW_PATH) / ("airflow/ui/dev" if dev_mode else "airflow/ui/dist") + + # During python tests or when the backend is run without having the frontend build + # those directories might not exist. App should not fail initializing in those scenarios. + Path(directory).mkdir(exist_ok=True) + + templates = Jinja2Templates(directory=directory) + + app.mount( + "/static", + StaticFiles( + directory=directory, + html=True, + ), + name="webapp_static_folder", + ) + + @app.get("/webapp/{rest_of_path:path}", response_class=HTMLResponse, include_in_schema=False) + def webapp(request: Request, rest_of_path: str): + return templates.TemplateResponse("/index.html", {"request": request}, media_type="text/html") + + +def init_plugins(app: FastAPI) -> None: + """Integrate FastAPI app plugins.""" + from airflow import plugins_manager + + plugins_manager.initialize_fastapi_plugins() + + # After calling initialize_fastapi_plugins, fastapi_apps cannot be None anymore. + for subapp_dict in cast(list, plugins_manager.fastapi_apps): + name = subapp_dict.get("name") + subapp = subapp_dict.get("app") + if subapp is None: + log.error("'app' key is missing for the fastapi app: %s", name) + continue + url_prefix = subapp_dict.get("url_prefix") + if url_prefix is None: + log.error("'url_prefix' key is missing for the fastapi app: %s", name) + continue + + log.debug("Adding subapplication %s under prefix %s", name, url_prefix) + app.mount(url_prefix, subapp) diff --git a/airflow/api_fastapi/serializers/__init__.py b/airflow/api_fastapi/core_api/openapi/__init__.py similarity index 100% rename from airflow/api_fastapi/serializers/__init__.py rename to airflow/api_fastapi/core_api/openapi/__init__.py diff --git a/airflow/api_fastapi/openapi/exceptions.py b/airflow/api_fastapi/core_api/openapi/exceptions.py similarity index 100% rename from airflow/api_fastapi/openapi/exceptions.py rename to airflow/api_fastapi/core_api/openapi/exceptions.py diff --git a/airflow/api_fastapi/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml similarity index 99% rename from airflow/api_fastapi/openapi/v1-generated.yaml rename to airflow/api_fastapi/core_api/openapi/v1-generated.yaml index 4b028c315f32e..d7ee6340c6680 100644 --- a/airflow/api_fastapi/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -1,6 +1,6 @@ openapi: 3.1.0 info: - title: FastAPI + title: Airflow API description: Airflow API. All endpoints located under ``/public`` can be used safely, are stable and backward compatible. Endpoints located under ``/ui`` are dedicated to the UI and are subject to breaking change depending on the need of the frontend. diff --git a/tests/api_fastapi/routes/__init__.py b/airflow/api_fastapi/core_api/routes/__init__.py similarity index 100% rename from tests/api_fastapi/routes/__init__.py rename to airflow/api_fastapi/core_api/routes/__init__.py diff --git a/airflow/api_fastapi/routes/public/__init__.py b/airflow/api_fastapi/core_api/routes/public/__init__.py similarity index 70% rename from airflow/api_fastapi/routes/public/__init__.py rename to airflow/api_fastapi/core_api/routes/public/__init__.py index 01a9b7f239621..3d43a7bbb0efb 100644 --- a/airflow/api_fastapi/routes/public/__init__.py +++ b/airflow/api_fastapi/core_api/routes/public/__init__.py @@ -17,12 +17,12 @@ from __future__ import annotations -from airflow.api_fastapi.routes.public.connections import connections_router -from airflow.api_fastapi.routes.public.dag_run import dag_run_router -from airflow.api_fastapi.routes.public.dags import dags_router -from airflow.api_fastapi.routes.public.monitor import monitor_router -from airflow.api_fastapi.routes.public.variables import variables_router -from airflow.api_fastapi.routes.router import AirflowRouter +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.routes.public.connections import connections_router +from airflow.api_fastapi.core_api.routes.public.dag_run import dag_run_router +from airflow.api_fastapi.core_api.routes.public.dags import dags_router +from airflow.api_fastapi.core_api.routes.public.monitor import monitor_router +from airflow.api_fastapi.core_api.routes.public.variables import variables_router public_router = AirflowRouter(prefix="/public") diff --git a/airflow/api_fastapi/routes/public/connections.py b/airflow/api_fastapi/core_api/routes/public/connections.py similarity index 87% rename from airflow/api_fastapi/routes/public/connections.py rename to airflow/api_fastapi/core_api/routes/public/connections.py index 76684835cf481..60898d2a63a6f 100644 --- a/airflow/api_fastapi/routes/public/connections.py +++ b/airflow/api_fastapi/core_api/routes/public/connections.py @@ -21,11 +21,14 @@ from sqlalchemy.orm import Session from typing_extensions import Annotated -from airflow.api_fastapi.db.common import get_session, paginated_select -from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.parameters import QueryLimit, QueryOffset, SortParam -from airflow.api_fastapi.routes.router import AirflowRouter -from airflow.api_fastapi.serializers.connections import ConnectionCollectionResponse, ConnectionResponse +from airflow.api_fastapi.common.db.common import get_session, paginated_select +from airflow.api_fastapi.common.parameters import QueryLimit, QueryOffset, SortParam +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.core_api.serializers.connections import ( + ConnectionCollectionResponse, + ConnectionResponse, +) from airflow.models import Connection connections_router = AirflowRouter(tags=["Connection"], prefix="/connections") diff --git a/airflow/api_fastapi/routes/public/dag_run.py b/airflow/api_fastapi/core_api/routes/public/dag_run.py similarity index 88% rename from airflow/api_fastapi/routes/public/dag_run.py rename to airflow/api_fastapi/core_api/routes/public/dag_run.py index 62e4d97a829bf..035d1b7fd7dc2 100644 --- a/airflow/api_fastapi/routes/public/dag_run.py +++ b/airflow/api_fastapi/core_api/routes/public/dag_run.py @@ -22,10 +22,10 @@ from sqlalchemy.orm import Session from typing_extensions import Annotated -from airflow.api_fastapi.db.common import get_session -from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.routes.router import AirflowRouter -from airflow.api_fastapi.serializers.dag_run import DAGRunResponse +from airflow.api_fastapi.common.db.common import get_session +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.core_api.serializers.dag_run import DAGRunResponse from airflow.models import DagRun dag_run_router = AirflowRouter(tags=["DagRun"], prefix="/dags/{dag_id}/dagRuns") diff --git a/airflow/api_fastapi/routes/public/dags.py b/airflow/api_fastapi/core_api/routes/public/dags.py similarity index 94% rename from airflow/api_fastapi/routes/public/dags.py rename to airflow/api_fastapi/core_api/routes/public/dags.py index d2b608e05176f..81293211fc92e 100644 --- a/airflow/api_fastapi/routes/public/dags.py +++ b/airflow/api_fastapi/core_api/routes/public/dags.py @@ -23,13 +23,12 @@ from typing_extensions import Annotated from airflow.api.common import delete_dag as delete_dag_module -from airflow.api_fastapi.db.common import ( +from airflow.api_fastapi.common.db.common import ( get_session, paginated_select, ) -from airflow.api_fastapi.db.dags import dags_select_with_latest_dag_run -from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.parameters import ( +from airflow.api_fastapi.common.db.dags import dags_select_with_latest_dag_run +from airflow.api_fastapi.common.parameters import ( QueryDagDisplayNamePatternSearch, QueryDagIdPatternSearch, QueryDagIdPatternSearchWithNone, @@ -42,8 +41,9 @@ QueryTagsFilter, SortParam, ) -from airflow.api_fastapi.routes.router import AirflowRouter -from airflow.api_fastapi.serializers.dags import ( +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.core_api.serializers.dags import ( DAGCollectionResponse, DAGDetailsResponse, DAGPatchBody, diff --git a/airflow/api_fastapi/routes/public/monitor.py b/airflow/api_fastapi/core_api/routes/public/monitor.py similarity index 89% rename from airflow/api_fastapi/routes/public/monitor.py rename to airflow/api_fastapi/core_api/routes/public/monitor.py index 30e8f1330a58b..e5d746e15c240 100644 --- a/airflow/api_fastapi/routes/public/monitor.py +++ b/airflow/api_fastapi/core_api/routes/public/monitor.py @@ -18,8 +18,8 @@ from __future__ import annotations from airflow.api.common.airflow_health import get_airflow_health -from airflow.api_fastapi.routes.router import AirflowRouter -from airflow.api_fastapi.serializers.monitor import HealthInfoSchema +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.serializers.monitor import HealthInfoSchema monitor_router = AirflowRouter(tags=["Monitor"], prefix="/monitor") diff --git a/airflow/api_fastapi/routes/public/variables.py b/airflow/api_fastapi/core_api/routes/public/variables.py similarity index 92% rename from airflow/api_fastapi/routes/public/variables.py rename to airflow/api_fastapi/core_api/routes/public/variables.py index 61cd84c700453..3a46c519c9234 100644 --- a/airflow/api_fastapi/routes/public/variables.py +++ b/airflow/api_fastapi/core_api/routes/public/variables.py @@ -21,10 +21,10 @@ from sqlalchemy.orm import Session from typing_extensions import Annotated -from airflow.api_fastapi.db.common import get_session -from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.routes.router import AirflowRouter -from airflow.api_fastapi.serializers.variables import VariableBody, VariableResponse +from airflow.api_fastapi.common.db.common import get_session +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.core_api.serializers.variables import VariableBody, VariableResponse from airflow.models.variable import Variable variables_router = AirflowRouter(tags=["Variable"], prefix="/variables") diff --git a/airflow/api_fastapi/routes/ui/__init__.py b/airflow/api_fastapi/core_api/routes/ui/__init__.py similarity index 81% rename from airflow/api_fastapi/routes/ui/__init__.py rename to airflow/api_fastapi/core_api/routes/ui/__init__.py index 2087549c98ea0..9cd16fcdd16b3 100644 --- a/airflow/api_fastapi/routes/ui/__init__.py +++ b/airflow/api_fastapi/core_api/routes/ui/__init__.py @@ -16,9 +16,9 @@ # under the License. from __future__ import annotations -from airflow.api_fastapi.routes.router import AirflowRouter -from airflow.api_fastapi.routes.ui.assets import assets_router -from airflow.api_fastapi.routes.ui.dashboard import dashboard_router +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.routes.ui.assets import assets_router +from airflow.api_fastapi.core_api.routes.ui.dashboard import dashboard_router ui_router = AirflowRouter(prefix="/ui") diff --git a/airflow/api_fastapi/routes/ui/assets.py b/airflow/api_fastapi/core_api/routes/ui/assets.py similarity index 96% rename from airflow/api_fastapi/routes/ui/assets.py rename to airflow/api_fastapi/core_api/routes/ui/assets.py index 31406b1b94db1..b8a17c7398424 100644 --- a/airflow/api_fastapi/routes/ui/assets.py +++ b/airflow/api_fastapi/core_api/routes/ui/assets.py @@ -22,8 +22,8 @@ from sqlalchemy.orm import Session from typing_extensions import Annotated -from airflow.api_fastapi.db.common import get_session -from airflow.api_fastapi.routes.router import AirflowRouter +from airflow.api_fastapi.common.db.common import get_session +from airflow.api_fastapi.common.router import AirflowRouter from airflow.models import DagModel from airflow.models.asset import AssetDagRunQueue, AssetEvent, AssetModel, DagScheduleAssetReference diff --git a/airflow/api_fastapi/routes/ui/dashboard.py b/airflow/api_fastapi/core_api/routes/ui/dashboard.py similarity index 90% rename from airflow/api_fastapi/routes/ui/dashboard.py rename to airflow/api_fastapi/core_api/routes/ui/dashboard.py index e7cec5fbf0ee1..e101ca78be7d9 100644 --- a/airflow/api_fastapi/routes/ui/dashboard.py +++ b/airflow/api_fastapi/core_api/routes/ui/dashboard.py @@ -23,17 +23,17 @@ from sqlalchemy.orm import Session from typing_extensions import Annotated -from airflow.api_fastapi.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.parameters import DateTimeQuery -from airflow.api_fastapi.serializers.dashboard import HistoricalMetricDataResponse +from airflow.api_fastapi.common.parameters import DateTimeQuery +from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.core_api.serializers.dashboard import HistoricalMetricDataResponse from airflow.models.dagrun import DagRun, DagRunType from airflow.models.taskinstance import TaskInstance from airflow.utils.state import DagRunState, TaskInstanceState if TYPE_CHECKING: from sqlalchemy.orm import Session -from airflow.api_fastapi.db.common import get_session -from airflow.api_fastapi.routes.router import AirflowRouter +from airflow.api_fastapi.common.db.common import get_session +from airflow.api_fastapi.common.router import AirflowRouter from airflow.utils import timezone dashboard_router = AirflowRouter(tags=["Dashboard"]) diff --git a/tests/api_fastapi/routes/public/__init__.py b/airflow/api_fastapi/core_api/serializers/__init__.py similarity index 100% rename from tests/api_fastapi/routes/public/__init__.py rename to airflow/api_fastapi/core_api/serializers/__init__.py diff --git a/airflow/api_fastapi/serializers/connections.py b/airflow/api_fastapi/core_api/serializers/connections.py similarity index 100% rename from airflow/api_fastapi/serializers/connections.py rename to airflow/api_fastapi/core_api/serializers/connections.py diff --git a/airflow/api_fastapi/serializers/dag_run.py b/airflow/api_fastapi/core_api/serializers/dag_run.py similarity index 100% rename from airflow/api_fastapi/serializers/dag_run.py rename to airflow/api_fastapi/core_api/serializers/dag_run.py diff --git a/airflow/api_fastapi/serializers/dags.py b/airflow/api_fastapi/core_api/serializers/dags.py similarity index 100% rename from airflow/api_fastapi/serializers/dags.py rename to airflow/api_fastapi/core_api/serializers/dags.py diff --git a/airflow/api_fastapi/serializers/dashboard.py b/airflow/api_fastapi/core_api/serializers/dashboard.py similarity index 100% rename from airflow/api_fastapi/serializers/dashboard.py rename to airflow/api_fastapi/core_api/serializers/dashboard.py diff --git a/airflow/api_fastapi/serializers/monitor.py b/airflow/api_fastapi/core_api/serializers/monitor.py similarity index 100% rename from airflow/api_fastapi/serializers/monitor.py rename to airflow/api_fastapi/core_api/serializers/monitor.py diff --git a/airflow/api_fastapi/serializers/variables.py b/airflow/api_fastapi/core_api/serializers/variables.py similarity index 100% rename from airflow/api_fastapi/serializers/variables.py rename to airflow/api_fastapi/core_api/serializers/variables.py diff --git a/airflow/ui/package.json b/airflow/ui/package.json index a7cf90bb5731b..c54c9af5bb203 100644 --- a/airflow/ui/package.json +++ b/airflow/ui/package.json @@ -11,7 +11,7 @@ "lint:fix": "eslint --fix && tsc --p tsconfig.app.json", "format": "pnpm prettier --write .", "preview": "vite preview", - "codegen": "openapi-rq -i \"../api_fastapi/openapi/v1-generated.yaml\" -c axios --format prettier -o openapi-gen --operationId", + "codegen": "openapi-rq -i \"../api_fastapi/core_api/openapi/v1-generated.yaml\" -c axios --format prettier -o openapi-gen --operationId", "test": "vitest run", "coverage": "vitest run --coverage" }, diff --git a/contributing-docs/17_adding_api_endpoints.rst b/contributing-docs/17_adding_api_endpoints.rst index 2a9ef9af7b0f2..05dccc72f1840 100644 --- a/contributing-docs/17_adding_api_endpoints.rst +++ b/contributing-docs/17_adding_api_endpoints.rst @@ -26,7 +26,7 @@ This documentation outlines the steps required to add a new API endpoint in Apac Introduction ------------ -The source code for the RestAPI is located under ``api_fastapi``. Endpoints are located under ``api_fastapi/views`` and contains different types of endpoints. The main two are ``public`` and ``ui``. +The source code for the RestAPI is located under ``api_fastapi/core_api``. Endpoints are located under ``api_fastapi/core_api/routes`` and contains different types of endpoints. The main two are ``public`` and ``ui``. Public endpoints are part of the public API, standardized, well documented and most importantly backward compatible. UI endpoints are custom endpoints made for the frontend that do not respect backward compatibility i.e they can be adapted at any time for UI needs. When adding an endpoint you should try as much as possible to make it reusable by the community, have a stable design in mind, standardized and therefore part of the public API. If this is not possible because the data types are too specific or subject to frequent change then adding it to the UI endpoints is more suitable. @@ -35,7 +35,7 @@ then adding it to the UI endpoints is more suitable. Step 1: Implement the Endpoint Logic ------------------------------------ 1. Considering the details above decide whether your endpoints should be part of the ``public`` or ``ui`` interface. -1. Navigate to the appropriate views directory in ``api_fastapi/views``. +1. Navigate to the appropriate routes directory in ``api_fastapi/core_api/routes``. 2. Register a new route for your endpoint with the appropriate HTTP method, query params, permissions, body type, etc. 3. Specify the appropriate Pydantic type in the return type annotation. @@ -62,7 +62,7 @@ Step 2: Add tests for your Endpoints ------------------------------------ 1. Verify manually with a local API that the endpoint behaves as expected. 2. Go to the test folder and initialize new tests. -3. Implements extensives tests to validate query param, permissions, error handling etc. +3. Implements extensive tests to validate query param, permissions, error handling etc. Step 3: Documentation diff --git a/scripts/in_container/run_update_fastapi_api_spec.py b/scripts/in_container/run_update_fastapi_api_spec.py index 5d31b0bee3f0c..b66587f720346 100644 --- a/scripts/in_container/run_update_fastapi_api_spec.py +++ b/scripts/in_container/run_update_fastapi_api_spec.py @@ -23,7 +23,7 @@ app = create_app() -OPENAPI_SPEC_FILE = "airflow/api_fastapi/openapi/v1-generated.yaml" +OPENAPI_SPEC_FILE = "airflow/api_fastapi/core_api/openapi/v1-generated.yaml" # The persisted openapi spec will list all endpoints (public and ui), this diff --git a/tests/api_fastapi/routes/ui/__init__.py b/tests/api_fastapi/core_api/__init__.py similarity index 100% rename from tests/api_fastapi/routes/ui/__init__.py rename to tests/api_fastapi/core_api/__init__.py diff --git a/tests/api_fastapi/core_api/routes/__init__.py b/tests/api_fastapi/core_api/routes/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/api_fastapi/core_api/routes/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/api_fastapi/core_api/routes/public/__init__.py b/tests/api_fastapi/core_api/routes/public/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/api_fastapi/core_api/routes/public/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/api_fastapi/routes/public/test_connections.py b/tests/api_fastapi/core_api/routes/public/test_connections.py similarity index 100% rename from tests/api_fastapi/routes/public/test_connections.py rename to tests/api_fastapi/core_api/routes/public/test_connections.py diff --git a/tests/api_fastapi/routes/public/test_dag_run.py b/tests/api_fastapi/core_api/routes/public/test_dag_run.py similarity index 100% rename from tests/api_fastapi/routes/public/test_dag_run.py rename to tests/api_fastapi/core_api/routes/public/test_dag_run.py diff --git a/tests/api_fastapi/routes/public/test_dags.py b/tests/api_fastapi/core_api/routes/public/test_dags.py similarity index 98% rename from tests/api_fastapi/routes/public/test_dags.py rename to tests/api_fastapi/core_api/routes/public/test_dags.py index 99ac939406f78..1055000f42562 100644 --- a/tests/api_fastapi/routes/public/test_dags.py +++ b/tests/api_fastapi/core_api/routes/public/test_dags.py @@ -293,7 +293,7 @@ def test_dag_details( "description": None, "doc_md": "details", "end_date": None, - "fileloc": "/opt/airflow/tests/api_fastapi/routes/public/test_dags.py", + "fileloc": "/opt/airflow/tests/api_fastapi/core_api/routes/public/test_dags.py", "file_token": file_token, "has_import_errors": False, "has_task_concurrency_limits": True, @@ -358,7 +358,7 @@ def test_get_dag(self, test_client, query_params, dag_id, expected_status_code, "dag_id": dag_id, "dag_display_name": dag_display_name, "description": None, - "fileloc": "/opt/airflow/tests/api_fastapi/routes/public/test_dags.py", + "fileloc": "/opt/airflow/tests/api_fastapi/core_api/routes/public/test_dags.py", "file_token": file_token, "is_paused": False, "is_active": True, diff --git a/tests/api_fastapi/routes/public/test_monitor.py b/tests/api_fastapi/core_api/routes/public/test_monitor.py similarity index 100% rename from tests/api_fastapi/routes/public/test_monitor.py rename to tests/api_fastapi/core_api/routes/public/test_monitor.py diff --git a/tests/api_fastapi/routes/public/test_variables.py b/tests/api_fastapi/core_api/routes/public/test_variables.py similarity index 100% rename from tests/api_fastapi/routes/public/test_variables.py rename to tests/api_fastapi/core_api/routes/public/test_variables.py diff --git a/tests/api_fastapi/core_api/routes/ui/__init__.py b/tests/api_fastapi/core_api/routes/ui/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/api_fastapi/core_api/routes/ui/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/api_fastapi/routes/ui/test_assets.py b/tests/api_fastapi/core_api/routes/ui/test_assets.py similarity index 100% rename from tests/api_fastapi/routes/ui/test_assets.py rename to tests/api_fastapi/core_api/routes/ui/test_assets.py diff --git a/tests/api_fastapi/routes/ui/test_dashboard.py b/tests/api_fastapi/core_api/routes/ui/test_dashboard.py similarity index 100% rename from tests/api_fastapi/routes/ui/test_dashboard.py rename to tests/api_fastapi/core_api/routes/ui/test_dashboard.py From 1be768e597bf437567d45bc0d2560ac4a65e639f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 16 Oct 2024 21:11:13 +0800 Subject: [PATCH 145/199] Tweak AssetAlias to match Asset for AIP-74 additions (#42814) --- airflow/assets/__init__.py | 13 +- ..._0_tweak_assetaliasmodel_to_match_asset.py | 76 ++ airflow/models/asset.py | 17 +- airflow/utils/db.py | 2 +- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 821 +++++++++--------- docs/apache-airflow/migrations-ref.rst | 4 +- 7 files changed, 518 insertions(+), 417 deletions(-) create mode 100644 airflow/migrations/versions/0038_3_0_0_tweak_assetaliasmodel_to_match_asset.py diff --git a/airflow/assets/__init__.py b/airflow/assets/__init__.py index 1580541847272..4dbc35fb95397 100644 --- a/airflow/assets/__init__.py +++ b/airflow/assets/__init__.py @@ -227,6 +227,11 @@ class AssetAlias(BaseAsset): """A represeation of asset alias which is used to create asset during the runtime.""" name: str = attr.field(validator=_validate_non_empty_identifier) + group: str = attr.field( + kw_only=True, + default="", + validator=[attr.validators.max_len(1500), _validate_identifier], + ) def iter_assets(self) -> Iterator[tuple[str, Asset]]: return iter(()) @@ -272,10 +277,10 @@ def _set_extra_default(extra: dict | None) -> dict: class Asset(os.PathLike, BaseAsset): """A representation of data dependencies between workflows.""" - name: str = attr.field() - uri: str = attr.field() - group: str = attr.field() - extra: dict[str, Any] = attr.field() + name: str + uri: str + group: str + extra: dict[str, Any] __version__: ClassVar[int] = 1 diff --git a/airflow/migrations/versions/0038_3_0_0_tweak_assetaliasmodel_to_match_asset.py b/airflow/migrations/versions/0038_3_0_0_tweak_assetaliasmodel_to_match_asset.py new file mode 100644 index 0000000000000..f1b57974f053c --- /dev/null +++ b/airflow/migrations/versions/0038_3_0_0_tweak_assetaliasmodel_to_match_asset.py @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Tweak AssetAliasModel to match AssetModel after AIP-76. + +This involves two changes: + +1. Add the 'group' column. +2. Reduce the 'name' column to 1500 characters. + +The first is straightforward. The second is technically not necessary (the alias +model does not need an extra field for uniqueness), but it is probably better +for the two 'name' fields to have matching behavior, to reduce potential user +confusion. + +Hopefully nobody would notice or be affected by this anyway since 1500 +characters is plenty enough. + +Revision ID: fb2d4922cd79 +Revises: 5a5d66100783 +Create Date: 2024-10-08 01:46:05.556368 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +# Revision identifiers, used by Alembic. +revision = "fb2d4922cd79" +down_revision = "5a5d66100783" +branch_labels = None +depends_on = None +airflow_version = "3.0.0" + +_STRING_COLUMN_TYPE = sa.String(length=1500).with_variant( + sa.String(length=1500, collation="latin1_general_cs"), + dialect_name="mysql", +) + + +def upgrade(): + """Tweak AssetAliasModel to match AssetModel.""" + with op.batch_alter_table("dataset_alias", schema=None) as batch_op: + batch_op.alter_column("name", type_=_STRING_COLUMN_TYPE, nullable=False) + batch_op.add_column(sa.Column("group", _STRING_COLUMN_TYPE, default=str, nullable=False)) + + +def downgrade(): + """Untweak AssetAliasModel to match AssetModel.""" + with op.batch_alter_table("dataset_alias", schema=None) as batch_op: + batch_op.drop_column("group") + batch_op.alter_column( + "name", + type_=sa.String(length=3000).with_variant( + sa.String(length=3000, collation="latin1_general_cs"), + dialect_name="mysql", + ), + nullable=False, + ) diff --git a/airflow/models/asset.py b/airflow/models/asset.py index b565c9a100ef7..79f9b7389439d 100644 --- a/airflow/models/asset.py +++ b/airflow/models/asset.py @@ -91,9 +91,9 @@ class AssetAliasModel(Base): id = Column(Integer, primary_key=True, autoincrement=True) name = Column( - String(length=3000).with_variant( + String(length=1500).with_variant( String( - length=3000, + length=1500, # latin1 allows for more indexed length in mysql # and this field should only be ascii chars collation="latin1_general_cs", @@ -102,6 +102,19 @@ class AssetAliasModel(Base): ), nullable=False, ) + group = Column( + String(length=1500).with_variant( + String( + length=1500, + # latin1 allows for more indexed length in mysql + # and this field should only be ascii chars + collation="latin1_general_cs", + ), + "mysql", + ), + default=str, + nullable=False, + ) __tablename__ = "dataset_alias" __table_args__ = ( diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 26af566c7e2bd..2fba821da6618 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -96,7 +96,7 @@ class MappedClassProtocol(Protocol): "2.9.0": "1949afb29106", "2.9.2": "686269002441", "2.10.0": "22ed7efa9da2", - "3.0.0": "5a5d66100783", + "3.0.0": "fb2d4922cd79", } diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index a751eca916568..ac9ab5b14d6cd 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -0ed26236c783f7524416c1377638fe18ff3520bd355160db48656585ff58524e \ No newline at end of file +b39a456ac1bd3aa91198d036eaa3859f25a47151164974f6b4fe7f998d41f720 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 73452a069c040..833f2f73bbb3e 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -4,11 +4,11 @@ - - + + %3 - + log @@ -433,18 +433,23 @@ dataset_alias - -dataset_alias - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL + +dataset_alias + +id + + [INTEGER] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL @@ -465,16 +470,16 @@ dataset_alias--dataset_alias_dataset - + 0..N -1 +1 dataset_alias--dataset_alias_dataset - + 0..N -1 +1 @@ -495,16 +500,16 @@ dataset_alias--dataset_alias_dataset_event - -0..N -1 + +0..N +1 dataset_alias--dataset_alias_dataset_event - -0..N -1 + +0..N +1 @@ -535,9 +540,9 @@ dataset_alias--dag_schedule_dataset_alias_reference - -0..N -1 + +0..N +1 @@ -691,9 +696,9 @@ dataset--task_outlet_dataset_reference - -0..N -1 + +0..N +1 @@ -719,9 +724,9 @@ dataset--dataset_dag_run_queue - -0..N -1 + +0..N +1 @@ -805,144 +810,144 @@ dag - -dag - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_display_name - - [VARCHAR(2000)] - -dataset_expression - - [JSON] - -default_view - - [VARCHAR(25)] - -description - - [TEXT] - -fileloc - - [VARCHAR(2000)] - -has_import_errors - - [BOOLEAN] - -has_task_concurrency_limits - - [BOOLEAN] - NOT NULL - -is_active - - [BOOLEAN] - -is_paused - - [BOOLEAN] - -last_expired - - [TIMESTAMP] - -last_parsed_time - - [TIMESTAMP] - -last_pickled - - [TIMESTAMP] - -max_active_runs - - [INTEGER] - -max_active_tasks - - [INTEGER] - NOT NULL - -max_consecutive_failed_dag_runs - - [INTEGER] - NOT NULL - -next_dagrun - - [TIMESTAMP] - -next_dagrun_create_after - - [TIMESTAMP] - -next_dagrun_data_interval_end - - [TIMESTAMP] - -next_dagrun_data_interval_start - - [TIMESTAMP] - -owners - - [VARCHAR(2000)] - -pickle_id - - [INTEGER] - -processor_subdir - - [VARCHAR(2000)] - -scheduler_lock - - [BOOLEAN] - -timetable_description - - [VARCHAR(1000)] - -timetable_summary - - [TEXT] + +dag + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_display_name + + [VARCHAR(2000)] + +dataset_expression + + [JSON] + +default_view + + [VARCHAR(25)] + +description + + [TEXT] + +fileloc + + [VARCHAR(2000)] + +has_import_errors + + [BOOLEAN] + +has_task_concurrency_limits + + [BOOLEAN] + NOT NULL + +is_active + + [BOOLEAN] + +is_paused + + [BOOLEAN] + +last_expired + + [TIMESTAMP] + +last_parsed_time + + [TIMESTAMP] + +last_pickled + + [TIMESTAMP] + +max_active_runs + + [INTEGER] + +max_active_tasks + + [INTEGER] + NOT NULL + +max_consecutive_failed_dag_runs + + [INTEGER] + NOT NULL + +next_dagrun + + [TIMESTAMP] + +next_dagrun_create_after + + [TIMESTAMP] + +next_dagrun_data_interval_end + + [TIMESTAMP] + +next_dagrun_data_interval_start + + [TIMESTAMP] + +owners + + [VARCHAR(2000)] + +pickle_id + + [INTEGER] + +processor_subdir + + [VARCHAR(2000)] + +scheduler_lock + + [BOOLEAN] + +timetable_description + + [VARCHAR(1000)] + +timetable_summary + + [TEXT] dag--dag_schedule_dataset_alias_reference - -0..N -1 + +0..N +1 dag--dag_schedule_dataset_reference - -0..N -1 + +0..N +1 dag--task_outlet_dataset_reference - -0..N -1 + +0..N +1 dag--dataset_dag_run_queue - -0..N -1 + +0..N +1 @@ -963,9 +968,9 @@ dag--dag_tag - -0..N -1 + +0..N +1 @@ -991,9 +996,9 @@ dag--dag_owner_attributes - -0..N -1 + +0..N +1 @@ -1024,9 +1029,9 @@ dag--dag_warning - -0..N -1 + +0..N +1 @@ -1467,14 +1472,14 @@ task_instance--task_reschedule -0..N +0..N 1 task_instance--task_reschedule -0..N +0..N 1 @@ -2107,318 +2112,318 @@ session - -session - -id - - [INTEGER] - NOT NULL - -data - - [BYTEA] - -expiry - - [TIMESTAMP] - -session_id - - [VARCHAR(255)] + +session + +id + + [INTEGER] + NOT NULL + +data + + [BYTEA] + +expiry + + [TIMESTAMP] + +session_id + + [VARCHAR(255)] alembic_version - -alembic_version - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version + +version_num + + [VARCHAR(32)] + NOT NULL ab_user - -ab_user - -id - - [INTEGER] - NOT NULL - -active - - [BOOLEAN] - -changed_by_fk - - [INTEGER] - -changed_on - - [TIMESTAMP] - -created_by_fk - - [INTEGER] - -created_on - - [TIMESTAMP] - -email - - [VARCHAR(512)] - NOT NULL - -fail_login_count - - [INTEGER] - -first_name - - [VARCHAR(256)] - NOT NULL - -last_login - - [TIMESTAMP] - -last_name - - [VARCHAR(256)] - NOT NULL - -login_count - - [INTEGER] - -password - - [VARCHAR(256)] - -username - - [VARCHAR(512)] - NOT NULL + +ab_user + +id + + [INTEGER] + NOT NULL + +active + + [BOOLEAN] + +changed_by_fk + + [INTEGER] + +changed_on + + [TIMESTAMP] + +created_by_fk + + [INTEGER] + +created_on + + [TIMESTAMP] + +email + + [VARCHAR(512)] + NOT NULL + +fail_login_count + + [INTEGER] + +first_name + + [VARCHAR(256)] + NOT NULL + +last_login + + [TIMESTAMP] + +last_name + + [VARCHAR(256)] + NOT NULL + +login_count + + [INTEGER] + +password + + [VARCHAR(256)] + +username + + [VARCHAR(512)] + NOT NULL ab_user--ab_user - -0..N -{0,1} + +0..N +{0,1} ab_user--ab_user - -0..N -{0,1} + +0..N +{0,1} ab_user_role - -ab_user_role - -id - - [INTEGER] - NOT NULL - -role_id - - [INTEGER] - -user_id - - [INTEGER] + +ab_user_role + +id + + [INTEGER] + NOT NULL + +role_id + + [INTEGER] + +user_id + + [INTEGER] ab_user--ab_user_role - -0..N -{0,1} + +0..N +{0,1} ab_register_user - -ab_register_user - -id - - [INTEGER] - NOT NULL - -email - - [VARCHAR(512)] - NOT NULL - -first_name - - [VARCHAR(256)] - NOT NULL - -last_name - - [VARCHAR(256)] - NOT NULL - -password - - [VARCHAR(256)] - -registration_date - - [TIMESTAMP] - -registration_hash - - [VARCHAR(256)] - -username - - [VARCHAR(512)] - NOT NULL + +ab_register_user + +id + + [INTEGER] + NOT NULL + +email + + [VARCHAR(512)] + NOT NULL + +first_name + + [VARCHAR(256)] + NOT NULL + +last_name + + [VARCHAR(256)] + NOT NULL + +password + + [VARCHAR(256)] + +registration_date + + [TIMESTAMP] + +registration_hash + + [VARCHAR(256)] + +username + + [VARCHAR(512)] + NOT NULL ab_permission - -ab_permission - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(100)] - NOT NULL + +ab_permission + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(100)] + NOT NULL ab_permission_view - -ab_permission_view - -id - - [INTEGER] - NOT NULL - -permission_id - - [INTEGER] - -view_menu_id - - [INTEGER] + +ab_permission_view + +id + + [INTEGER] + NOT NULL + +permission_id + + [INTEGER] + +view_menu_id + + [INTEGER] ab_permission--ab_permission_view - -0..N -{0,1} + +0..N +{0,1} ab_permission_view_role - -ab_permission_view_role - -id - - [INTEGER] - NOT NULL - -permission_view_id - - [INTEGER] - -role_id - - [INTEGER] + +ab_permission_view_role + +id + + [INTEGER] + NOT NULL + +permission_view_id + + [INTEGER] + +role_id + + [INTEGER] ab_permission_view--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} ab_view_menu - -ab_view_menu - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(250)] - NOT NULL + +ab_view_menu + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(250)] + NOT NULL ab_view_menu--ab_permission_view - -0..N -{0,1} + +0..N +{0,1} ab_role - -ab_role - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(64)] - NOT NULL + +ab_role + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(64)] + NOT NULL ab_role--ab_user_role - -0..N -{0,1} + +0..N +{0,1} ab_role--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} alembic_version_fab - -alembic_version_fab - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version_fab + +version_num + + [VARCHAR(32)] + NOT NULL diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index f917717d49efe..c4db207712e12 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``5a5d66100783`` (head) | ``c3389cd7793f`` | ``3.0.0`` | Add AssetActive to track orphaning instead of a flag. | +| ``fb2d4922cd79`` (head) | ``5a5d66100783`` | ``3.0.0`` | Tweak AssetAliasModel to match AssetModel after AIP-76. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``5a5d66100783`` | ``c3389cd7793f`` | ``3.0.0`` | Add AssetActive to track orphaning instead of a flag. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``c3389cd7793f`` | ``0d9e73a75ee4`` | ``3.0.0`` | Add backfill to dag run model. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ From 1f1c90ef1b252087630dcfdfe1c554f6059eb6b4 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 16 Oct 2024 16:01:01 +0100 Subject: [PATCH 146/199] Fix pytest from working outside breeze (#43082) This was missed in https://github.com/apache/airflow/pull/42985 . Without this `airflow.providers.__path__` had 2 registered paths: ``` ['/Users/kaxilnaik/Documents/GitHub/astronomer/airflow/providers/src/airflow/providers', '/Users/kaxilnaik/Documents/GitHub/astronomer/airflow/providers'] ``` This prevents the tests from running outside of breeze and we get the following error: ``` ERROR tests/core/test_settings.py::test_usage_data_collection_disabled[true-True-True] - airflow.exceptions.AirflowConfigException: ("The provider apache-airflow-providers-src-airflow-providers-amazon is attempting to contribute configuration section aws that has already been added before. The source of it: apache-airflow-providers-amazon. This is forbidden. A provider can only add new sections. It cannot contribute options to existing sections or override other provider's configuration.", ) ``` We get this error because the *Providers Manager* uses `airflow.providers.__path__` to register providers. Because we have 2 paths, it registers the same provider twice leading two the above error. https://github.com/apache/airflow/blob/75b22940ac4d36c31380669da2aa32fe46d70d32/airflow/providers_manager.py#L662 Example registration: ``` ('apache-airflow-providers-src-airflow-providers-yandex', {'yandex': {'description': 'This section contains settings for Yandex Cloud ', ... ('apache-airflow-providers-yandex', {'yandex': {'description': 'This section contains settings for Yandex Cloud ' ``` This wasn't a problem in breeze as it sets `AIRFLOW_SOURCES` env var in Dockerfile https://github.com/apache/airflow/blob/75b22940ac4d36c31380669da2aa32fe46d70d32/scripts/docker/entrypoint_ci.sh#L24 --- tests_common/pytest_plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests_common/pytest_plugin.py b/tests_common/pytest_plugin.py index 60638d8a5a756..bf34ae5be5f4b 100644 --- a/tests_common/pytest_plugin.py +++ b/tests_common/pytest_plugin.py @@ -114,7 +114,7 @@ _airflow_sources = os.getenv("AIRFLOW_SOURCES", None) AIRFLOW_SOURCES_ROOT_DIR = ( - Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[2] + Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[1] ).resolve() AIRFLOW_TESTS_DIR = AIRFLOW_SOURCES_ROOT_DIR / "tests" @@ -376,6 +376,8 @@ def pytest_configure(config: pytest.Config) -> None: if path == desired: break else: + # This "desired" path should be the Airflow source directory (repo root) + assert (AIRFLOW_SOURCES_ROOT_DIR / ".asf.yaml").exists(), f"Path {desired} is not Airflow root" sys.path.append(desired) if (backend := config.getoption("backend", default=None)) and backend not in SUPPORTED_DB_BACKENDS: From 943d5b9780a9a9fd0ece6d727e1ae62f95760c21 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:04:11 +0300 Subject: [PATCH 147/199] Fix typo in ``pyproject.toml`` (#43077) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a5dadf0e96651..71f93248bddd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -342,7 +342,7 @@ section-order = [ # Make sure we put the "dev" imports at the end, not as a third-party module [tool.ruff.lint.isort.sections] -testing = ["dev", "providers.tests", "task_sdk.tests", "test_common", "tests"] +testing = ["dev", "providers.tests", "task_sdk.tests", "tests_common", "tests"] [tool.ruff.lint.extend-per-file-ignores] "airflow/__init__.py" = ["F401", "TCH004"] From 67d28b470ba9218852636a896df7b5541385d2a2 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Wed, 16 Oct 2024 08:07:13 -0700 Subject: [PATCH 148/199] Remove DAG.get_num_active_runs (#43067) We don't need this function. There's already an almost-identical function on DagRun that we can use, namely DagRun.active_runs_of_dags. Also simplified that one. --- airflow/jobs/scheduler_job_runner.py | 13 ++++++------- airflow/models/dag.py | 22 ---------------------- airflow/models/dagrun.py | 25 ++++++++++++------------- newsfragments/43067.significant.rst | 4 ++++ tests/jobs/test_scheduler_job.py | 4 ++-- 5 files changed, 24 insertions(+), 44 deletions(-) create mode 100644 newsfragments/43067.significant.rst diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index 7958adb64cf96..f583cb414ece9 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -1337,9 +1337,8 @@ def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) - .all() ) - active_runs_of_dags = Counter( - DagRun.active_runs_of_dags(dag_ids=(dm.dag_id for dm in dag_models), session=session), - ) + dag_ids = (dm.dag_id for dm in dag_models) + active_runs_of_dags = Counter(DagRun.active_runs_of_dags(dag_ids=dag_ids, session=session)) for dag_model in dag_models: dag = self.dagbag.get_dag(dag_model.dag_id, session=session) @@ -1512,11 +1511,11 @@ def _should_update_dag_next_dagruns( if not dag.timetable.can_be_scheduled: return False - # get active dag runs from DB if not available - if not total_active_runs: - total_active_runs = dag.get_num_active_runs(only_running=False, session=session) + if total_active_runs is None: + runs_dict = DagRun.active_runs_of_dags(dag_ids=[dag.dag_id], session=session) + total_active_runs = runs_dict.get(dag.dag_id, 0) - if total_active_runs and total_active_runs >= dag.max_active_runs: + if total_active_runs >= dag.max_active_runs: self.log.info( "DAG %s is at (or above) max_active_runs (%d of %d), not creating any more runs", dag_model.dag_id, diff --git a/airflow/models/dag.py b/airflow/models/dag.py index f8d9f55e56e18..f0a7d7f56be2c 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -1283,28 +1283,6 @@ def get_active_runs(self): return active_dates - @provide_session - def get_num_active_runs(self, external_trigger=None, only_running=True, session=NEW_SESSION): - """ - Return the number of active "running" dag runs. - - :param external_trigger: True for externally triggered active dag runs - :param session: - :return: number greater than 0 for active dag runs - """ - query = select(func.count()).where(DagRun.dag_id == self.dag_id) - if only_running: - query = query.where(DagRun.state == DagRunState.RUNNING) - else: - query = query.where(DagRun.state.in_({DagRunState.RUNNING, DagRunState.QUEUED})) - - if external_trigger is not None: - query = query.where( - DagRun.external_trigger == (expression.true() if external_trigger else expression.false()) - ) - - return session.scalar(query) - @staticmethod @internal_api_call @provide_session diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index cad82e72b8b2d..4fd689d616b47 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -386,21 +386,20 @@ def refresh_from_db(self, session: Session = NEW_SESSION) -> None: @provide_session def active_runs_of_dags( cls, - dag_ids: Iterable[str] | None = None, - only_running: bool = False, + dag_ids: Iterable[str], session: Session = NEW_SESSION, ) -> dict[str, int]: - """Get the number of active dag runs for each dag.""" - query = select(cls.dag_id, func.count("*")) - if dag_ids is not None: - # 'set' called to avoid duplicate dag_ids, but converted back to 'list' - # because SQLAlchemy doesn't accept a set here. - query = query.where(cls.dag_id.in_(set(dag_ids))) - if only_running: - query = query.where(cls.state == DagRunState.RUNNING) - else: - query = query.where(cls.state.in_((DagRunState.RUNNING, DagRunState.QUEUED))) - query = query.group_by(cls.dag_id) + """ + Get the number of active dag runs for each dag. + + :meta private: + """ + query = ( + select(cls.dag_id, func.count("*")) + .where(cls.dag_id.in_(set(dag_ids))) + .where(cls.state.in_((DagRunState.RUNNING, DagRunState.QUEUED))) + .group_by(cls.dag_id) + ) return dict(iter(session.execute(query))) @classmethod diff --git a/newsfragments/43067.significant.rst b/newsfragments/43067.significant.rst new file mode 100644 index 0000000000000..d57fec6be9e72 --- /dev/null +++ b/newsfragments/43067.significant.rst @@ -0,0 +1,4 @@ +Remove DAG.get_num_active_runs + +We don't need this function. There's already an almost-identical function on DagRun that we can use, namely DagRun.active_runs_of_dags. +Also, make DagRun.active_runs_of_dags private. diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 01d1c5fe7a331..13ccfc74a0ffb 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -4477,14 +4477,14 @@ def complete_one_dagrun(): model: DagModel = session.get(DagModel, dag.dag_id) # Pre-condition - assert DagRun.active_runs_of_dags(session=session) == {"test_dag": 3} + assert DagRun.active_runs_of_dags(dag_ids=["test_dag"], session=session) == {"test_dag": 3} assert model.next_dagrun == timezone.DateTime(2016, 1, 3, tzinfo=UTC) assert model.next_dagrun_create_after is None complete_one_dagrun() - assert DagRun.active_runs_of_dags(session=session) == {"test_dag": 3} + assert DagRun.active_runs_of_dags(dag_ids=["test_dag"], session=session) == {"test_dag": 3} for _ in range(5): self.job_runner._do_scheduling(session) From 8876c04f99c0e09408666c8eb8e56aee2269c688 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Wed, 16 Oct 2024 23:41:32 +0800 Subject: [PATCH 149/199] Fix UI lint pre-commit hook (#43086) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 467f0e44d806b..29762c19136c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1159,7 +1159,7 @@ repos: description: TS types generation / ESLint / Prettier new UI files language: node types_or: [javascript, ts, tsx, yaml, css, json] - files: ^airflow/ui/|^airflow/api_fastapi/openapi/v1-generated\.yaml$ + files: ^airflow/ui/|^airflow/api_fastapi/core_api/openapi/v1-generated\.yaml$ entry: ./scripts/ci/pre_commit/lint_ui.py additional_dependencies: ['pnpm@9.7.1'] pass_filenames: false From 9f9bb8b77da2dea64b834cec292cb369af1aaa35 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 16 Oct 2024 16:45:51 +0100 Subject: [PATCH 150/199] Fixed failing static checks (#43087) For some reasons https://github.com/apache/airflow/pull/43077 passed but main was failing. This PR fixes it. --- .../alibaba/cloud/log/test_oss_task_handler.py | 5 +++-- .../amazon/aws/auth_manager/avp/test_facade.py | 3 ++- .../aws/auth_manager/cli/test_avp_commands.py | 5 +++-- .../test_aws_security_manager_override.py | 1 + .../aws/auth_manager/test_aws_auth_manager.py | 7 ++++--- .../amazon/aws/auth_manager/views/test_auth.py | 5 +++-- .../aws/executors/batch/test_batch_executor.py | 5 +++-- .../aws/executors/ecs/test_ecs_executor.py | 7 ++++--- .../tests/amazon/aws/hooks/test_base_aws.py | 3 ++- providers/tests/amazon/aws/hooks/test_s3.py | 3 ++- .../tests/amazon/aws/links/test_base_aws.py | 5 +++-- .../aws/log/test_cloudwatch_task_handler.py | 3 ++- .../tests/amazon/aws/log/test_s3_task_handler.py | 3 ++- .../amazon/aws/secrets/test_systems_manager.py | 3 ++- .../amazon/aws/transfers/test_redshift_to_s3.py | 3 ++- .../amazon/aws/transfers/test_s3_to_redshift.py | 3 ++- .../amazon/aws/transfers/test_s3_to_sftp.py | 3 ++- .../amazon/aws/transfers/test_sftp_to_s3.py | 3 ++- providers/tests/apache/hive/hooks/test_hive.py | 2 +- providers/tests/apache/livy/hooks/test_livy.py | 3 ++- .../tests/apache/spark/hooks/test_spark_sql.py | 3 ++- .../tests/atlassian/jira/hooks/test_jira.py | 3 ++- .../tests/atlassian/jira/operators/test_jira.py | 3 ++- .../tests/atlassian/jira/sensors/test_jira.py | 3 ++- .../tests/celery/cli/test_celery_command.py | 5 +++-- .../celery/executors/test_celery_executor.py | 7 ++++--- .../celery/log_handlers/test_log_handlers.py | 5 +++-- .../kubernetes/cli/test_kubernetes_command.py | 3 ++- .../executors/test_kubernetes_executor.py | 5 +++-- .../cncf/kubernetes/hooks/test_kubernetes.py | 5 +++-- .../kubernetes/log_handlers/test_log_handlers.py | 5 +++-- .../tests/cncf/kubernetes/operators/test_pod.py | 3 ++- .../operators/test_spark_kubernetes.py | 3 ++- providers/tests/cncf/kubernetes/test_client.py | 3 ++- .../cncf/kubernetes/test_template_rendering.py | 3 ++- .../common/io/operators/test_file_transfer.py | 4 ++-- providers/tests/common/io/xcom/test_backend.py | 7 ++++--- providers/tests/common/sql/hooks/test_dbapi.py | 3 ++- providers/tests/common/sql/hooks/test_sql.py | 2 +- .../tests/common/sql/hooks/test_sqlparse.py | 3 ++- providers/tests/common/sql/operators/test_sql.py | 3 ++- .../common/sql/operators/test_sql_execute.py | 3 ++- providers/tests/common/sql/sensors/test_sql.py | 3 ++- providers/tests/common/sql/test_utils.py | 3 ++- .../plugins/test_databricks_workflow.py | 3 ++- .../edge/api_endpoints/test_rpc_api_endpoint.py | 5 +++-- providers/tests/edge/cli/test_edge_command.py | 3 ++- .../edge/plugins/test_edge_executor_plugin.py | 3 ++- .../elasticsearch/log/test_es_task_handler.py | 4 ++-- .../api/auth/backend/test_basic_auth.py | 3 ++- .../api/auth/backend/test_session.py | 3 ++- .../api_endpoints/test_asset_endpoint.py | 6 +++--- .../fab/auth_manager/api_endpoints/test_auth.py | 1 + .../api_endpoints/test_backfill_endpoint.py | 14 +++++++------- .../fab/auth_manager/api_endpoints/test_cors.py | 1 + .../api_endpoints/test_dag_endpoint.py | 6 +++--- .../api_endpoints/test_dag_run_endpoint.py | 4 ++-- .../api_endpoints/test_dag_source_endpoint.py | 4 ++-- .../api_endpoints/test_dag_warning_endpoint.py | 4 ++-- .../api_endpoints/test_event_log_endpoint.py | 4 ++-- .../api_endpoints/test_import_error_endpoint.py | 6 +++--- .../test_role_and_permission_endpoint.py | 4 ++-- .../api_endpoints/test_task_instance_endpoint.py | 4 ++-- .../api_endpoints/test_user_endpoint.py | 6 +++--- .../api_endpoints/test_variable_endpoint.py | 4 ++-- .../api_endpoints/test_xcom_endpoint.py | 4 ++-- .../cli_commands/test_role_command.py | 5 +++-- .../cli_commands/test_sync_perm_command.py | 3 ++- .../cli_commands/test_user_command.py | 3 ++- .../fab/auth_manager/cli_commands/test_utils.py | 1 + providers/tests/fab/auth_manager/conftest.py | 5 +++-- .../fab/auth_manager/decorators/test_auth.py | 3 ++- .../fab/auth_manager/schemas/test_user_schema.py | 2 +- providers/tests/fab/auth_manager/test_models.py | 1 + .../tests/fab/auth_manager/test_security.py | 12 ++++++------ .../fab/auth_manager/views/test_permissions.py | 4 ++-- .../fab/auth_manager/views/test_roles_list.py | 4 ++-- .../tests/fab/auth_manager/views/test_user.py | 4 ++-- .../fab/auth_manager/views/test_user_edit.py | 4 ++-- .../fab/auth_manager/views/test_user_stats.py | 4 ++-- .../google/cloud/hooks/test_bigquery_system.py | 2 +- .../tests/google/cloud/hooks/test_kms_system.py | 2 +- .../cloud/hooks/test_secret_manager_system.py | 2 +- .../google/cloud/log/test_gcs_task_handler.py | 5 +++-- .../cloud/log/test_gcs_task_handler_system.py | 14 +++++++------- .../cloud/log/test_stackdriver_task_handler.py | 7 ++++--- .../log/test_stackdriver_task_handler_system.py | 14 +++++++------- .../google/cloud/operators/test_bigquery.py | 13 +++++++------ .../google/cloud/operators/test_bigquery_dts.py | 3 ++- .../cloud/operators/test_dataprep_system.py | 7 ++++--- .../google/cloud/operators/test_dataproc.py | 5 +++-- .../cloud/operators/test_datastore_system.py | 4 ++-- .../tests/google/cloud/operators/test_looker.py | 3 ++- .../cloud/secrets/test_secret_manager_system.py | 2 +- .../transfers/test_facebook_ads_to_gcs_system.py | 10 +++++----- .../transfers/test_salesforce_to_gcs_system.py | 8 ++++---- .../google/cloud/utils/gcp_authenticator.py | 6 +++--- .../common/auth_backend/test_google_openid.py | 5 +++-- .../operators/test_display_video_system.py | 12 ++++++------ providers/tests/hashicorp/hooks/test_vault.py | 3 ++- providers/tests/imap/hooks/test_imap.py | 3 ++- .../azure/log/test_wasb_task_handler.py | 5 +++-- providers/tests/mongo/hooks/test_mongo.py | 3 ++- providers/tests/mysql/hooks/test_mysql.py | 4 ++-- .../tests/openlineage/extractors/test_bash.py | 3 ++- .../tests/openlineage/extractors/test_manager.py | 3 ++- .../tests/openlineage/extractors/test_python.py | 3 ++- .../tests/openlineage/plugins/test_adapter.py | 5 +++-- .../tests/openlineage/plugins/test_execution.py | 5 +++-- .../tests/openlineage/plugins/test_listener.py | 5 +++-- .../openlineage/plugins/test_openlineage.py | 1 + .../tests/openlineage/plugins/test_utils.py | 3 ++- providers/tests/openlineage/test_conf.py | 3 ++- providers/tests/openlineage/utils/test_utils.py | 6 +++--- .../tests/opensearch/log/test_os_task_handler.py | 4 ++-- .../tests/redis/log/test_redis_task_handler.py | 3 ++- providers/tests/sftp/operators/test_sftp.py | 3 ++- providers/tests/smtp/hooks/test_smtp.py | 3 ++- providers/tests/smtp/notifications/test_smtp.py | 5 +++-- providers/tests/ssh/operators/test_ssh.py | 3 ++- providers/tests/standard/operators/test_bash.py | 3 ++- .../tests/standard/operators/test_datetime.py | 3 ++- .../tests/standard/operators/test_weekday.py | 3 ++- providers/tests/standard/sensors/test_weekday.py | 3 ++- .../tests/system/amazon/aws/example_s3_to_sql.py | 3 +-- .../amazon/aws/tests/test_aws_auth_manager.py | 4 ++-- .../tests/system/dbt/cloud/example_dbt_cloud.py | 4 ++-- .../tests/system/yandex/example_yandexcloud.py | 3 ++- .../yandex/example_yandexcloud_dataproc.py | 4 ++-- .../example_yandexcloud_dataproc_lightweight.py | 4 ++-- .../system/yandex/example_yandexcloud_yq.py | 4 ++-- providers/tests/yandex/hooks/test_yandex.py | 4 ++-- providers/tests/yandex/links/test_yq.py | 5 +++-- providers/tests/yandex/operators/test_yq.py | 1 + tests/always/test_connection.py | 1 + tests/always/test_example_dags.py | 1 + tests/always/test_secrets.py | 1 + tests/always/test_secrets_backends.py | 1 + tests/always/test_secrets_local_filesystem.py | 1 + tests/api_connexion/conftest.py | 1 + .../endpoints/test_asset_endpoint.py | 1 + .../endpoints/test_backfill_endpoint.py | 1 + .../endpoints/test_connection_endpoint.py | 1 + .../api_connexion/endpoints/test_dag_endpoint.py | 1 + .../api_connexion/endpoints/test_dag_parsing.py | 1 + .../endpoints/test_dag_run_endpoint.py | 1 + .../endpoints/test_dag_source_endpoint.py | 1 + .../endpoints/test_dag_stats_endpoint.py | 1 + .../endpoints/test_dag_warning_endpoint.py | 1 + .../endpoints/test_event_log_endpoint.py | 1 + .../endpoints/test_extra_link_endpoint.py | 1 + .../endpoints/test_import_error_endpoint.py | 1 + .../api_connexion/endpoints/test_log_endpoint.py | 1 + .../test_mapped_task_instance_endpoint.py | 1 + .../endpoints/test_plugin_endpoint.py | 1 + .../endpoints/test_pool_endpoint.py | 1 + .../endpoints/test_provider_endpoint.py | 1 + .../endpoints/test_task_endpoint.py | 1 + .../endpoints/test_task_instance_endpoint.py | 1 + .../endpoints/test_variable_endpoint.py | 1 + .../endpoints/test_xcom_endpoint.py | 1 + .../schemas/test_connection_schema.py | 1 + .../api_connexion/schemas/test_dag_run_schema.py | 1 + .../api_connexion/schemas/test_dataset_schema.py | 1 + tests/api_connexion/schemas/test_error_schema.py | 1 + .../api_connexion/schemas/test_plugin_schema.py | 1 + tests/api_connexion/schemas/test_pool_schemas.py | 1 + tests/api_connexion/schemas/test_xcom_schema.py | 1 + tests/api_connexion/test_auth.py | 1 + tests/api_connexion/test_parameters.py | 1 + .../core_api/routes/public/test_connections.py | 1 + .../core_api/routes/public/test_dag_run.py | 1 + .../core_api/routes/public/test_dags.py | 1 + .../core_api/routes/public/test_monitor.py | 1 + .../core_api/routes/public/test_variables.py | 1 + .../core_api/routes/ui/test_assets.py | 1 + .../core_api/routes/ui/test_dashboard.py | 1 + .../endpoints/test_rpc_api_endpoint.py | 1 + tests/api_internal/test_internal_api_call.py | 1 + tests/auth/managers/simple/views/test_auth.py | 1 + tests/cli/commands/test_backfill_command.py | 1 + tests/cli/commands/test_celery_command.py | 1 + tests/cli/commands/test_config_command.py | 1 + tests/cli/commands/test_connection_command.py | 1 + tests/cli/commands/test_dag_command.py | 4 ++-- tests/cli/commands/test_dag_processor_command.py | 1 + tests/cli/commands/test_info_command.py | 1 + tests/cli/commands/test_internal_api_command.py | 2 +- tests/cli/commands/test_jobs_command.py | 1 + tests/cli/commands/test_kerberos_command.py | 1 + tests/cli/commands/test_kubernetes_command.py | 1 + tests/cli/commands/test_plugins_command.py | 2 +- .../commands/test_rotate_fernet_key_command.py | 1 + tests/cli/commands/test_scheduler_command.py | 1 + tests/cli/commands/test_task_command.py | 1 + tests/cli/commands/test_variable_command.py | 1 + tests/cli/commands/test_webserver_command.py | 2 +- tests/cli/conftest.py | 1 + tests/cli/test_cli_parser.py | 1 + tests/core/test_configuration.py | 4 ++-- tests/core/test_core.py | 1 + tests/core/test_example_dags_system.py | 1 + tests/core/test_impersonation_tests.py | 1 + tests/core/test_logging_config.py | 1 + tests/core/test_otel_tracer.py | 1 + tests/core/test_sentry.py | 1 + tests/core/test_settings.py | 1 + tests/core/test_sqlalchemy_config.py | 1 + tests/core/test_stats.py | 1 + tests/dag_processing/test_job_runner.py | 6 +++--- tests/dag_processing/test_processor.py | 1 + tests/dags/test_miscellaneous.py | 1 + tests/dags/test_sensor.py | 1 + tests/decorators/test_bash.py | 1 + tests/decorators/test_python.py | 2 +- tests/executors/test_executor_loader.py | 1 + .../cli/commands/test_celery_command.py | 1 + .../executors/test_celery_executor.py | 1 + tests/integration/security/test_kerberos.py | 1 + tests/jobs/test_base_job.py | 2 +- tests/jobs/test_local_task_job.py | 1 + tests/jobs/test_scheduler_job.py | 10 +++++----- tests/jobs/test_triggerer_job.py | 2 +- tests/jobs/test_triggerer_job_logging.py | 1 + tests/lineage/test_hook.py | 1 + tests/lineage/test_lineage.py | 1 + tests/listeners/class_listener.py | 1 + .../listeners/test_dag_import_error_listener.py | 4 ++-- tests/models/test_backfill.py | 1 + tests/models/test_base.py | 1 + tests/models/test_baseoperator.py | 2 +- tests/models/test_cleartasks.py | 4 ++-- tests/models/test_dag.py | 16 ++++++++-------- tests/models/test_dagbag.py | 6 +++--- tests/models/test_dagcode.py | 1 + tests/models/test_dagrun.py | 4 ++-- tests/models/test_dagwarning.py | 1 + tests/models/test_mappedoperator.py | 4 ++-- tests/models/test_param.py | 1 + tests/models/test_pool.py | 1 + tests/models/test_renderedtifields.py | 1 + tests/models/test_serialized_dag.py | 1 + tests/models/test_skipmixin.py | 1 + tests/models/test_taskinstance.py | 4 ++-- tests/models/test_timestamp.py | 1 + tests/models/test_trigger.py | 1 + tests/models/test_variable.py | 1 + tests/models/test_xcom.py | 1 + tests/models/test_xcom_arg.py | 1 + tests/operators/test_branch_operator.py | 1 + tests/operators/test_email.py | 1 + tests/operators/test_latest_only_operator.py | 1 + tests/operators/test_python.py | 1 + tests/plugins/test_plugin.py | 6 +++--- tests/plugins/test_plugins_manager.py | 1 + tests/secrets/test_cache.py | 1 + tests/security/test_kerberos.py | 1 + tests/sensors/test_base.py | 1 + tests/sensors/test_external_task_sensor.py | 4 ++-- tests/serialization/test_dag_serialization.py | 1 + tests/serialization/test_pydantic_models.py | 2 +- tests/serialization/test_serde.py | 1 + tests/serialization/test_serialized_objects.py | 1 + .../task_runner/test_standard_task_runner.py | 4 ++-- .../deps/test_pool_slots_available_dep.py | 1 + tests/ti_deps/deps/test_prev_dagrun_dep.py | 1 + .../ti_deps/deps/test_ready_to_reschedule_dep.py | 1 + tests/utils/log/test_log_reader.py | 1 + tests/utils/log/test_secrets_masker.py | 1 + tests/utils/log/test_task_context_logger.py | 1 + tests/utils/test_db.py | 1 + tests/utils/test_db_cleanup.py | 1 + tests/utils/test_db_manager.py | 1 + tests/utils/test_dot_renderer.py | 1 + tests/utils/test_email.py | 1 + tests/utils/test_file.py | 2 +- tests/utils/test_helpers.py | 1 + tests/utils/test_log_handlers.py | 1 + tests/utils/test_net.py | 1 + tests/utils/test_serve_logs.py | 1 + tests/utils/test_sqlalchemy.py | 1 + tests/utils/test_state.py | 2 +- tests/utils/test_task_group.py | 2 +- .../test_task_handler_with_custom_formatter.py | 1 + tests/utils/test_types.py | 2 +- tests/www/test_app.py | 1 + tests/www/test_utils.py | 1 + tests/www/views/conftest.py | 1 + tests/www/views/test_anonymous_as_admin_role.py | 1 + tests/www/views/test_session.py | 1 + tests/www/views/test_views.py | 3 +++ tests/www/views/test_views_acl.py | 4 ++-- tests/www/views/test_views_base.py | 1 + tests/www/views/test_views_cluster_activity.py | 1 + tests/www/views/test_views_configuration.py | 1 + tests/www/views/test_views_connection.py | 1 + tests/www/views/test_views_custom_user_views.py | 10 +++++----- tests/www/views/test_views_dagrun.py | 12 ++++++------ tests/www/views/test_views_dataset.py | 1 + tests/www/views/test_views_decorators.py | 1 + tests/www/views/test_views_extra_links.py | 1 + tests/www/views/test_views_grid.py | 1 + tests/www/views/test_views_home.py | 4 ++-- tests/www/views/test_views_log.py | 1 + tests/www/views/test_views_mount.py | 1 + tests/www/views/test_views_paused.py | 1 + tests/www/views/test_views_pool.py | 1 + tests/www/views/test_views_rate_limit.py | 1 + tests/www/views/test_views_rendered.py | 1 + tests/www/views/test_views_tasks.py | 12 ++++++------ tests/www/views/test_views_trigger_dag.py | 4 ++-- tests/www/views/test_views_variable.py | 4 ++-- tests_common/pytest_plugin.py | 2 ++ tests_common/test_utils/api_connexion_utils.py | 1 + tests_common/test_utils/azure_system_helpers.py | 1 + tests_common/test_utils/db.py | 2 ++ tests_common/test_utils/gcp_system_helpers.py | 4 ++-- tests_common/test_utils/mock_operators.py | 1 + tests_common/test_utils/system_tests_class.py | 1 + 319 files changed, 560 insertions(+), 317 deletions(-) diff --git a/providers/tests/alibaba/cloud/log/test_oss_task_handler.py b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py index ff65dd89ce4ef..b6c22d5234211 100644 --- a/providers/tests/alibaba/cloud/log/test_oss_task_handler.py +++ b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py @@ -22,13 +22,14 @@ from unittest.mock import PropertyMock import pytest -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.providers.alibaba.cloud.log.oss_task_handler import OSSTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs + pytestmark = pytest.mark.db_test OSS_TASK_HANDLER_STRING = "airflow.providers.alibaba.cloud.log.oss_task_handler.{}" diff --git a/providers/tests/amazon/aws/auth_manager/avp/test_facade.py b/providers/tests/amazon/aws/auth_manager/avp/test_facade.py index af1149aec45fd..1ea265d256787 100644 --- a/providers/tests/amazon/aws/auth_manager/avp/test_facade.py +++ b/providers/tests/amazon/aws/auth_manager/avp/test_facade.py @@ -21,7 +21,6 @@ from unittest.mock import Mock import pytest -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities, get_action_id, get_entity_type @@ -29,6 +28,8 @@ from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser from airflow.utils.helpers import prune_dict +from tests_common.test_utils.config import conf_vars + if TYPE_CHECKING: from airflow.auth.managers.base_auth_manager import ResourceMethod diff --git a/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py index c523beec564c7..faafd1ce497ea 100644 --- a/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py +++ b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py @@ -20,12 +20,13 @@ from unittest.mock import ANY, Mock, patch import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS -from tests_common.test_utils.config import conf_vars from airflow.cli import cli_parser from airflow.providers.amazon.aws.auth_manager.cli.avp_commands import init_avp, update_schema +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS +from tests_common.test_utils.config import conf_vars + mock_boto3 = Mock() pytestmark = [ diff --git a/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py b/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py index 1c57bc14d83ed..91ee6e740db97 100644 --- a/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py +++ b/providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py @@ -20,6 +20,7 @@ import pytest from flask import Flask + from tests_common.test_utils.compat import ignore_provider_compatibility_error python3_saml = pytest.importorskip("python3-saml") diff --git a/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py index ba93b2105a81e..c8c1bb03d4bb6 100644 --- a/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py +++ b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py @@ -22,9 +22,6 @@ import pytest from flask import Flask, session from flask_appbuilder.menu import MenuItem -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.www import check_content_in_response from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities from airflow.providers.amazon.aws.auth_manager.avp.facade import AwsAuthManagerAmazonVerifiedPermissionsFacade @@ -42,6 +39,10 @@ from airflow.www import app as application from airflow.www.extensions.init_appbuilder import init_appbuilder +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response + try: from airflow.auth.managers.models.resource_details import ( AccessView, diff --git a/providers/tests/amazon/aws/auth_manager/views/test_auth.py b/providers/tests/amazon/aws/auth_manager/views/test_auth.py index 919eb59ead059..43ba306766df8 100644 --- a/providers/tests/amazon/aws/auth_manager/views/test_auth.py +++ b/providers/tests/amazon/aws/auth_manager/views/test_auth.py @@ -20,12 +20,13 @@ import pytest from flask import session, url_for -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.www import app as application +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.config import conf_vars + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Test requires Airflow 2.9+"), pytest.mark.skip_if_database_isolation_mode, diff --git a/providers/tests/amazon/aws/executors/batch/test_batch_executor.py b/providers/tests/amazon/aws/executors/batch/test_batch_executor.py index 85272186b3324..92c3f4f371cb6 100644 --- a/providers/tests/amazon/aws/executors/batch/test_batch_executor.py +++ b/providers/tests/amazon/aws/executors/batch/test_batch_executor.py @@ -26,8 +26,6 @@ import yaml from botocore.exceptions import ClientError, NoCredentialsError from semver import VersionInfo -from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.executors.base_executor import BaseExecutor @@ -48,6 +46,9 @@ from airflow.utils.state import State from airflow.version import version as airflow_version_str +from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from tests_common.test_utils.config import conf_vars + airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3])) ARN1 = "arn1" diff --git a/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py b/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py index 80b7d85a9eded..1ca78d39c7750 100644 --- a/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py +++ b/providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py @@ -32,9 +32,6 @@ from botocore.exceptions import ClientError from inflection import camelize from semver import VersionInfo -from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.executors.base_executor import BaseExecutor @@ -60,6 +57,10 @@ from airflow.utils.timezone import utcnow from airflow.version import version as airflow_version_str +from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3])) diff --git a/providers/tests/amazon/aws/hooks/test_base_aws.py b/providers/tests/amazon/aws/hooks/test_base_aws.py index 8919203da0f01..f0b579d10c0a3 100644 --- a/providers/tests/amazon/aws/hooks/test_base_aws.py +++ b/providers/tests/amazon/aws/hooks/test_base_aws.py @@ -38,7 +38,6 @@ from botocore.utils import FileWebIdentityTokenLoader from moto import mock_aws from moto.core import DEFAULT_ACCOUNT_ID -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.models.connection import Connection @@ -51,6 +50,8 @@ ) from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper +from tests_common.test_utils.config import conf_vars + pytest.importorskip("aiobotocore") MOCK_AWS_CONN_ID = "mock-conn-id" diff --git a/providers/tests/amazon/aws/hooks/test_s3.py b/providers/tests/amazon/aws/hooks/test_s3.py index 89d29dae02dde..727322299f174 100644 --- a/providers/tests/amazon/aws/hooks/test_s3.py +++ b/providers/tests/amazon/aws/hooks/test_s3.py @@ -30,7 +30,6 @@ import pytest from botocore.exceptions import ClientError from moto import mock_aws -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS from airflow.exceptions import AirflowException from airflow.models import Connection @@ -44,6 +43,8 @@ ) from airflow.utils.timezone import datetime +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS + @pytest.fixture def mocked_s3_res(): diff --git a/providers/tests/amazon/aws/links/test_base_aws.py b/providers/tests/amazon/aws/links/test_base_aws.py index 738fb6acc482c..664132cef0082 100644 --- a/providers/tests/amazon/aws/links/test_base_aws.py +++ b/providers/tests/amazon/aws/links/test_base_aws.py @@ -21,13 +21,14 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.mock_operators import MockOperator from airflow.models.xcom import XCom from airflow.providers.amazon.aws.links.base_aws import BaseAwsLink from airflow.serialization.serialized_objects import SerializedDAG +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.mock_operators import MockOperator + if TYPE_CHECKING: from airflow.models.taskinstance import TaskInstance diff --git a/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py index d07699d9a7049..cdca8735b2d3c 100644 --- a/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py +++ b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py @@ -26,7 +26,6 @@ import boto3 import pytest from moto import mock_aws -from tests_common.test_utils.config import conf_vars from watchtower import CloudWatchLogHandler from airflow.models import DAG, DagRun, TaskInstance @@ -37,6 +36,8 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars + def get_time_str(time_in_milliseconds): dt_time = dt.fromtimestamp(time_in_milliseconds / 1000.0, tz=timezone.utc) diff --git a/providers/tests/amazon/aws/log/test_s3_task_handler.py b/providers/tests/amazon/aws/log/test_s3_task_handler.py index a471bf0075479..e98cf18461849 100644 --- a/providers/tests/amazon/aws/log/test_s3_task_handler.py +++ b/providers/tests/amazon/aws/log/test_s3_task_handler.py @@ -26,7 +26,6 @@ import pytest from botocore.exceptions import ClientError from moto import mock_aws -from tests_common.test_utils.config import conf_vars from airflow.models import DAG, DagRun, TaskInstance from airflow.operators.empty import EmptyOperator @@ -35,6 +34,8 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars + @pytest.fixture(autouse=True) def s3mock(): diff --git a/providers/tests/amazon/aws/secrets/test_systems_manager.py b/providers/tests/amazon/aws/secrets/test_systems_manager.py index e1c035ab42047..ecb3996263a2b 100644 --- a/providers/tests/amazon/aws/secrets/test_systems_manager.py +++ b/providers/tests/amazon/aws/secrets/test_systems_manager.py @@ -21,11 +21,12 @@ import pytest from moto import mock_aws -from tests_common.test_utils.config import conf_vars from airflow.configuration import initialize_secrets_backends from airflow.providers.amazon.aws.secrets.systems_manager import SystemsManagerParameterStoreBackend +from tests_common.test_utils.config import conf_vars + URI_CONNECTION = pytest.param( "postgres://my-login:my-pass@my-host:5432/my-schema?param1=val1¶m2=val2", id="uri-connection" ) diff --git a/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py b/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py index 605705a2b663f..0bc4b83c3552c 100644 --- a/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_redshift_to_s3.py @@ -22,13 +22,14 @@ import pytest from boto3.session import Session -from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces from airflow.exceptions import AirflowException from airflow.models.connection import Connection from airflow.providers.amazon.aws.transfers.redshift_to_s3 import RedshiftToS3Operator from airflow.providers.amazon.aws.utils.redshift import build_credentials_block +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces + class TestRedshiftToS3Transfer: @pytest.mark.parametrize("table_as_file_name, expected_s3_key", [[True, "key/table_"], [False, "key"]]) diff --git a/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py b/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py index c9d9ca337d111..59d819e0137cb 100644 --- a/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py +++ b/providers/tests/amazon/aws/transfers/test_s3_to_redshift.py @@ -22,7 +22,6 @@ import pytest from boto3.session import Session -from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces from airflow.exceptions import AirflowException from airflow.models.connection import Connection @@ -35,6 +34,8 @@ SchemaDatasetFacetFields, ) +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces + class TestS3ToRedshiftTransfer: @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_connection") diff --git a/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py b/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py index bcab8b2909af3..088f95e3fc925 100644 --- a/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py +++ b/providers/tests/amazon/aws/transfers/test_s3_to_sftp.py @@ -20,7 +20,6 @@ import boto3 import pytest from moto import mock_aws -from tests_common.test_utils.config import conf_vars from airflow.models import DAG from airflow.providers.amazon.aws.hooks.s3 import S3Hook @@ -29,6 +28,8 @@ from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py b/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py index 980c8a19daa55..7b812712d7ef2 100644 --- a/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_sftp_to_s3.py @@ -20,7 +20,6 @@ import boto3 import pytest from moto import mock_aws -from tests_common.test_utils.config import conf_vars from airflow.models import DAG from airflow.providers.amazon.aws.hooks.s3 import S3Hook @@ -29,6 +28,8 @@ from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test BUCKET = "test-bucket" diff --git a/providers/tests/apache/hive/hooks/test_hive.py b/providers/tests/apache/hive/hooks/test_hive.py index d975ce50fb692..1e9dc687584ca 100644 --- a/providers/tests/apache/hive/hooks/test_hive.py +++ b/providers/tests/apache/hive/hooks/test_hive.py @@ -25,7 +25,6 @@ import pandas as pd import pytest from hmsclient import HMSClient -from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces from airflow.exceptions import AirflowException from airflow.models.connection import Connection @@ -42,6 +41,7 @@ MockHiveServer2Hook, MockSubProcess, ) +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() diff --git a/providers/tests/apache/livy/hooks/test_livy.py b/providers/tests/apache/livy/hooks/test_livy.py index d2968591f90f4..c188d5853aa93 100644 --- a/providers/tests/apache/livy/hooks/test_livy.py +++ b/providers/tests/apache/livy/hooks/test_livy.py @@ -25,13 +25,14 @@ import requests from aiohttp import ClientResponseError, RequestInfo from requests.exceptions import RequestException -from tests_common.test_utils.db import clear_db_connections from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.apache.livy.hooks.livy import BatchState, LivyAsyncHook, LivyHook from airflow.utils import db +from tests_common.test_utils.db import clear_db_connections + pytestmark = pytest.mark.skip_if_database_isolation_mode LIVY_CONN_ID = LivyHook.default_conn_name diff --git a/providers/tests/apache/spark/hooks/test_spark_sql.py b/providers/tests/apache/spark/hooks/test_spark_sql.py index 179e75ffc7470..3ea9ab20e900b 100644 --- a/providers/tests/apache/spark/hooks/test_spark_sql.py +++ b/providers/tests/apache/spark/hooks/test_spark_sql.py @@ -22,13 +22,14 @@ from unittest.mock import call, patch import pytest -from tests_common.test_utils.db import clear_db_connections from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.apache.spark.hooks.spark_sql import SparkSqlHook from airflow.utils import db +from tests_common.test_utils.db import clear_db_connections + pytestmark = pytest.mark.db_test diff --git a/providers/tests/atlassian/jira/hooks/test_jira.py b/providers/tests/atlassian/jira/hooks/test_jira.py index fbb7913d128f0..d90a91400f973 100644 --- a/providers/tests/atlassian/jira/hooks/test_jira.py +++ b/providers/tests/atlassian/jira/hooks/test_jira.py @@ -20,12 +20,13 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import connection_as_json from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.atlassian.jira.hooks.jira import JiraHook +from tests_common.test_utils.compat import connection_as_json + @pytest.fixture def mocked_jira_client(): diff --git a/providers/tests/atlassian/jira/operators/test_jira.py b/providers/tests/atlassian/jira/operators/test_jira.py index 67c63a60d61d4..45f06fe46a3ae 100644 --- a/providers/tests/atlassian/jira/operators/test_jira.py +++ b/providers/tests/atlassian/jira/operators/test_jira.py @@ -20,12 +20,13 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import connection_as_json from airflow.models import Connection from airflow.providers.atlassian.jira.operators.jira import JiraOperator from airflow.utils import timezone +from tests_common.test_utils.compat import connection_as_json + DEFAULT_DATE = timezone.datetime(2017, 1, 1) MINIMAL_TEST_TICKET = { "id": "911539", diff --git a/providers/tests/atlassian/jira/sensors/test_jira.py b/providers/tests/atlassian/jira/sensors/test_jira.py index ca027b7b5989d..653c899d65e7f 100644 --- a/providers/tests/atlassian/jira/sensors/test_jira.py +++ b/providers/tests/atlassian/jira/sensors/test_jira.py @@ -20,12 +20,13 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import connection_as_json from airflow.models import Connection from airflow.providers.atlassian.jira.sensors.jira import JiraTicketSensor from airflow.utils import timezone +from tests_common.test_utils.compat import connection_as_json + DEFAULT_DATE = timezone.datetime(2017, 1, 1) MINIMAL_TEST_TICKET = { "id": "911539", diff --git a/providers/tests/celery/cli/test_celery_command.py b/providers/tests/celery/cli/test_celery_command.py index 0d0310f7452ea..707ef3a4d7359 100644 --- a/providers/tests/celery/cli/test_celery_command.py +++ b/providers/tests/celery/cli/test_celery_command.py @@ -24,8 +24,6 @@ import pytest import sqlalchemy -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests_common.test_utils.config import conf_vars import airflow from airflow.cli import cli_parser @@ -33,6 +31,9 @@ from airflow.executors import executor_loader from airflow.providers.celery.cli import celery_command +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/celery/executors/test_celery_executor.py b/providers/tests/celery/executors/test_celery_executor.py index f72ee35b0f50f..71fae6691c6fe 100644 --- a/providers/tests/celery/executors/test_celery_executor.py +++ b/providers/tests/celery/executors/test_celery_executor.py @@ -32,9 +32,6 @@ from celery import Celery from celery.result import AsyncResult from kombu.asynchronous import set_event_loop -from tests_common.test_utils import db -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests_common.test_utils.config import conf_vars from airflow.configuration import conf from airflow.models.baseoperator import BaseOperator @@ -45,6 +42,10 @@ from airflow.utils import timezone from airflow.utils.state import State +from tests_common.test_utils import db +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/celery/log_handlers/test_log_handlers.py b/providers/tests/celery/log_handlers/test_log_handlers.py index 5d7d4d473d2fc..d47029cdd4dd5 100644 --- a/providers/tests/celery/log_handlers/test_log_handlers.py +++ b/providers/tests/celery/log_handlers/test_log_handlers.py @@ -23,8 +23,6 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.config import conf_vars from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.executors import executor_loader @@ -38,6 +36,9 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars + if AIRFLOW_V_3_0_PLUS: pass diff --git a/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py b/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py index 4d277c3b51e2d..2d745e60954a8 100644 --- a/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py +++ b/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py @@ -24,12 +24,13 @@ import kubernetes import pytest from dateutil.parser import parse -from tests_common.test_utils.config import conf_vars from airflow.cli import cli_parser from airflow.executors import executor_loader from airflow.providers.cncf.kubernetes.cli import kubernetes_command +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py index 85077ba1776bd..9d425edcb9d06 100644 --- a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -26,8 +26,6 @@ import yaml from kubernetes.client import models as k8s from kubernetes.client.rest import ApiException -from tests_common.test_utils.compat import BashOperator -from tests_common.test_utils.config import conf_vars from urllib3 import HTTPResponse from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning @@ -57,6 +55,9 @@ from airflow.utils import timezone from airflow.utils.state import State, TaskInstanceState +from tests_common.test_utils.compat import BashOperator +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py b/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py index d90ed0504401a..d8e73b99b700e 100644 --- a/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py +++ b/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py @@ -29,8 +29,6 @@ from kubernetes.client.rest import ApiException from kubernetes.config import ConfigException from sqlalchemy.orm import make_transient -from tests_common.test_utils.db import clear_db_connections -from tests_common.test_utils.providers import get_provider_min_airflow_version from airflow.exceptions import AirflowException, AirflowNotFoundException from airflow.hooks.base import BaseHook @@ -39,6 +37,9 @@ from airflow.utils import db from airflow.utils.db import merge_conn +from tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.providers import get_provider_min_airflow_version + pytestmark = pytest.mark.db_test KUBE_CONFIG_PATH = os.getenv("KUBECONFIG", "~/.kube/config") diff --git a/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py index 7ef025d7f29d4..45b68d015e4b4 100644 --- a/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py +++ b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py @@ -26,8 +26,6 @@ import pytest from kubernetes.client import models as k8s -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.config import conf_vars from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.executors import executor_loader @@ -44,6 +42,9 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/cncf/kubernetes/operators/test_pod.py b/providers/tests/cncf/kubernetes/operators/test_pod.py index a272a5c4603f6..a39b1c6d88937 100644 --- a/providers/tests/cncf/kubernetes/operators/test_pod.py +++ b/providers/tests/cncf/kubernetes/operators/test_pod.py @@ -27,7 +27,6 @@ import pytest from kubernetes.client import ApiClient, V1Pod, V1PodSecurityContext, V1PodStatus, models as k8s from kubernetes.client.exceptions import ApiException -from tests_common.test_utils import db from urllib3 import HTTPResponse from airflow.exceptions import ( @@ -52,6 +51,8 @@ from airflow.utils.session import create_session from airflow.utils.types import DagRunType +from tests_common.test_utils import db + pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py index ea15480ba1212..0b4c07be71d8c 100644 --- a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py +++ b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py @@ -27,7 +27,6 @@ import pytest import yaml from kubernetes.client import models as k8s -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow import DAG from airflow.models import Connection, DagRun, TaskInstance @@ -35,6 +34,8 @@ from airflow.utils import db, timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + @patch("airflow.providers.cncf.kubernetes.operators.spark_kubernetes.KubernetesHook") def test_spark_kubernetes_operator(mock_kubernetes_hook, data_file): diff --git a/providers/tests/cncf/kubernetes/test_client.py b/providers/tests/cncf/kubernetes/test_client.py index ce3a961eef44a..5bb18a0992fc5 100644 --- a/providers/tests/cncf/kubernetes/test_client.py +++ b/providers/tests/cncf/kubernetes/test_client.py @@ -21,7 +21,6 @@ import pytest from kubernetes.client import Configuration -from tests_common.test_utils.config import conf_vars from urllib3.connection import HTTPConnection, HTTPSConnection from airflow.providers.cncf.kubernetes.kube_client import ( @@ -30,6 +29,8 @@ get_kube_client, ) +from tests_common.test_utils.config import conf_vars + class TestClient: @mock.patch("airflow.providers.cncf.kubernetes.kube_client.config") diff --git a/providers/tests/cncf/kubernetes/test_template_rendering.py b/providers/tests/cncf/kubernetes/test_template_rendering.py index bf67059ed11ff..8eb43cd1eaeb0 100644 --- a/providers/tests/cncf/kubernetes/test_template_rendering.py +++ b/providers/tests/cncf/kubernetes/test_template_rendering.py @@ -21,7 +21,6 @@ import pytest from sqlalchemy.orm import make_transient -from tests_common.test_utils.compat import BashOperator from airflow.models.renderedtifields import RenderedTaskInstanceFields, RenderedTaskInstanceFields as RTIF from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec, render_k8s_pod_yaml @@ -29,6 +28,8 @@ from airflow.utils.session import create_session from airflow.version import version +from tests_common.test_utils.compat import BashOperator + pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] DEFAULT_DATE = timezone.datetime(2021, 9, 9) diff --git a/providers/tests/common/io/operators/test_file_transfer.py b/providers/tests/common/io/operators/test_file_transfer.py index 4ce23de04534c..65751bb3ef7da 100644 --- a/providers/tests/common/io/operators/test_file_transfer.py +++ b/providers/tests/common/io/operators/test_file_transfer.py @@ -19,10 +19,10 @@ from unittest import mock -from tests_common.test_utils.compat import ignore_provider_compatibility_error - from airflow.providers.common.compat.openlineage.facet import Dataset +from tests_common.test_utils.compat import ignore_provider_compatibility_error + with ignore_provider_compatibility_error("2.8.0", __file__): from airflow.providers.common.io.operators.file_transfer import FileTransferOperator diff --git a/providers/tests/common/io/xcom/test_backend.py b/providers/tests/common/io/xcom/test_backend.py index a60fd872bc851..b88340c62e662 100644 --- a/providers/tests/common/io/xcom/test_backend.py +++ b/providers/tests/common/io/xcom/test_backend.py @@ -18,6 +18,7 @@ from __future__ import annotations import pytest + from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS, ignore_provider_compatibility_error from tests_common.test_utils.db import is_db_isolation_mode @@ -34,12 +35,12 @@ with ignore_provider_compatibility_error("2.8.0", __file__): from airflow.providers.common.io.xcom.backend import XComObjectStorageBackend -from tests_common.test_utils import db -from tests_common.test_utils.config import conf_vars - from airflow.utils import timezone from airflow.utils.xcom import XCOM_RETURN_KEY +from tests_common.test_utils import db +from tests_common.test_utils.config import conf_vars + @pytest.fixture(autouse=True) def reset_db(): diff --git a/providers/tests/common/sql/hooks/test_dbapi.py b/providers/tests/common/sql/hooks/test_dbapi.py index 0b4d7dc72e4b1..d94b817eeeb82 100644 --- a/providers/tests/common/sql/hooks/test_dbapi.py +++ b/providers/tests/common/sql/hooks/test_dbapi.py @@ -24,13 +24,14 @@ import pytest from pyodbc import Cursor -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.hooks.base import BaseHook from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler, fetch_one_handler +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/common/sql/hooks/test_sql.py b/providers/tests/common/sql/hooks/test_sql.py index 9908a512f8599..c1f919c704f72 100644 --- a/providers/tests/common/sql/hooks/test_sql.py +++ b/providers/tests/common/sql/hooks/test_sql.py @@ -24,7 +24,6 @@ from unittest.mock import MagicMock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.exceptions import AirflowProviderDeprecationWarning @@ -33,6 +32,7 @@ from airflow.utils.session import provide_session from providers.tests.common.sql.test_utils import mock_hook +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), diff --git a/providers/tests/common/sql/hooks/test_sqlparse.py b/providers/tests/common/sql/hooks/test_sqlparse.py index a32f5ee20880b..c8cca3aa35e81 100644 --- a/providers/tests/common/sql/hooks/test_sqlparse.py +++ b/providers/tests/common/sql/hooks/test_sqlparse.py @@ -17,10 +17,11 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.providers.common.sql.hooks.sql import DbApiHook +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/common/sql/operators/test_sql.py b/providers/tests/common/sql/operators/test_sql.py index d2a371311c1d0..5026a5ec989a8 100644 --- a/providers/tests/common/sql/operators/test_sql.py +++ b/providers/tests/common/sql/operators/test_sql.py @@ -22,7 +22,6 @@ from unittest.mock import MagicMock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_3_0_PLUS from airflow import DAG from airflow.exceptions import AirflowException @@ -45,6 +44,8 @@ from airflow.utils.session import create_session from airflow.utils.state import State +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_3_0_PLUS + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/common/sql/operators/test_sql_execute.py b/providers/tests/common/sql/operators/test_sql_execute.py index f03c80866ca39..ee51dae1d5a32 100644 --- a/providers/tests/common/sql/operators/test_sql_execute.py +++ b/providers/tests/common/sql/operators/test_sql_execute.py @@ -22,7 +22,6 @@ from unittest.mock import MagicMock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.models import Connection from airflow.providers.common.compat.openlineage.facet import ( @@ -35,6 +34,8 @@ from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.openlineage.extractors.base import OperatorLineage +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/common/sql/sensors/test_sql.py b/providers/tests/common/sql/sensors/test_sql.py index c44bbf1d9d1ab..3b46af7be578d 100644 --- a/providers/tests/common/sql/sensors/test_sql.py +++ b/providers/tests/common/sql/sensors/test_sql.py @@ -20,7 +20,6 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from airflow.exceptions import AirflowException from airflow.models.dag import DAG @@ -28,6 +27,8 @@ from airflow.providers.common.sql.sensors.sql import SqlSensor from airflow.utils.timezone import datetime +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.8.0+ only"), pytest.mark.skip_if_database_isolation_mode, diff --git a/providers/tests/common/sql/test_utils.py b/providers/tests/common/sql/test_utils.py index 50a217c87a2cc..1c8625d1e9a9c 100644 --- a/providers/tests/common/sql/test_utils.py +++ b/providers/tests/common/sql/test_utils.py @@ -21,10 +21,11 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from airflow.models import Connection +from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), ] diff --git a/providers/tests/databricks/plugins/test_databricks_workflow.py b/providers/tests/databricks/plugins/test_databricks_workflow.py index 72d6b3f321f39..35d3496c66e47 100644 --- a/providers/tests/databricks/plugins/test_databricks_workflow.py +++ b/providers/tests/databricks/plugins/test_databricks_workflow.py @@ -21,7 +21,6 @@ import pytest from flask import url_for -from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES from airflow.exceptions import AirflowException from airflow.models.dagrun import DagRun @@ -42,6 +41,8 @@ ) from airflow.www.app import create_app +from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES + DAG_ID = "test_dag" TASK_ID = "test_task" RUN_ID = "test_run_1" diff --git a/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py b/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py index 4a185971e67a7..d8c2426214600 100644 --- a/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py +++ b/providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py @@ -21,8 +21,6 @@ from unittest import mock import pytest -from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules -from tests_common.test_utils.mock_plugins import mock_plugin_manager from airflow.api_connexion.exceptions import PermissionDenied from airflow.configuration import conf @@ -43,6 +41,9 @@ from airflow.utils.state import State from airflow.www import app +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules +from tests_common.test_utils.mock_plugins import mock_plugin_manager + # Note: Sounds a bit strange to disable internal API tests in isolation mode but... # As long as the test is modelled to run its own internal API endpoints, it is conflicting # to the test setup with a dedicated internal API server. diff --git a/providers/tests/edge/cli/test_edge_command.py b/providers/tests/edge/cli/test_edge_command.py index e6e07d032baa6..13bcfcc55700c 100644 --- a/providers/tests/edge/cli/test_edge_command.py +++ b/providers/tests/edge/cli/test_edge_command.py @@ -23,7 +23,6 @@ import pytest import time_machine -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.providers.edge.cli.edge_command import ( @@ -35,6 +34,8 @@ from airflow.providers.edge.models.edge_worker import EdgeWorker, EdgeWorkerState from airflow.utils.state import TaskInstanceState +from tests_common.test_utils.config import conf_vars + pytest.importorskip("pydantic", minversion="2.0.0") # Ignore the following error for mocking diff --git a/providers/tests/edge/plugins/test_edge_executor_plugin.py b/providers/tests/edge/plugins/test_edge_executor_plugin.py index bdd4847865539..dabd4df3e0512 100644 --- a/providers/tests/edge/plugins/test_edge_executor_plugin.py +++ b/providers/tests/edge/plugins/test_edge_executor_plugin.py @@ -19,11 +19,12 @@ import importlib import pytest -from tests_common.test_utils.config import conf_vars from airflow.plugins_manager import AirflowPlugin from airflow.providers.edge.plugins import edge_executor_plugin +from tests_common.test_utils.config import conf_vars + def test_plugin_inactive(): with conf_vars({("edge", "api_enabled"): "false"}): diff --git a/providers/tests/elasticsearch/log/test_es_task_handler.py b/providers/tests/elasticsearch/log/test_es_task_handler.py index adb10090355b1..d8d328969148f 100644 --- a/providers/tests/elasticsearch/log/test_es_task_handler.py +++ b/providers/tests/elasticsearch/log/test_es_task_handler.py @@ -31,8 +31,6 @@ import elasticsearch import pendulum import pytest -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.configuration import conf from airflow.providers.elasticsearch.log.es_response import ElasticSearchResponse @@ -48,6 +46,8 @@ from providers.tests.elasticsearch.log.elasticmock import elasticmock from providers.tests.elasticsearch.log.elasticmock.utilities import SearchFailedException +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py b/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py index 65db6a654ae23..52281df0f9981 100644 --- a/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_basic_auth.py @@ -21,11 +21,12 @@ import pytest from flask import Response from flask_appbuilder.const import AUTH_LDAP -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from airflow.providers.fab.auth_manager.api.auth.backend.basic_auth import requires_authentication from airflow.www import app as application +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), ] diff --git a/providers/tests/fab/auth_manager/api/auth/backend/test_session.py b/providers/tests/fab/auth_manager/api/auth/backend/test_session.py index 405eafe11dfc4..4f157c032e28b 100644 --- a/providers/tests/fab/auth_manager/api/auth/backend/test_session.py +++ b/providers/tests/fab/auth_manager/api/auth/backend/test_session.py @@ -20,11 +20,12 @@ import pytest from flask import Response -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from airflow.providers.fab.auth_manager.api.auth.backend.session import requires_authentication from airflow.www import app as application +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS + pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), ] diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py index 104e7943f4cd1..33b7944497c3a 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py @@ -20,15 +20,15 @@ import pytest import time_machine -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_assets, clear_db_runs -from tests_common.test_utils.www import _check_last_log from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.security import permissions from airflow.utils import timezone from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_assets, clear_db_runs +from tests_common.test_utils.www import _check_last_log try: from airflow.models.asset import AssetDagRunQueue, AssetModel diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_auth.py b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py index 6a90b7ec4b30e..7fb499a51fd97 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_auth.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_auth.py @@ -20,6 +20,7 @@ import pytest from flask_login import current_user + from tests_common.test_utils.api_connexion_utils import assert_401 from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py index 4d7a5fc08776a..36244b079511a 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py @@ -22,13 +22,6 @@ import pendulum import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import ( - clear_db_backfills, - clear_db_dags, - clear_db_runs, - clear_db_serialized_dags, -) from airflow.models import DagBag, DagModel from airflow.models.dag import DAG @@ -39,6 +32,13 @@ from airflow.utils.session import provide_session from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import ( + clear_db_backfills, + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, +) try: from airflow.models.backfill import Backfill diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_cors.py b/providers/tests/fab/auth_manager/api_endpoints/test_cors.py index 3741d71fb8b96..1a70bd20a557e 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_cors.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_cors.py @@ -19,6 +19,7 @@ from base64 import b64encode import pytest + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_pools diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py index f2bd83efa331c..305a7ca3459ab 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py @@ -21,9 +21,6 @@ import pendulum import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from tests_common.test_utils.www import _check_last_log from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import DagBag, DagModel @@ -33,6 +30,9 @@ from airflow.utils.session import provide_session from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests_common.test_utils.www import _check_last_log pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py index f822cddf49ee0..05bac3394ef0f 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py @@ -19,8 +19,6 @@ from datetime import timedelta import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags from airflow.models.dag import DAG, DagModel from airflow.models.dagrun import DagRun @@ -35,6 +33,8 @@ delete_roles, delete_user, ) +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags try: from airflow.utils.types import DagRunTriggeredByType, DagRunType diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py index 8cfb6c58f5320..bc4ba7041caf3 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py @@ -21,13 +21,13 @@ from typing import TYPE_CHECKING import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags from airflow.models import DagBag from airflow.security import permissions from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py index ec8a72d74ee48..7d1c2b45a91be 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py @@ -17,8 +17,6 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags from airflow.models.dag import DagModel from airflow.models.dagwarning import DagWarning @@ -26,6 +24,8 @@ from airflow.utils.session import create_session from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py index 7d7868582934e..9f00c7943e8b4 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py @@ -17,14 +17,14 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_logs from airflow.models import Log from airflow.security import permissions from airflow.utils import timezone from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_logs pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py index b9b8d4ff9a421..1224c9c96c4a7 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_import_error_endpoint.py @@ -17,15 +17,15 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError -from tests_common.test_utils.db import clear_db_dags, clear_db_import_errors -from tests_common.test_utils.permissions import _resource_name from airflow.models.dag import DagModel from airflow.security import permissions from airflow.utils import timezone from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError +from tests_common.test_utils.db import clear_db_dags, clear_db_import_errors +from tests_common.test_utils.permissions import _resource_name pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py index 986b55e5ff42e..98451a894ca76 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py @@ -17,8 +17,6 @@ from __future__ import annotations import pytest -from tests_common.test_utils.api_connexion_utils import assert_401 -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.security import permissions @@ -29,6 +27,8 @@ delete_role, delete_user, ) +from tests_common.test_utils.api_connexion_utils import assert_401 +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import Role diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py index f0063212c697c..a9a25c1f8a590 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py @@ -20,8 +20,6 @@ import urllib import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import DagRun, TaskInstance @@ -36,6 +34,8 @@ delete_roles, delete_user, ) +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py index 29f84850d9ba6..d1830359556e2 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_user_endpoint.py @@ -20,9 +20,6 @@ import pytest from sqlalchemy.sql.functions import count -from tests_common.test_utils.api_connexion_utils import assert_401 -from tests_common.test_utils.compat import ignore_provider_compatibility_error -from tests_common.test_utils.config import conf_vars from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.security import permissions @@ -34,6 +31,9 @@ delete_role, delete_user, ) +from tests_common.test_utils.api_connexion_utils import assert_401 +from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.config import conf_vars with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import User diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py index 038dea9e72df9..997cac8ce9029 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_variable_endpoint.py @@ -17,13 +17,13 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_variables from airflow.models import Variable from airflow.security import permissions from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_variables pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py index f45cf58de6a95..e40db31eb2385 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py @@ -19,8 +19,6 @@ from datetime import timedelta import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom from airflow.models.dag import DagModel from airflow.models.dagrun import DagRun @@ -33,6 +31,8 @@ from airflow.utils.types import DagRunType from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom pytestmark = [ pytest.mark.db_test, diff --git a/providers/tests/fab/auth_manager/cli_commands/test_role_command.py b/providers/tests/fab/auth_manager/cli_commands/test_role_command.py index d07cfc61242fe..e04ad3e586e6e 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_role_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_role_command.py @@ -23,11 +23,12 @@ from typing import TYPE_CHECKING import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error -from tests_common.test_utils.config import conf_vars from airflow.cli import cli_parser +from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.config import conf_vars + with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import role_command from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder diff --git a/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py b/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py index a0345909dc2df..c2bcc62c80e7e 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_sync_perm_command.py @@ -20,10 +20,11 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.cli import cli_parser +from tests_common.test_utils.compat import ignore_provider_compatibility_error + with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import sync_perm_command diff --git a/providers/tests/fab/auth_manager/cli_commands/test_user_command.py b/providers/tests/fab/auth_manager/cli_commands/test_user_command.py index 6ccd4c99716ab..051ae646a4463 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_user_command.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_user_command.py @@ -24,10 +24,11 @@ from io import StringIO import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.cli import cli_parser +from tests_common.test_utils.compat import ignore_provider_compatibility_error + with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import user_command from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder diff --git a/providers/tests/fab/auth_manager/cli_commands/test_utils.py b/providers/tests/fab/auth_manager/cli_commands/test_utils.py index 193317249ed11..cacf614a82fd0 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_utils.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_utils.py @@ -17,6 +17,7 @@ from __future__ import annotations import pytest + from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): diff --git a/providers/tests/fab/auth_manager/conftest.py b/providers/tests/fab/auth_manager/conftest.py index 970889e2ec397..d400a7b86a027 100644 --- a/providers/tests/fab/auth_manager/conftest.py +++ b/providers/tests/fab/auth_manager/conftest.py @@ -17,11 +17,12 @@ from __future__ import annotations import pytest -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules from airflow.www import app +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules + @pytest.fixture(scope="session") def minimal_app_for_auth_api(): diff --git a/providers/tests/fab/auth_manager/decorators/test_auth.py b/providers/tests/fab/auth_manager/decorators/test_auth.py index d6f60bf42f1fb..51b79ee25ad9d 100644 --- a/providers/tests/fab/auth_manager/decorators/test_auth.py +++ b/providers/tests/fab/auth_manager/decorators/test_auth.py @@ -19,10 +19,11 @@ from unittest.mock import Mock, patch import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.security.permissions import ACTION_CAN_READ, RESOURCE_DAG +from tests_common.test_utils.compat import ignore_provider_compatibility_error + permissions = [(ACTION_CAN_READ, RESOURCE_DAG)] with ignore_provider_compatibility_error("2.9.0+", __file__): diff --git a/providers/tests/fab/auth_manager/schemas/test_user_schema.py b/providers/tests/fab/auth_manager/schemas/test_user_schema.py index 9f8dbd3742fa8..87cdc42f4a317 100644 --- a/providers/tests/fab/auth_manager/schemas/test_user_schema.py +++ b/providers/tests/fab/auth_manager/schemas/test_user_schema.py @@ -17,11 +17,11 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.utils import timezone from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role +from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import User diff --git a/providers/tests/fab/auth_manager/test_models.py b/providers/tests/fab/auth_manager/test_models.py index 76b69c3dea734..a092bd3e311be 100644 --- a/providers/tests/fab/auth_manager/test_models.py +++ b/providers/tests/fab/auth_manager/test_models.py @@ -19,6 +19,7 @@ from unittest import mock from sqlalchemy import Column, MetaData, String, Table + from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): diff --git a/providers/tests/fab/auth_manager/test_security.py b/providers/tests/fab/auth_manager/test_security.py index 8156999f01615..632ea5cb1f4b2 100644 --- a/providers/tests/fab/auth_manager/test_security.py +++ b/providers/tests/fab/auth_manager/test_security.py @@ -31,23 +31,19 @@ from flask_appbuilder import SQLA, Model, expose, has_access from flask_appbuilder.views import BaseView, ModelView from sqlalchemy import Column, Date, Float, Integer, String -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.configuration import initialize_config from airflow.exceptions import AirflowException from airflow.models import DagModel from airflow.models.dag import DAG +from tests_common.test_utils.compat import ignore_provider_compatibility_error + with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager from airflow.providers.fab.auth_manager.models import assoc_permission_role from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser -from tests_common.test_utils.asserts import assert_queries_count -from tests_common.test_utils.db import clear_db_dags, clear_db_runs -from tests_common.test_utils.mock_security_manager import MockSecurityManager -from tests_common.test_utils.permissions import _resource_name - from airflow.security import permissions from airflow.security.permissions import ACTION_CAN_READ from airflow.www import app as application @@ -62,6 +58,10 @@ delete_user, set_user_single_role, ) +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.db import clear_db_dags, clear_db_runs +from tests_common.test_utils.mock_security_manager import MockSecurityManager +from tests_common.test_utils.permissions import _resource_name if TYPE_CHECKING: from airflow.security.permissions import RESOURCE_ASSET diff --git a/providers/tests/fab/auth_manager/views/test_permissions.py b/providers/tests/fab/auth_manager/views/test_permissions.py index eeb005b47e69a..6402bb42bd8fa 100644 --- a/providers/tests/fab/auth_manager/views/test_permissions.py +++ b/providers/tests/fab/auth_manager/views/test_permissions.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/providers/tests/fab/auth_manager/views/test_roles_list.py b/providers/tests/fab/auth_manager/views/test_roles_list.py index f94907c3e0504..d6d8234f3759c 100644 --- a/providers/tests/fab/auth_manager/views/test_roles_list.py +++ b/providers/tests/fab/auth_manager/views/test_roles_list.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/providers/tests/fab/auth_manager/views/test_user.py b/providers/tests/fab/auth_manager/views/test_user.py index 140d7cd331255..bc2d8eebe7e8c 100644 --- a/providers/tests/fab/auth_manager/views/test_user.py +++ b/providers/tests/fab/auth_manager/views/test_user.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/providers/tests/fab/auth_manager/views/test_user_edit.py b/providers/tests/fab/auth_manager/views/test_user_edit.py index fb994dbd66ec7..2d290275e6d0c 100644 --- a/providers/tests/fab/auth_manager/views/test_user_edit.py +++ b/providers/tests/fab/auth_manager/views/test_user_edit.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/providers/tests/fab/auth_manager/views/test_user_stats.py b/providers/tests/fab/auth_manager/views/test_user_stats.py index bb7d627fdaa64..c6a234ff0417c 100644 --- a/providers/tests/fab/auth_manager/views/test_user_stats.py +++ b/providers/tests/fab/auth_manager/views/test_user_stats.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/providers/tests/google/cloud/hooks/test_bigquery_system.py b/providers/tests/google/cloud/hooks/test_bigquery_system.py index 454ab63079794..9f203e5d4af3d 100644 --- a/providers/tests/google/cloud/hooks/test_bigquery_system.py +++ b/providers/tests/google/cloud/hooks/test_bigquery_system.py @@ -20,11 +20,11 @@ from __future__ import annotations import pytest -from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest from airflow.providers.google.cloud.hooks import bigquery as hook from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest @pytest.mark.system("google.cloud") diff --git a/providers/tests/google/cloud/hooks/test_kms_system.py b/providers/tests/google/cloud/hooks/test_kms_system.py index 36c5bfa207f3a..be2e0cc6cecf0 100644 --- a/providers/tests/google/cloud/hooks/test_kms_system.py +++ b/providers/tests/google/cloud/hooks/test_kms_system.py @@ -21,11 +21,11 @@ from tempfile import TemporaryDirectory import pytest -from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from airflow.providers.google.cloud.hooks.kms import CloudKMSHook from providers.tests.google.cloud.utils.gcp_authenticator import GCP_KMS_KEY +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context # To prevent resource name collisions, key ring and key resources CANNOT be deleted, so # to avoid cluttering the project, we only create the key once during project initialization. diff --git a/providers/tests/google/cloud/hooks/test_secret_manager_system.py b/providers/tests/google/cloud/hooks/test_secret_manager_system.py index 9f512f5f23c8f..74bc08f9d6c79 100644 --- a/providers/tests/google/cloud/hooks/test_secret_manager_system.py +++ b/providers/tests/google/cloud/hooks/test_secret_manager_system.py @@ -19,11 +19,11 @@ import os import pytest -from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from airflow.providers.google.cloud.hooks.secret_manager import SecretsManagerHook from providers.tests.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context TEST_SECRET_ID = os.environ.get("GCP_SECRET_MANAGER_SECRET_ID", "test-secret") TEST_SECRET_VALUE = os.environ.get("GCP_SECRET_MANAGER_SECRET_VALUE", "test-secret-value") diff --git a/providers/tests/google/cloud/log/test_gcs_task_handler.py b/providers/tests/google/cloud/log/test_gcs_task_handler.py index 369f4b888058b..22ad5f06af8d6 100644 --- a/providers/tests/google/cloud/log/test_gcs_task_handler.py +++ b/providers/tests/google/cloud/log/test_gcs_task_handler.py @@ -23,13 +23,14 @@ from unittest.mock import MagicMock import pytest -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.providers.google.cloud.log.gcs_task_handler import GCSTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs + @pytest.mark.db_test class TestGCSTaskHandler: diff --git a/providers/tests/google/cloud/log/test_gcs_task_handler_system.py b/providers/tests/google/cloud/log/test_gcs_task_handler_system.py index e3e2bd2cafccc..bf5c52601884a 100644 --- a/providers/tests/google/cloud/log/test_gcs_task_handler_system.py +++ b/providers/tests/google/cloud/log/test_gcs_task_handler_system.py @@ -23,13 +23,6 @@ from unittest import mock import pytest -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_connections, clear_db_runs -from tests_common.test_utils.gcp_system_helpers import ( - GoogleSystemTest, - provide_gcp_context, - resolve_full_gcp_key_path, -) from airflow import settings from airflow.example_dags import example_complex @@ -38,6 +31,13 @@ from airflow.utils.session import provide_session from providers.tests.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_connections, clear_db_runs +from tests_common.test_utils.gcp_system_helpers import ( + GoogleSystemTest, + provide_gcp_context, + resolve_full_gcp_key_path, +) @pytest.mark.system("google") diff --git a/providers/tests/google/cloud/log/test_stackdriver_task_handler.py b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py index 60752f3b0d3f3..947e6906291b4 100644 --- a/providers/tests/google/cloud/log/test_stackdriver_task_handler.py +++ b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py @@ -24,15 +24,16 @@ import pytest from google.cloud.logging import Resource from google.cloud.logging_v2.types import ListLogEntriesRequest, ListLogEntriesResponse, LogEntry -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.exceptions import RemovedInAirflow3Warning from airflow.providers.google.cloud.log.stackdriver_task_handler import StackdriverTaskHandler from airflow.utils import timezone from airflow.utils.state import TaskInstanceState +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs + def _create_list_log_entries_response_mock(messages, token): return ListLogEntriesResponse( diff --git a/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py b/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py index da01e9c770570..9e67ec2e5548c 100644 --- a/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py +++ b/providers/tests/google/cloud/log/test_stackdriver_task_handler_system.py @@ -23,13 +23,6 @@ from unittest import mock import pytest -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_runs -from tests_common.test_utils.gcp_system_helpers import ( - GoogleSystemTest, - provide_gcp_context, - resolve_full_gcp_key_path, -) from airflow import settings from airflow.example_dags import example_complex @@ -38,6 +31,13 @@ from airflow.utils.session import provide_session from providers.tests.google.cloud.utils.gcp_authenticator import GCP_STACKDRIVER +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_runs +from tests_common.test_utils.gcp_system_helpers import ( + GoogleSystemTest, + provide_gcp_context, + resolve_full_gcp_key_path, +) @pytest.mark.system("google") diff --git a/providers/tests/google/cloud/operators/test_bigquery.py b/providers/tests/google/cloud/operators/test_bigquery.py index 1edc67c45a761..a3beddb9d8732 100644 --- a/providers/tests/google/cloud/operators/test_bigquery.py +++ b/providers/tests/google/cloud/operators/test_bigquery.py @@ -27,12 +27,6 @@ import pytest from google.cloud.bigquery import DEFAULT_RETRY, ScalarQueryParameter from google.cloud.exceptions import Conflict -from tests_common.test_utils.db import ( - clear_db_dags, - clear_db_runs, - clear_db_serialized_dags, - clear_db_xcom, -) from airflow.exceptions import ( AirflowException, @@ -81,6 +75,13 @@ from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.timezone import datetime +from tests_common.test_utils.db import ( + clear_db_dags, + clear_db_runs, + clear_db_serialized_dags, + clear_db_xcom, +) + pytestmark = pytest.mark.db_test diff --git a/providers/tests/google/cloud/operators/test_bigquery_dts.py b/providers/tests/google/cloud/operators/test_bigquery_dts.py index b196237b5bf84..189c2623fcf3d 100644 --- a/providers/tests/google/cloud/operators/test_bigquery_dts.py +++ b/providers/tests/google/cloud/operators/test_bigquery_dts.py @@ -21,7 +21,6 @@ from google.api_core.gapic_v1.method import DEFAULT from google.cloud.bigquery_datatransfer_v1 import StartManualTransferRunsResponse, TransferConfig, TransferRun -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.providers.google.cloud.operators.bigquery_dts import ( BigQueryCreateDataTransferOperator, @@ -29,6 +28,8 @@ BigQueryDeleteDataTransferConfigOperator, ) +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + PROJECT_ID = "id" diff --git a/providers/tests/google/cloud/operators/test_dataprep_system.py b/providers/tests/google/cloud/operators/test_dataprep_system.py index 9706859f02577..57da7cf5dd40e 100644 --- a/providers/tests/google/cloud/operators/test_dataprep_system.py +++ b/providers/tests/google/cloud/operators/test_dataprep_system.py @@ -21,13 +21,14 @@ import os import pytest -from tests_common.test_utils.db import clear_db_connections -from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest -from tests_common.test_utils.system_tests import get_test_run from airflow.models import Connection from airflow.utils.session import create_session +from tests_common.test_utils.db import clear_db_connections +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest +from tests_common.test_utils.system_tests import get_test_run + TOKEN = os.environ.get("DATAPREP_TOKEN") EXTRA = {"token": TOKEN} diff --git a/providers/tests/google/cloud/operators/test_dataproc.py b/providers/tests/google/cloud/operators/test_dataproc.py index a8b747d564a6f..babe432a2fe17 100644 --- a/providers/tests/google/cloud/operators/test_dataproc.py +++ b/providers/tests/google/cloud/operators/test_dataproc.py @@ -27,8 +27,6 @@ from google.api_core.retry_async import AsyncRetry from google.cloud import dataproc from google.cloud.dataproc_v1 import Batch, Cluster, JobStatus -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_VERSION -from tests_common.test_utils.db import clear_db_runs, clear_db_xcom from airflow.exceptions import ( AirflowException, @@ -82,6 +80,9 @@ from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.timezone import datetime +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_VERSION +from tests_common.test_utils.db import clear_db_runs, clear_db_xcom + AIRFLOW_VERSION_LABEL = "v" + str(AIRFLOW_VERSION).replace(".", "-").replace("+", "-") cluster_params = inspect.signature(ClusterGenerator.__init__).parameters diff --git a/providers/tests/google/cloud/operators/test_datastore_system.py b/providers/tests/google/cloud/operators/test_datastore_system.py index 3a9e82241146c..8c78479bebfd2 100644 --- a/providers/tests/google/cloud/operators/test_datastore_system.py +++ b/providers/tests/google/cloud/operators/test_datastore_system.py @@ -20,14 +20,14 @@ import os import pytest + +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_DATASTORE_KEY from tests_common.test_utils.gcp_system_helpers import ( CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context, ) -from providers.tests.google.cloud.utils.gcp_authenticator import GCP_DATASTORE_KEY - BUCKET = os.environ.get("GCP_DATASTORE_BUCKET", "datastore-system-test") diff --git a/providers/tests/google/cloud/operators/test_looker.py b/providers/tests/google/cloud/operators/test_looker.py index faefc0eca5a08..d24c3cae85ad0 100644 --- a/providers/tests/google/cloud/operators/test_looker.py +++ b/providers/tests/google/cloud/operators/test_looker.py @@ -20,13 +20,14 @@ from unittest.mock import MagicMock import pytest -from tests_common.test_utils.db import clear_db_runs, clear_db_xcom from airflow.exceptions import AirflowException from airflow.models import DAG, DagBag from airflow.providers.google.cloud.operators.looker import LookerStartPdtBuildOperator from airflow.utils.timezone import datetime +from tests_common.test_utils.db import clear_db_runs, clear_db_xcom + OPERATOR_PATH = "airflow.providers.google.cloud.operators.looker.{}" TASK_ID = "task-id" diff --git a/providers/tests/google/cloud/secrets/test_secret_manager_system.py b/providers/tests/google/cloud/secrets/test_secret_manager_system.py index 04b7a9fe1f56f..191329afb4003 100644 --- a/providers/tests/google/cloud/secrets/test_secret_manager_system.py +++ b/providers/tests/google/cloud/secrets/test_secret_manager_system.py @@ -22,9 +22,9 @@ from unittest import mock import pytest -from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context from providers.tests.google.cloud.utils.gcp_authenticator import GCP_SECRET_MANAGER_KEY +from tests_common.test_utils.gcp_system_helpers import GoogleSystemTest, provide_gcp_context BACKEND_IMPORT_PATH = "airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend" diff --git a/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py index 4942a36648c21..d574b4d0aaa52 100644 --- a/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_facebook_ads_to_gcs_system.py @@ -22,11 +22,6 @@ from contextlib import contextmanager import pytest -from tests_common.test_utils.gcp_system_helpers import ( - GoogleSystemTest, - provide_gcp_context, -) -from tests_common.test_utils.system_tests import get_test_run from airflow.exceptions import AirflowException from airflow.models import Connection @@ -34,6 +29,11 @@ from airflow.utils.process_utils import patch_environ from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY +from tests_common.test_utils.gcp_system_helpers import ( + GoogleSystemTest, + provide_gcp_context, +) +from tests_common.test_utils.system_tests import get_test_run CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") FACEBOOK_KEY = "facebook.json" diff --git a/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py index e525ca1adde21..358f84266e12d 100644 --- a/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py +++ b/providers/tests/google/cloud/transfers/test_salesforce_to_gcs_system.py @@ -19,6 +19,10 @@ import os import pytest + +from airflow.providers.google.cloud.example_dags import example_salesforce_to_gcs + +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY from tests_common.test_utils.gcp_system_helpers import ( GoogleSystemTest, provide_gcp_context, @@ -26,10 +30,6 @@ from tests_common.test_utils.salesforce_system_helpers import provide_salesforce_connection from tests_common.test_utils.system_tests import get_test_run -from airflow.providers.google.cloud.example_dags import example_salesforce_to_gcs - -from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY - CREDENTIALS_DIR = os.environ.get("CREDENTIALS_DIR", "/files/airflow-breeze-config/keys") SALESFORCE_KEY = "salesforce.json" SALESFORCE_CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, SALESFORCE_KEY) diff --git a/providers/tests/google/cloud/utils/gcp_authenticator.py b/providers/tests/google/cloud/utils/gcp_authenticator.py index 36bca98e9a6a7..cee778b9f634a 100644 --- a/providers/tests/google/cloud/utils/gcp_authenticator.py +++ b/providers/tests/google/cloud/utils/gcp_authenticator.py @@ -21,13 +21,13 @@ import os import subprocess -from tests_common.test_utils import AIRFLOW_MAIN_FOLDER -from tests_common.test_utils.logging_command_executor import CommandExecutor - from airflow import settings from airflow.exceptions import AirflowException from airflow.models import Connection +from tests_common.test_utils import AIRFLOW_MAIN_FOLDER +from tests_common.test_utils.logging_command_executor import CommandExecutor + # Please keep these variables in alphabetical order. GCP_AI_KEY = "gcp_ai.json" diff --git a/providers/tests/google/common/auth_backend/test_google_openid.py b/providers/tests/google/common/auth_backend/test_google_openid.py index 02f08e6107070..dc5b0f3553d6c 100644 --- a/providers/tests/google/common/auth_backend/test_google_openid.py +++ b/providers/tests/google/common/auth_backend/test_google_openid.py @@ -20,13 +20,14 @@ import pytest from google.auth.exceptions import GoogleAuthError + +from airflow.www.app import create_app + from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_pools from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules -from airflow.www.app import create_app - @pytest.fixture(scope="module") def google_openid_app(): diff --git a/providers/tests/google/marketing_platform/operators/test_display_video_system.py b/providers/tests/google/marketing_platform/operators/test_display_video_system.py index 57e93a0544a15..bface0ebc59b4 100644 --- a/providers/tests/google/marketing_platform/operators/test_display_video_system.py +++ b/providers/tests/google/marketing_platform/operators/test_display_video_system.py @@ -17,12 +17,6 @@ from __future__ import annotations import pytest -from tests_common.test_utils.gcp_system_helpers import ( - MARKETING_DAG_FOLDER, - GoogleSystemTest, - provide_gcp_context, -) -from tests_common.test_utils.system_tests import get_test_run from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.providers.google.marketing_platform.example_dags.example_display_video import ( @@ -32,6 +26,12 @@ ) from providers.tests.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GMP_KEY +from tests_common.test_utils.gcp_system_helpers import ( + MARKETING_DAG_FOLDER, + GoogleSystemTest, + provide_gcp_context, +) +from tests_common.test_utils.system_tests import get_test_run # Requires the following scope: SCOPES = [ diff --git a/providers/tests/hashicorp/hooks/test_vault.py b/providers/tests/hashicorp/hooks/test_vault.py index 8aa555567bb4c..ed9587cd2534c 100644 --- a/providers/tests/hashicorp/hooks/test_vault.py +++ b/providers/tests/hashicorp/hooks/test_vault.py @@ -22,12 +22,13 @@ import pytest from hvac.exceptions import VaultError -from tests_common.test_utils.config import conf_vars from airflow.configuration import AirflowConfigParser from airflow.exceptions import AirflowConfigException from airflow.providers.hashicorp.hooks.vault import VaultHook +from tests_common.test_utils.config import conf_vars + class TestVaultHook: @staticmethod diff --git a/providers/tests/imap/hooks/test_imap.py b/providers/tests/imap/hooks/test_imap.py index 9d2fcad5c74b3..51a8721f7a952 100644 --- a/providers/tests/imap/hooks/test_imap.py +++ b/providers/tests/imap/hooks/test_imap.py @@ -22,13 +22,14 @@ from unittest.mock import Mock, mock_open, patch import pytest -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.imap.hooks.imap import ImapHook from airflow.utils import db +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/microsoft/azure/log/test_wasb_task_handler.py b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py index 2200a65f4d0d7..5f5493e9ff06d 100644 --- a/providers/tests/microsoft/azure/log/test_wasb_task_handler.py +++ b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py @@ -24,14 +24,15 @@ import pytest from azure.common import AzureHttpError -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.providers.microsoft.azure.hooks.wasb import WasbHook from airflow.providers.microsoft.azure.log.wasb_task_handler import WasbTaskHandler from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs + pytestmark = pytest.mark.db_test diff --git a/providers/tests/mongo/hooks/test_mongo.py b/providers/tests/mongo/hooks/test_mongo.py index 9cd5b4f0677d5..01e58f6081e37 100644 --- a/providers/tests/mongo/hooks/test_mongo.py +++ b/providers/tests/mongo/hooks/test_mongo.py @@ -23,12 +23,13 @@ import pymongo import pytest -from tests_common.test_utils.compat import connection_as_json from airflow.exceptions import AirflowConfigException, AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.mongo.hooks.mongo import MongoHook +from tests_common.test_utils.compat import connection_as_json + pytestmark = pytest.mark.db_test if TYPE_CHECKING: diff --git a/providers/tests/mysql/hooks/test_mysql.py b/providers/tests/mysql/hooks/test_mysql.py index 49358ac30dcff..6643c6f5b407d 100644 --- a/providers/tests/mysql/hooks/test_mysql.py +++ b/providers/tests/mysql/hooks/test_mysql.py @@ -36,10 +36,10 @@ pytest.skip("MySQL not available", allow_module_level=True) -from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces - from airflow.utils import timezone +from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces + SSL_DICT = {"cert": "/tmp/client-cert.pem", "ca": "/tmp/server-ca.pem", "key": "/tmp/client-key.pem"} diff --git a/providers/tests/openlineage/extractors/test_bash.py b/providers/tests/openlineage/extractors/test_bash.py index 8782ce161c637..31ef2ce80d65d 100644 --- a/providers/tests/openlineage/extractors/test_bash.py +++ b/providers/tests/openlineage/extractors/test_bash.py @@ -23,12 +23,13 @@ import pytest from openlineage.client.facet_v2 import source_code_job -from tests_common.test_utils.compat import BashOperator from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.openlineage.extractors.bash import BashExtractor +from tests_common.test_utils.compat import BashOperator + pytestmark = pytest.mark.db_test with DAG( diff --git a/providers/tests/openlineage/extractors/test_manager.py b/providers/tests/openlineage/extractors/test_manager.py index dbd4a8e40368c..044c0df147672 100644 --- a/providers/tests/openlineage/extractors/test_manager.py +++ b/providers/tests/openlineage/extractors/test_manager.py @@ -24,7 +24,6 @@ import pytest from openlineage.client.event_v2 import Dataset as OpenLineageDataset from openlineage.client.facet_v2 import documentation_dataset, ownership_dataset, schema_dataset -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS from airflow.io.path import ObjectStoragePath from airflow.lineage.entities import Column, File, Table, User @@ -36,6 +35,8 @@ from airflow.providers.openlineage.utils.utils import Asset from airflow.utils.state import State +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS + if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/providers/tests/openlineage/extractors/test_python.py b/providers/tests/openlineage/extractors/test_python.py index 92a9d073a1109..367eb0711dea5 100644 --- a/providers/tests/openlineage/extractors/test_python.py +++ b/providers/tests/openlineage/extractors/test_python.py @@ -25,13 +25,14 @@ import pytest from openlineage.client.facet_v2 import source_code_job -from tests_common.test_utils.compat import BashOperator from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.operators.python import PythonOperator from airflow.providers.openlineage.extractors.python import PythonExtractor +from tests_common.test_utils.compat import BashOperator + pytestmark = pytest.mark.db_test dag = DAG( diff --git a/providers/tests/openlineage/plugins/test_adapter.py b/providers/tests/openlineage/plugins/test_adapter.py index 8d57fad296279..a2af9f393da58 100644 --- a/providers/tests/openlineage/plugins/test_adapter.py +++ b/providers/tests/openlineage/plugins/test_adapter.py @@ -36,8 +36,6 @@ processing_engine_run, sql_job, ) -from tests_common.test_utils.compat import BashOperator -from tests_common.test_utils.config import conf_vars from airflow import DAG from airflow.models.dagrun import DagRun, DagRunState @@ -54,6 +52,9 @@ from airflow.providers.openlineage.utils.utils import get_airflow_job_facet from airflow.utils.task_group import TaskGroup +from tests_common.test_utils.compat import BashOperator +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/openlineage/plugins/test_execution.py b/providers/tests/openlineage/plugins/test_execution.py index 09262adc4a6f9..0c0f9fc6adac2 100644 --- a/providers/tests/openlineage/plugins/test_execution.py +++ b/providers/tests/openlineage/plugins/test_execution.py @@ -26,8 +26,6 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.config import conf_vars from airflow.jobs.job import Job from airflow.jobs.local_task_job_runner import LocalTaskJobRunner @@ -39,6 +37,9 @@ from airflow.utils import timezone from airflow.utils.state import State +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/openlineage/plugins/test_listener.py b/providers/tests/openlineage/plugins/test_listener.py index b859160cf8424..1c17b5841674d 100644 --- a/providers/tests/openlineage/plugins/test_listener.py +++ b/providers/tests/openlineage/plugins/test_listener.py @@ -29,8 +29,6 @@ from openlineage.client import OpenLineageClient from openlineage.client.transport import ConsoleTransport from openlineage.client.transport.console import ConsoleConfig -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.config import conf_vars from airflow.models import DAG, DagRun, TaskInstance from airflow.models.baseoperator import BaseOperator @@ -41,6 +39,9 @@ from airflow.providers.openlineage.utils.selective_enable import disable_lineage, enable_lineage from airflow.utils.state import DagRunState, State +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.config import conf_vars + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/openlineage/plugins/test_openlineage.py b/providers/tests/openlineage/plugins/test_openlineage.py index e1c36001ebfeb..cb50dfc817004 100644 --- a/providers/tests/openlineage/plugins/test_openlineage.py +++ b/providers/tests/openlineage/plugins/test_openlineage.py @@ -22,6 +22,7 @@ from unittest.mock import patch import pytest + from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES from tests_common.test_utils.config import conf_vars diff --git a/providers/tests/openlineage/plugins/test_utils.py b/providers/tests/openlineage/plugins/test_utils.py index f0685c1351685..12f7ed32a8259 100644 --- a/providers/tests/openlineage/plugins/test_utils.py +++ b/providers/tests/openlineage/plugins/test_utils.py @@ -27,7 +27,6 @@ from attrs import define from openlineage.client.utils import RedactMixin from pkg_resources import parse_version -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator from airflow.models import DAG as AIRFLOW_DAG, DagModel from airflow.providers.openlineage.plugins.facets import AirflowDebugRunFacet @@ -45,6 +44,8 @@ from airflow.utils.log.secrets_masker import _secrets_masker from airflow.utils.state import State +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/openlineage/test_conf.py b/providers/tests/openlineage/test_conf.py index b1dd57d4d8415..4944ba86e47f1 100644 --- a/providers/tests/openlineage/test_conf.py +++ b/providers/tests/openlineage/test_conf.py @@ -20,7 +20,6 @@ from unittest import mock import pytest -from tests_common.test_utils.config import conf_vars, env_vars from airflow.exceptions import AirflowConfigException from airflow.providers.openlineage.conf import ( @@ -40,6 +39,8 @@ transport, ) +from tests_common.test_utils.config import conf_vars, env_vars + _CONFIG_SECTION = "openlineage" _VAR_CONFIG_PATH = "OPENLINEAGE_CONFIG" _CONFIG_OPTION_CONFIG_PATH = "config_path" diff --git a/providers/tests/openlineage/utils/test_utils.py b/providers/tests/openlineage/utils/test_utils.py index 5adeceb440dc1..c41f088b66143 100644 --- a/providers/tests/openlineage/utils/test_utils.py +++ b/providers/tests/openlineage/utils/test_utils.py @@ -21,9 +21,6 @@ import pathlib from unittest.mock import MagicMock, patch -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator -from tests_common.test_utils.mock_operators import MockOperator - from airflow import DAG from airflow.decorators import task from airflow.models.baseoperator import BaseOperator @@ -47,6 +44,9 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator +from tests_common.test_utils.mock_operators import MockOperator + BASH_OPERATOR_PATH = "airflow.providers.standard.operators.bash" if not AIRFLOW_V_2_10_PLUS: BASH_OPERATOR_PATH = "airflow.operators.bash" diff --git a/providers/tests/opensearch/log/test_os_task_handler.py b/providers/tests/opensearch/log/test_os_task_handler.py index 81ac75c855aa9..36e07893fdbb9 100644 --- a/providers/tests/opensearch/log/test_os_task_handler.py +++ b/providers/tests/opensearch/log/test_os_task_handler.py @@ -32,8 +32,6 @@ opensearchpy = pytest.importorskip("opensearchpy") from opensearchpy.exceptions import NotFoundError -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from airflow.configuration import conf from airflow.providers.opensearch.log.os_response import OpensearchResponse @@ -47,6 +45,8 @@ from airflow.utils.timezone import datetime from providers.tests.opensearch.conftest import MockClient +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/providers/tests/redis/log/test_redis_task_handler.py b/providers/tests/redis/log/test_redis_task_handler.py index 0d83f3a303136..b010b18191a4a 100644 --- a/providers/tests/redis/log/test_redis_task_handler.py +++ b/providers/tests/redis/log/test_redis_task_handler.py @@ -21,7 +21,6 @@ from unittest.mock import patch import pytest -from tests_common.test_utils.config import conf_vars from airflow.models import DAG, DagRun, TaskInstance from airflow.operators.empty import EmptyOperator @@ -30,6 +29,8 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/sftp/operators/test_sftp.py b/providers/tests/sftp/operators/test_sftp.py index 9d5689c881485..7b7e79cb881ef 100644 --- a/providers/tests/sftp/operators/test_sftp.py +++ b/providers/tests/sftp/operators/test_sftp.py @@ -25,7 +25,6 @@ import paramiko import pytest -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import DAG, Connection @@ -37,6 +36,8 @@ from airflow.utils import timezone from airflow.utils.timezone import datetime +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/smtp/hooks/test_smtp.py b/providers/tests/smtp/hooks/test_smtp.py index fe9471c424933..06fbb246c215e 100644 --- a/providers/tests/smtp/hooks/test_smtp.py +++ b/providers/tests/smtp/hooks/test_smtp.py @@ -25,13 +25,14 @@ from unittest.mock import Mock, patch import pytest -from tests_common.test_utils.config import conf_vars from airflow.models import Connection from airflow.providers.smtp.hooks.smtp import SmtpHook from airflow.utils import db from airflow.utils.session import create_session +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/smtp/notifications/test_smtp.py b/providers/tests/smtp/notifications/test_smtp.py index 95b1618aa4fb3..934a4d406a40e 100644 --- a/providers/tests/smtp/notifications/test_smtp.py +++ b/providers/tests/smtp/notifications/test_smtp.py @@ -21,8 +21,6 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS -from tests_common.test_utils.config import conf_vars from airflow.configuration import conf from airflow.operators.empty import EmptyOperator @@ -33,6 +31,9 @@ ) from airflow.utils import timezone +from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test SMTP_API_DEFAULT_CONN_ID = SmtpHook.default_conn_name diff --git a/providers/tests/ssh/operators/test_ssh.py b/providers/tests/ssh/operators/test_ssh.py index 55431bd3b629e..810f5363d1bf5 100644 --- a/providers/tests/ssh/operators/test_ssh.py +++ b/providers/tests/ssh/operators/test_ssh.py @@ -24,7 +24,6 @@ import pytest from paramiko.client import SSHClient -from tests_common.test_utils.config import conf_vars from airflow.exceptions import AirflowException, AirflowSkipException, AirflowTaskTimeout from airflow.models import TaskInstance @@ -33,6 +32,8 @@ from airflow.utils.timezone import datetime from airflow.utils.types import NOTSET +from tests_common.test_utils.config import conf_vars + pytestmark = pytest.mark.db_test diff --git a/providers/tests/standard/operators/test_bash.py b/providers/tests/standard/operators/test_bash.py index f040442ecccd3..a3b8a0b82186b 100644 --- a/providers/tests/standard/operators/test_bash.py +++ b/providers/tests/standard/operators/test_bash.py @@ -26,7 +26,6 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.exceptions import AirflowException, AirflowSkipException, AirflowTaskTimeout from airflow.providers.standard.operators.bash import BashOperator @@ -34,6 +33,8 @@ from airflow.utils.state import State from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/standard/operators/test_datetime.py b/providers/tests/standard/operators/test_datetime.py index da7f98f0e3543..b729781f19fd3 100644 --- a/providers/tests/standard/operators/test_datetime.py +++ b/providers/tests/standard/operators/test_datetime.py @@ -21,7 +21,6 @@ import pytest import time_machine -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.exceptions import AirflowException from airflow.models.dagrun import DagRun @@ -32,6 +31,8 @@ from airflow.utils.session import create_session from airflow.utils.state import State +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/standard/operators/test_weekday.py b/providers/tests/standard/operators/test_weekday.py index 576fd10b54542..87ceed00ef624 100644 --- a/providers/tests/standard/operators/test_weekday.py +++ b/providers/tests/standard/operators/test_weekday.py @@ -21,7 +21,6 @@ import pytest import time_machine -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from airflow.exceptions import AirflowException from airflow.models.dagrun import DagRun @@ -34,6 +33,8 @@ from airflow.utils.state import State from airflow.utils.weekday import WeekDay +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/providers/tests/standard/sensors/test_weekday.py b/providers/tests/standard/sensors/test_weekday.py index 5bdbf4bb79c81..ab3960c4119f7 100644 --- a/providers/tests/standard/sensors/test_weekday.py +++ b/providers/tests/standard/sensors/test_weekday.py @@ -20,7 +20,6 @@ from datetime import timedelta import pytest -from tests_common.test_utils import db from airflow.exceptions import AirflowSensorTimeout from airflow.models import DagBag @@ -29,6 +28,8 @@ from airflow.utils.timezone import datetime from airflow.utils.weekday import WeekDay +from tests_common.test_utils import db + pytestmark = pytest.mark.db_test diff --git a/providers/tests/system/amazon/aws/example_s3_to_sql.py b/providers/tests/system/amazon/aws/example_s3_to_sql.py index d7f452bfeb087..6ca4f8aa9fe9f 100644 --- a/providers/tests/system/amazon/aws/example_s3_to_sql.py +++ b/providers/tests/system/amazon/aws/example_s3_to_sql.py @@ -18,8 +18,6 @@ from datetime import datetime -from tests_common.test_utils.watcher import watcher - from airflow import settings from airflow.decorators import task from airflow.models import Connection @@ -43,6 +41,7 @@ from airflow.utils.trigger_rule import TriggerRule from providers.tests.system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder +from tests_common.test_utils.watcher import watcher # Externally fetched variables: SECURITY_GROUP_KEY = "SECURITY_GROUP" diff --git a/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py b/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py index 2656474cbbf51..c4cf206577ddd 100644 --- a/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py +++ b/providers/tests/system/amazon/aws/tests/test_aws_auth_manager.py @@ -21,12 +21,12 @@ import boto3 import pytest -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.www import check_content_in_response from airflow.www import app as application from providers.tests.system.amazon.aws.utils import set_env_id +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response SAML_METADATA_URL = "/saml/metadata" SAML_METADATA_PARSED = { diff --git a/providers/tests/system/dbt/cloud/example_dbt_cloud.py b/providers/tests/system/dbt/cloud/example_dbt_cloud.py index 7a9a58a87a67b..b36f7439fd698 100644 --- a/providers/tests/system/dbt/cloud/example_dbt_cloud.py +++ b/providers/tests/system/dbt/cloud/example_dbt_cloud.py @@ -18,8 +18,6 @@ from datetime import datetime -from tests_common.test_utils.system_tests import get_test_env_id - from airflow.models import DAG from airflow.operators.empty import EmptyOperator from airflow.providers.dbt.cloud.operators.dbt import ( @@ -30,6 +28,8 @@ from airflow.providers.dbt.cloud.sensors.dbt import DbtCloudJobRunSensor from airflow.utils.edgemodifier import Label +from tests_common.test_utils.system_tests import get_test_env_id + ENV_ID = get_test_env_id() DAG_ID = "example_dbt_cloud" diff --git a/providers/tests/system/yandex/example_yandexcloud.py b/providers/tests/system/yandex/example_yandexcloud.py index 85284af2642ef..3cb7226208baa 100644 --- a/providers/tests/system/yandex/example_yandexcloud.py +++ b/providers/tests/system/yandex/example_yandexcloud.py @@ -27,13 +27,14 @@ import yandex.cloud.dataproc.v1.job_service_pb2_grpc as job_service_grpc_pb import yandex.cloud.dataproc.v1.subcluster_pb2 as subcluster_pb from google.protobuf.json_format import MessageToDict -from tests_common.test_utils.system_tests import get_test_env_id from yandexcloud.operations import OperationError from airflow import DAG from airflow.decorators import task from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook +from tests_common.test_utils.system_tests import get_test_env_id + ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_hook" diff --git a/providers/tests/system/yandex/example_yandexcloud_dataproc.py b/providers/tests/system/yandex/example_yandexcloud_dataproc.py index 13cabfc0c48a8..53cea189d8036 100644 --- a/providers/tests/system/yandex/example_yandexcloud_dataproc.py +++ b/providers/tests/system/yandex/example_yandexcloud_dataproc.py @@ -19,8 +19,6 @@ import uuid from datetime import datetime -from tests_common.test_utils.system_tests import get_test_env_id - from airflow import DAG from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, @@ -34,6 +32,8 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule +from tests_common.test_utils.system_tests import get_test_env_id + # should be filled with appropriate ids diff --git a/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py b/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py index fe77142f3d069..92b7fd30c67d9 100644 --- a/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py +++ b/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py @@ -18,8 +18,6 @@ from datetime import datetime -from tests_common.test_utils.system_tests import get_test_env_id - from airflow import DAG from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, @@ -30,6 +28,8 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule +from tests_common.test_utils.system_tests import get_test_env_id + # should be filled with appropriate ids diff --git a/providers/tests/system/yandex/example_yandexcloud_yq.py b/providers/tests/system/yandex/example_yandexcloud_yq.py index a83e8bd789f97..474e3018ca7ee 100644 --- a/providers/tests/system/yandex/example_yandexcloud_yq.py +++ b/providers/tests/system/yandex/example_yandexcloud_yq.py @@ -18,12 +18,12 @@ from datetime import datetime -from tests_common.test_utils.system_tests import get_test_env_id - from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator from airflow.providers.yandex.operators.yq import YQExecuteQueryOperator +from tests_common.test_utils.system_tests import get_test_env_id + ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_yq" diff --git a/providers/tests/yandex/hooks/test_yandex.py b/providers/tests/yandex/hooks/test_yandex.py index eacb2708e944a..7066c3e7553b1 100644 --- a/providers/tests/yandex/hooks/test_yandex.py +++ b/providers/tests/yandex/hooks/test_yandex.py @@ -25,11 +25,11 @@ yandexcloud = pytest.importorskip("yandexcloud") -from tests_common.test_utils.config import conf_vars - from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook +from tests_common.test_utils.config import conf_vars + class TestYandexHook: @mock.patch("airflow.hooks.base.BaseHook.get_connection") diff --git a/providers/tests/yandex/links/test_yq.py b/providers/tests/yandex/links/test_yq.py index dfd255d41c0eb..e1033366ae06b 100644 --- a/providers/tests/yandex/links/test_yq.py +++ b/providers/tests/yandex/links/test_yq.py @@ -19,13 +19,14 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.mock_operators import MockOperator from airflow.models.taskinstance import TaskInstance from airflow.models.xcom import XCom from airflow.providers.yandex.links.yq import YQLink +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.mock_operators import MockOperator + yandexcloud = pytest.importorskip("yandexcloud") diff --git a/providers/tests/yandex/operators/test_yq.py b/providers/tests/yandex/operators/test_yq.py index 8c5204497230d..32a454c7e90bf 100644 --- a/providers/tests/yandex/operators/test_yq.py +++ b/providers/tests/yandex/operators/test_yq.py @@ -21,6 +21,7 @@ from unittest.mock import MagicMock, call, patch import pytest + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS yandexcloud = pytest.importorskip("yandexcloud") diff --git a/tests/always/test_connection.py b/tests/always/test_connection.py index e289c7d2172ed..03a8e136f2a0a 100644 --- a/tests/always/test_connection.py +++ b/tests/always/test_connection.py @@ -33,6 +33,7 @@ from airflow.models import Connection, crypto from airflow.providers.sqlite.hooks.sqlite import SqliteHook from airflow.providers_manager import HookInfo + from tests_common.test_utils.config import conf_vars ConnectionParts = namedtuple("ConnectionParts", ["conn_type", "login", "password", "host", "port", "schema"]) diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index edfc9b46f916e..db482aadd59ef 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -29,6 +29,7 @@ from airflow.models import DagBag from airflow.utils import yaml + from tests_common.test_utils.asserts import assert_queries_count AIRFLOW_SOURCES_ROOT = Path(__file__).resolve().parents[2] diff --git a/tests/always/test_secrets.py b/tests/always/test_secrets.py index 1e47fc13ce7c9..e58c8ccac6ec3 100644 --- a/tests/always/test_secrets.py +++ b/tests/always/test_secrets.py @@ -24,6 +24,7 @@ from airflow.configuration import ensure_secrets_loaded, initialize_secrets_backends from airflow.models import Connection, Variable from airflow.secrets.cache import SecretCache + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_variables diff --git a/tests/always/test_secrets_backends.py b/tests/always/test_secrets_backends.py index a74c7ac4e95f0..9a87de3f3c3b4 100644 --- a/tests/always/test_secrets_backends.py +++ b/tests/always/test_secrets_backends.py @@ -28,6 +28,7 @@ from airflow.secrets.environment_variables import EnvironmentVariablesBackend from airflow.secrets.metastore import MetastoreBackend from airflow.utils.session import create_session + from tests_common.test_utils.db import clear_db_connections, clear_db_variables pytestmark = pytest.mark.db_test diff --git a/tests/always/test_secrets_local_filesystem.py b/tests/always/test_secrets_local_filesystem.py index 0a84120d2d087..e4f7de7df6722 100644 --- a/tests/always/test_secrets_local_filesystem.py +++ b/tests/always/test_secrets_local_filesystem.py @@ -29,6 +29,7 @@ from airflow.models import Variable from airflow.secrets import local_filesystem from airflow.secrets.local_filesystem import LocalFilesystemBackend + from tests_common.test_utils.config import conf_vars diff --git a/tests/api_connexion/conftest.py b/tests/api_connexion/conftest.py index 0d91a8dc34a76..39e8da7212ae7 100644 --- a/tests/api_connexion/conftest.py +++ b/tests/api_connexion/conftest.py @@ -19,6 +19,7 @@ import pytest from airflow.www import app + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index e02844561f8c4..a790fb986f255 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -36,6 +36,7 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.types import DagRunType + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.config import conf_vars diff --git a/tests/api_connexion/endpoints/test_backfill_endpoint.py b/tests/api_connexion/endpoints/test_backfill_endpoint.py index bc5a214654368..d17131f083dd2 100644 --- a/tests/api_connexion/endpoints/test_backfill_endpoint.py +++ b/tests/api_connexion/endpoints/test_backfill_endpoint.py @@ -29,6 +29,7 @@ from airflow.operators.empty import EmptyOperator from airflow.utils import timezone from airflow.utils.session import provide_session + from tests_common.test_utils.api_connexion_utils import create_user, delete_user from tests_common.test_utils.db import ( clear_db_backfills, diff --git a/tests/api_connexion/endpoints/test_connection_endpoint.py b/tests/api_connexion/endpoints/test_connection_endpoint.py index b82fac609bd9e..e23a43a14c4f1 100644 --- a/tests/api_connexion/endpoints/test_connection_endpoint.py +++ b/tests/api_connexion/endpoints/test_connection_endpoint.py @@ -25,6 +25,7 @@ from airflow.models import Connection from airflow.secrets.environment_variables import CONN_ENV_PREFIX from airflow.utils.session import provide_session + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_connections diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py index fd3b8b7d90d17..f197b24f0a406 100644 --- a/tests/api_connexion/endpoints/test_dag_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_endpoint.py @@ -30,6 +30,7 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.session import provide_session from airflow.utils.state import TaskInstanceState + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags diff --git a/tests/api_connexion/endpoints/test_dag_parsing.py b/tests/api_connexion/endpoints/test_dag_parsing.py index 53648150c2c01..ce41b9b3b4251 100644 --- a/tests/api_connexion/endpoints/test_dag_parsing.py +++ b/tests/api_connexion/endpoints/test_dag_parsing.py @@ -24,6 +24,7 @@ from airflow.models import DagBag from airflow.models.dagbag import DagPriorityParsingRequest + from tests_common.test_utils.api_connexion_utils import create_user, delete_user from tests_common.test_utils.db import clear_db_dag_parsing_requests diff --git a/tests/api_connexion/endpoints/test_dag_run_endpoint.py b/tests/api_connexion/endpoints/test_dag_run_endpoint.py index bad9ff1481c4d..537ae55ebee8a 100644 --- a/tests/api_connexion/endpoints/test_dag_run_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_run_endpoint.py @@ -34,6 +34,7 @@ from airflow.utils.session import create_session, provide_session from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars diff --git a/tests/api_connexion/endpoints/test_dag_source_endpoint.py b/tests/api_connexion/endpoints/test_dag_source_endpoint.py index 8a5c91ae670a3..4b8c4f3c47320 100644 --- a/tests/api_connexion/endpoints/test_dag_source_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_source_endpoint.py @@ -23,6 +23,7 @@ import pytest from airflow.models import DagBag + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags diff --git a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py index 179662cd8f4ed..90db64560a9f7 100644 --- a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py @@ -26,6 +26,7 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType + from tests_common.test_utils.api_connexion_utils import create_user, delete_user from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags diff --git a/tests/api_connexion/endpoints/test_dag_warning_endpoint.py b/tests/api_connexion/endpoints/test_dag_warning_endpoint.py index f3521ed883c73..b2d74f542008c 100644 --- a/tests/api_connexion/endpoints/test_dag_warning_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_warning_endpoint.py @@ -23,6 +23,7 @@ from airflow.models.dag import DagModel from airflow.models.dagwarning import DagWarning from airflow.utils.session import create_session + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags diff --git a/tests/api_connexion/endpoints/test_event_log_endpoint.py b/tests/api_connexion/endpoints/test_event_log_endpoint.py index bcd668c93cce6..00b5f7e668778 100644 --- a/tests/api_connexion/endpoints/test_event_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_event_log_endpoint.py @@ -21,6 +21,7 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import Log from airflow.utils import timezone + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_logs diff --git a/tests/api_connexion/endpoints/test_extra_link_endpoint.py b/tests/api_connexion/endpoints/test_extra_link_endpoint.py index 30b8234cd746a..800006391207b 100644 --- a/tests/api_connexion/endpoints/test_extra_link_endpoint.py +++ b/tests/api_connexion/endpoints/test_extra_link_endpoint.py @@ -30,6 +30,7 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType + from tests_common.test_utils.api_connexion_utils import create_user, delete_user from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink from tests_common.test_utils.db import clear_db_runs, clear_db_xcom diff --git a/tests/api_connexion/endpoints/test_import_error_endpoint.py b/tests/api_connexion/endpoints/test_import_error_endpoint.py index 0e2ea9f97200c..38b3f3fee4fb2 100644 --- a/tests/api_connexion/endpoints/test_import_error_endpoint.py +++ b/tests/api_connexion/endpoints/test_import_error_endpoint.py @@ -23,6 +23,7 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.utils import timezone from airflow.utils.session import provide_session + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.compat import ParseImportError from tests_common.test_utils.config import conf_vars diff --git a/tests/api_connexion/endpoints/test_log_endpoint.py b/tests/api_connexion/endpoints/test_log_endpoint.py index ef542a74d7a8f..248b8a46872bf 100644 --- a/tests/api_connexion/endpoints/test_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_log_endpoint.py @@ -32,6 +32,7 @@ from airflow.operators.empty import EmptyOperator from airflow.utils import timezone from airflow.utils.types import DagRunType + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.db import clear_db_runs diff --git a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py index 80fdd72726f17..237ef5910c782 100644 --- a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py @@ -32,6 +32,7 @@ from airflow.utils.session import provide_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields from tests_common.test_utils.mock_operators import MockOperator diff --git a/tests/api_connexion/endpoints/test_plugin_endpoint.py b/tests/api_connexion/endpoints/test_plugin_endpoint.py index b04ec7b7d5823..924de84dc0d48 100644 --- a/tests/api_connexion/endpoints/test_plugin_endpoint.py +++ b/tests/api_connexion/endpoints/test_plugin_endpoint.py @@ -28,6 +28,7 @@ from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.timetables.base import Timetable from airflow.utils.module_loading import qualname + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.compat import BaseOperatorLink from tests_common.test_utils.config import conf_vars diff --git a/tests/api_connexion/endpoints/test_pool_endpoint.py b/tests/api_connexion/endpoints/test_pool_endpoint.py index d19dbf836d025..dfe9ded929802 100644 --- a/tests/api_connexion/endpoints/test_pool_endpoint.py +++ b/tests/api_connexion/endpoints/test_pool_endpoint.py @@ -21,6 +21,7 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models.pool import Pool from airflow.utils.session import provide_session + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_pools diff --git a/tests/api_connexion/endpoints/test_provider_endpoint.py b/tests/api_connexion/endpoints/test_provider_endpoint.py index c526aabe8c005..e2d00d1978234 100644 --- a/tests/api_connexion/endpoints/test_provider_endpoint.py +++ b/tests/api_connexion/endpoints/test_provider_endpoint.py @@ -21,6 +21,7 @@ import pytest from airflow.providers_manager import ProviderInfo + from tests_common.test_utils.api_connexion_utils import create_user, delete_user pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/endpoints/test_task_endpoint.py b/tests/api_connexion/endpoints/test_task_endpoint.py index 567394208510b..b558f8dbf161e 100644 --- a/tests/api_connexion/endpoints/test_task_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_endpoint.py @@ -27,6 +27,7 @@ from airflow.models.expandinput import EXPAND_INPUT_EMPTY from airflow.models.serialized_dag import SerializedDagModel from airflow.operators.empty import EmptyOperator + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags diff --git a/tests/api_connexion/endpoints/test_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_task_instance_endpoint.py index 5746740205570..1051aa21e0cc9 100644 --- a/tests/api_connexion/endpoints/test_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_instance_endpoint.py @@ -35,6 +35,7 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields from tests_common.test_utils.www import _check_last_log diff --git a/tests/api_connexion/endpoints/test_variable_endpoint.py b/tests/api_connexion/endpoints/test_variable_endpoint.py index 640251ebc14f3..c3c1c91cb8ed4 100644 --- a/tests/api_connexion/endpoints/test_variable_endpoint.py +++ b/tests/api_connexion/endpoints/test_variable_endpoint.py @@ -22,6 +22,7 @@ from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import Variable + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_variables diff --git a/tests/api_connexion/endpoints/test_xcom_endpoint.py b/tests/api_connexion/endpoints/test_xcom_endpoint.py index 12d56ae9894ea..6f12f77e08d4b 100644 --- a/tests/api_connexion/endpoints/test_xcom_endpoint.py +++ b/tests/api_connexion/endpoints/test_xcom_endpoint.py @@ -30,6 +30,7 @@ from airflow.utils.session import create_session from airflow.utils.timezone import utcnow from airflow.utils.types import DagRunType + from tests_common.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom diff --git a/tests/api_connexion/schemas/test_connection_schema.py b/tests/api_connexion/schemas/test_connection_schema.py index e52b573875138..782339c6afe43 100644 --- a/tests/api_connexion/schemas/test_connection_schema.py +++ b/tests/api_connexion/schemas/test_connection_schema.py @@ -30,6 +30,7 @@ ) from airflow.models import Connection from airflow.utils.session import create_session, provide_session + from tests_common.test_utils.db import clear_db_connections pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_dag_run_schema.py b/tests/api_connexion/schemas/test_dag_run_schema.py index 929a8af451112..2c1ec142d5087 100644 --- a/tests/api_connexion/schemas/test_dag_run_schema.py +++ b/tests/api_connexion/schemas/test_dag_run_schema.py @@ -29,6 +29,7 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.db import clear_db_runs diff --git a/tests/api_connexion/schemas/test_dataset_schema.py b/tests/api_connexion/schemas/test_dataset_schema.py index 669b87f6cf636..1f5140b076b10 100644 --- a/tests/api_connexion/schemas/test_dataset_schema.py +++ b/tests/api_connexion/schemas/test_dataset_schema.py @@ -30,6 +30,7 @@ from airflow.assets import Asset from airflow.models.asset import AssetAliasModel, AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator + from tests_common.test_utils.db import clear_db_assets, clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_error_schema.py b/tests/api_connexion/schemas/test_error_schema.py index 5f31dfe2f1460..61fc69ff5181a 100644 --- a/tests/api_connexion/schemas/test_error_schema.py +++ b/tests/api_connexion/schemas/test_error_schema.py @@ -25,6 +25,7 @@ ) from airflow.utils import timezone from airflow.utils.session import provide_session + from tests_common.test_utils.compat import ParseImportError from tests_common.test_utils.db import clear_db_import_errors diff --git a/tests/api_connexion/schemas/test_plugin_schema.py b/tests/api_connexion/schemas/test_plugin_schema.py index df550a8301bdd..0c7141e3493f8 100644 --- a/tests/api_connexion/schemas/test_plugin_schema.py +++ b/tests/api_connexion/schemas/test_plugin_schema.py @@ -28,6 +28,7 @@ ) from airflow.hooks.base import BaseHook from airflow.plugins_manager import AirflowPlugin + from tests_common.test_utils.compat import BaseOperatorLink diff --git a/tests/api_connexion/schemas/test_pool_schemas.py b/tests/api_connexion/schemas/test_pool_schemas.py index 3e5f3d7f02904..094973ea85d0c 100644 --- a/tests/api_connexion/schemas/test_pool_schemas.py +++ b/tests/api_connexion/schemas/test_pool_schemas.py @@ -21,6 +21,7 @@ from airflow.api_connexion.schemas.pool_schema import PoolCollection, pool_collection_schema, pool_schema from airflow.models.pool import Pool from airflow.utils.session import provide_session + from tests_common.test_utils.db import clear_db_pools pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/schemas/test_xcom_schema.py b/tests/api_connexion/schemas/test_xcom_schema.py index 366d707878b9c..8714b3430fac8 100644 --- a/tests/api_connexion/schemas/test_xcom_schema.py +++ b/tests/api_connexion/schemas/test_xcom_schema.py @@ -30,6 +30,7 @@ from airflow.models import DagRun, XCom from airflow.utils.dates import parse_execution_date from airflow.utils.session import create_session + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/api_connexion/test_auth.py b/tests/api_connexion/test_auth.py index f3c35bc5fb372..fb85b28bfb1e0 100644 --- a/tests/api_connexion/test_auth.py +++ b/tests/api_connexion/test_auth.py @@ -22,6 +22,7 @@ from airflow.auth.managers.simple.simple_auth_manager import SimpleAuthManager from airflow.auth.managers.simple.user import SimpleAuthManagerUser + from tests_common.test_utils.api_connexion_utils import assert_401 from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_pools diff --git a/tests/api_connexion/test_parameters.py b/tests/api_connexion/test_parameters.py index 26fdcfa7aeca0..e64a32c4e65f1 100644 --- a/tests/api_connexion/test_parameters.py +++ b/tests/api_connexion/test_parameters.py @@ -29,6 +29,7 @@ validate_istimezone, ) from airflow.utils import timezone + from tests_common.test_utils.config import conf_vars diff --git a/tests/api_fastapi/core_api/routes/public/test_connections.py b/tests/api_fastapi/core_api/routes/public/test_connections.py index a80f2467b9b72..8c2e3b6d58f7f 100644 --- a/tests/api_fastapi/core_api/routes/public/test_connections.py +++ b/tests/api_fastapi/core_api/routes/public/test_connections.py @@ -20,6 +20,7 @@ from airflow.models import Connection from airflow.utils.session import provide_session + from tests_common.test_utils.db import clear_db_connections pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/core_api/routes/public/test_dag_run.py b/tests/api_fastapi/core_api/routes/public/test_dag_run.py index 3c60b307bbbd2..3c665d5e18daf 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dag_run.py +++ b/tests/api_fastapi/core_api/routes/public/test_dag_run.py @@ -25,6 +25,7 @@ from airflow.utils.session import provide_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunTriggeredByType, DagRunType + from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/core_api/routes/public/test_dags.py b/tests/api_fastapi/core_api/routes/public/test_dags.py index 1055000f42562..edc350c27b84a 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dags.py +++ b/tests/api_fastapi/core_api/routes/public/test_dags.py @@ -27,6 +27,7 @@ from airflow.utils.session import provide_session from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType + from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/core_api/routes/public/test_monitor.py b/tests/api_fastapi/core_api/routes/public/test_monitor.py index e37047dc906c6..f4cdfb7a5511a 100644 --- a/tests/api_fastapi/core_api/routes/public/test_monitor.py +++ b/tests/api_fastapi/core_api/routes/public/test_monitor.py @@ -26,6 +26,7 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import State + from tests_common.test_utils.db import clear_db_jobs HEALTHY = "healthy" diff --git a/tests/api_fastapi/core_api/routes/public/test_variables.py b/tests/api_fastapi/core_api/routes/public/test_variables.py index 21f86c428b397..0a52d1971075b 100644 --- a/tests/api_fastapi/core_api/routes/public/test_variables.py +++ b/tests/api_fastapi/core_api/routes/public/test_variables.py @@ -20,6 +20,7 @@ from airflow.models.variable import Variable from airflow.utils.session import provide_session + from tests_common.test_utils.db import clear_db_variables pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/core_api/routes/ui/test_assets.py b/tests/api_fastapi/core_api/routes/ui/test_assets.py index d06631ec88b59..7662ff5a02fa8 100644 --- a/tests/api_fastapi/core_api/routes/ui/test_assets.py +++ b/tests/api_fastapi/core_api/routes/ui/test_assets.py @@ -20,6 +20,7 @@ from airflow.assets import Asset from airflow.operators.empty import EmptyOperator + from tests_common.test_utils.db import initial_db_init pytestmark = pytest.mark.db_test diff --git a/tests/api_fastapi/core_api/routes/ui/test_dashboard.py b/tests/api_fastapi/core_api/routes/ui/test_dashboard.py index f4a061f1dd0ff..8ad9da12ac8bf 100644 --- a/tests/api_fastapi/core_api/routes/ui/test_dashboard.py +++ b/tests/api_fastapi/core_api/routes/ui/test_dashboard.py @@ -26,6 +26,7 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType + from tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/api_internal/endpoints/test_rpc_api_endpoint.py b/tests/api_internal/endpoints/test_rpc_api_endpoint.py index 72ce2afe4c19f..5813b28790dbb 100644 --- a/tests/api_internal/endpoints/test_rpc_api_endpoint.py +++ b/tests/api_internal/endpoints/test_rpc_api_endpoint.py @@ -34,6 +34,7 @@ from airflow.utils.jwt_signer import JWTSigner from airflow.utils.state import State from airflow.www import app + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules diff --git a/tests/api_internal/test_internal_api_call.py b/tests/api_internal/test_internal_api_call.py index ff9c01e3b554d..fe817afbfc7c5 100644 --- a/tests/api_internal/test_internal_api_call.py +++ b/tests/api_internal/test_internal_api_call.py @@ -35,6 +35,7 @@ from airflow.serialization.serialized_objects import BaseSerialization from airflow.settings import _ENABLE_AIP_44 from airflow.utils.state import State + from tests_common.test_utils.config import conf_vars if TYPE_CHECKING: diff --git a/tests/auth/managers/simple/views/test_auth.py b/tests/auth/managers/simple/views/test_auth.py index 49f6237fc7e44..cd4628c2d02b6 100644 --- a/tests/auth/managers/simple/views/test_auth.py +++ b/tests/auth/managers/simple/views/test_auth.py @@ -23,6 +23,7 @@ from airflow.auth.managers.simple.simple_auth_manager import SimpleAuthManager from airflow.www import app as application + from tests_common.test_utils.config import conf_vars diff --git a/tests/cli/commands/test_backfill_command.py b/tests/cli/commands/test_backfill_command.py index e798436f24445..eeb3234c520e1 100644 --- a/tests/cli/commands/test_backfill_command.py +++ b/tests/cli/commands/test_backfill_command.py @@ -28,6 +28,7 @@ from airflow.cli import cli_parser from airflow.models import DagBag from airflow.utils import timezone + from tests_common.test_utils.db import clear_db_backfills, clear_db_dags, clear_db_runs DEFAULT_DATE = timezone.make_aware(datetime(2015, 1, 1), timezone=timezone.utc) diff --git a/tests/cli/commands/test_celery_command.py b/tests/cli/commands/test_celery_command.py index 7e20a9c933f94..ae2b71a590949 100644 --- a/tests/cli/commands/test_celery_command.py +++ b/tests/cli/commands/test_celery_command.py @@ -30,6 +30,7 @@ from airflow.cli.commands import celery_command from airflow.configuration import conf from airflow.executors import executor_loader + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_config_command.py b/tests/cli/commands/test_config_command.py index 3202990664285..4ca30b9e74f1c 100644 --- a/tests/cli/commands/test_config_command.py +++ b/tests/cli/commands/test_config_command.py @@ -22,6 +22,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import config_command + from tests_common.test_utils.config import conf_vars STATSD_CONFIG_BEGIN_WITH = "# `StatsD `" diff --git a/tests/cli/commands/test_connection_command.py b/tests/cli/commands/test_connection_command.py index a04775433f468..5551d6861e6dc 100644 --- a/tests/cli/commands/test_connection_command.py +++ b/tests/cli/commands/test_connection_command.py @@ -33,6 +33,7 @@ from airflow.models import Connection from airflow.utils.db import merge_conn from airflow.utils.session import create_session + from tests_common.test_utils.db import clear_db_connections pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py index fe48ec5f357cb..7acc3a85b9345 100644 --- a/tests/cli/commands/test_dag_command.py +++ b/tests/cli/commands/test_dag_command.py @@ -46,10 +46,10 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from tests.models import TEST_DAGS_FOLDER +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.db import clear_db_dags, clear_db_runs DEFAULT_DATE = timezone.make_aware(datetime(2015, 1, 1), timezone=timezone.utc) if pendulum.__version__.startswith("3"): diff --git a/tests/cli/commands/test_dag_processor_command.py b/tests/cli/commands/test_dag_processor_command.py index 71a30dcd7d89b..157379123e53a 100644 --- a/tests/cli/commands/test_dag_processor_command.py +++ b/tests/cli/commands/test_dag_processor_command.py @@ -24,6 +24,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import dag_processor_command from airflow.configuration import conf + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_info_command.py b/tests/cli/commands/test_info_command.py index ca5db5819c04b..e7573631f10b9 100644 --- a/tests/cli/commands/test_info_command.py +++ b/tests/cli/commands/test_info_command.py @@ -32,6 +32,7 @@ from airflow.config_templates import airflow_local_settings from airflow.logging_config import configure_logging from airflow.version import version as airflow_version + from tests_common.test_utils.config import conf_vars diff --git a/tests/cli/commands/test_internal_api_command.py b/tests/cli/commands/test_internal_api_command.py index f3dd664d1a273..443fa2bb49782 100644 --- a/tests/cli/commands/test_internal_api_command.py +++ b/tests/cli/commands/test_internal_api_command.py @@ -31,9 +31,9 @@ from airflow.cli.commands import internal_api_command from airflow.cli.commands.internal_api_command import GunicornMonitor from airflow.settings import _ENABLE_AIP_44 -from tests_common.test_utils.config import conf_vars from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass +from tests_common.test_utils.config import conf_vars console = Console(width=400, color_system="standard") diff --git a/tests/cli/commands/test_jobs_command.py b/tests/cli/commands/test_jobs_command.py index 1c52aa71334e4..64d0a39607b13 100644 --- a/tests/cli/commands/test_jobs_command.py +++ b/tests/cli/commands/test_jobs_command.py @@ -27,6 +27,7 @@ from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.utils.session import create_session from airflow.utils.state import JobState, State + from tests_common.test_utils.db import clear_db_jobs diff --git a/tests/cli/commands/test_kerberos_command.py b/tests/cli/commands/test_kerberos_command.py index 5d2f2e513bcc5..eaffe65f3a2d6 100644 --- a/tests/cli/commands/test_kerberos_command.py +++ b/tests/cli/commands/test_kerberos_command.py @@ -23,6 +23,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import kerberos_command from airflow.security.kerberos import KerberosMode + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_kubernetes_command.py b/tests/cli/commands/test_kubernetes_command.py index dbde7a89664ed..9813b00fb0869 100644 --- a/tests/cli/commands/test_kubernetes_command.py +++ b/tests/cli/commands/test_kubernetes_command.py @@ -28,6 +28,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import kubernetes_command from airflow.executors import executor_loader + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_plugins_command.py b/tests/cli/commands/test_plugins_command.py index 00d6cee5a0b5d..d07641ec841d1 100644 --- a/tests/cli/commands/test_plugins_command.py +++ b/tests/cli/commands/test_plugins_command.py @@ -28,9 +28,9 @@ from airflow.hooks.base import BaseHook from airflow.listeners.listener import get_listener_manager from airflow.plugins_manager import AirflowPlugin -from tests_common.test_utils.mock_plugins import mock_plugin_manager from tests.plugins.test_plugin import AirflowTestPlugin as ComplexAirflowPlugin +from tests_common.test_utils.mock_plugins import mock_plugin_manager pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_rotate_fernet_key_command.py b/tests/cli/commands/test_rotate_fernet_key_command.py index 473c911d9a10c..8529ad0040198 100644 --- a/tests/cli/commands/test_rotate_fernet_key_command.py +++ b/tests/cli/commands/test_rotate_fernet_key_command.py @@ -26,6 +26,7 @@ from airflow.hooks.base import BaseHook from airflow.models import Connection, Variable from airflow.utils.session import provide_session + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_connections, clear_db_variables diff --git a/tests/cli/commands/test_scheduler_command.py b/tests/cli/commands/test_scheduler_command.py index a6adb514f203b..4fc795add90c8 100644 --- a/tests/cli/commands/test_scheduler_command.py +++ b/tests/cli/commands/test_scheduler_command.py @@ -29,6 +29,7 @@ from airflow.executors import executor_loader from airflow.utils.scheduler_health import HealthServer, serve_health_check from airflow.utils.serve_logs import serve_logs + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index 823172cffc555..67204a1ad7b70 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -51,6 +51,7 @@ from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_pools, clear_db_runs diff --git a/tests/cli/commands/test_variable_command.py b/tests/cli/commands/test_variable_command.py index 630e7d321d3e0..e37b0ab672140 100644 --- a/tests/cli/commands/test_variable_command.py +++ b/tests/cli/commands/test_variable_command.py @@ -29,6 +29,7 @@ from airflow.cli.commands import variable_command from airflow.models import Variable from airflow.utils.session import create_session + from tests_common.test_utils.db import clear_db_variables pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/cli/commands/test_webserver_command.py b/tests/cli/commands/test_webserver_command.py index a2cba19c9a216..2eaa3950010e2 100644 --- a/tests/cli/commands/test_webserver_command.py +++ b/tests/cli/commands/test_webserver_command.py @@ -30,9 +30,9 @@ from airflow.cli import cli_parser from airflow.cli.commands import webserver_command from airflow.cli.commands.webserver_command import GunicornMonitor -from tests_common.test_utils.config import conf_vars from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass +from tests_common.test_utils.config import conf_vars console = Console(width=400, color_system="standard") diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 83aeecdbb1ffb..17f891c94f88b 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -25,6 +25,7 @@ from airflow.models.dagbag import DagBag from airflow.providers.celery.executors import celery_executor, celery_kubernetes_executor from airflow.providers.cncf.kubernetes.executors import kubernetes_executor, local_kubernetes_executor + from tests_common.test_utils.config import conf_vars # Create custom executors here because conftest is imported first diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index f777908c23d19..0336dcb1eade3 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -43,6 +43,7 @@ from airflow.executors.local_executor import LocalExecutor from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 1f376c955f503..20109c6cd29b7 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -42,8 +42,6 @@ write_default_airflow_configuration_if_needed, ) from airflow.providers_manager import ProvidersManager -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.reset_warning_registry import reset_warning_registry from tests.utils.test_config import ( remove_all_configurations, @@ -51,6 +49,8 @@ set_sensitive_config_values, use_config, ) +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.reset_warning_registry import reset_warning_registry HOME_DIR = os.path.expanduser("~") diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 2186d04f195b2..db6271347422a 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -32,6 +32,7 @@ from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType + from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/core/test_example_dags_system.py b/tests/core/test_example_dags_system.py index a0d2a60d85299..dfa6b81027b2b 100644 --- a/tests/core/test_example_dags_system.py +++ b/tests/core/test_example_dags_system.py @@ -28,6 +28,7 @@ from airflow.utils.module_loading import import_string from airflow.utils.state import DagRunState from airflow.utils.trigger_rule import TriggerRule + from tests_common.test_utils.system_tests import get_test_run from tests_common.test_utils.system_tests_class import SystemTest diff --git a/tests/core/test_impersonation_tests.py b/tests/core/test_impersonation_tests.py index 0683b619d60f4..c7f06533b8ba7 100644 --- a/tests/core/test_impersonation_tests.py +++ b/tests/core/test_impersonation_tests.py @@ -32,6 +32,7 @@ from airflow.utils.db import add_default_pool_if_not_exists from airflow.utils.state import State from airflow.utils.timezone import datetime + from tests_common.test_utils import db # The entire module into the quarantined mark, this might have unpredictable side effects to other tests diff --git a/tests/core/test_logging_config.py b/tests/core/test_logging_config.py index aeb6d36ced04f..6464e93bdf6bf 100644 --- a/tests/core/test_logging_config.py +++ b/tests/core/test_logging_config.py @@ -29,6 +29,7 @@ import pytest from airflow.configuration import conf + from tests_common.test_utils.config import conf_vars SETTINGS_FILE_VALID = """ diff --git a/tests/core/test_otel_tracer.py b/tests/core/test_otel_tracer.py index 36ebb9f09ce07..666e78e2dc84f 100644 --- a/tests/core/test_otel_tracer.py +++ b/tests/core/test_otel_tracer.py @@ -26,6 +26,7 @@ from airflow.traces import TRACEPARENT, TRACESTATE, otel_tracer, utils from airflow.traces.tracer import Trace + from tests_common.test_utils.config import env_vars diff --git a/tests/core/test_sentry.py b/tests/core/test_sentry.py index 972e45dbba247..a8cb04577b4e1 100644 --- a/tests/core/test_sentry.py +++ b/tests/core/test_sentry.py @@ -30,6 +30,7 @@ from airflow.utils import timezone from airflow.utils.module_loading import import_string from airflow.utils.state import State + from tests_common.test_utils.config import conf_vars EXECUTION_DATE = timezone.utcnow() diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index 27b40ad7d1fae..00f0d14b6d519 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -33,6 +33,7 @@ from airflow.exceptions import AirflowClusterPolicyViolation, AirflowConfigException from airflow.settings import _ENABLE_AIP_44, TracebackSession, is_usage_data_collection_enabled from airflow.utils.session import create_session + from tests_common.test_utils.config import conf_vars SETTINGS_FILE_POLICY = """ diff --git a/tests/core/test_sqlalchemy_config.py b/tests/core/test_sqlalchemy_config.py index 99fb90d3ba2e3..1ce879683400b 100644 --- a/tests/core/test_sqlalchemy_config.py +++ b/tests/core/test_sqlalchemy_config.py @@ -25,6 +25,7 @@ from airflow import settings from airflow.api_internal.internal_api_call import InternalApiConfig from airflow.exceptions import AirflowConfigException + from tests_common.test_utils.config import conf_vars SQL_ALCHEMY_CONNECT_ARGS = {"test": 43503, "dict": {"is": 1, "supported": "too"}} diff --git a/tests/core/test_stats.py b/tests/core/test_stats.py index e2969204d1d32..dc6a6737f13e5 100644 --- a/tests/core/test_stats.py +++ b/tests/core/test_stats.py @@ -36,6 +36,7 @@ PatternAllowListValidator, PatternBlockListValidator, ) + from tests_common.test_utils.config import conf_vars diff --git a/tests/dag_processing/test_job_runner.py b/tests/dag_processing/test_job_runner.py index e16ec2d1dddcc..891223e2cd67f 100644 --- a/tests/dag_processing/test_job_runner.py +++ b/tests/dag_processing/test_job_runner.py @@ -58,6 +58,9 @@ from airflow.utils import timezone from airflow.utils.net import get_hostname from airflow.utils.session import create_session + +from tests.core.test_logging_config import SETTINGS_FILE_VALID, settings_context +from tests.models import TEST_DAGS_FOLDER from tests_common.test_utils.compat import ParseImportError from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import ( @@ -68,9 +71,6 @@ clear_db_serialized_dags, ) -from tests.core.test_logging_config import SETTINGS_FILE_VALID, settings_context -from tests.models import TEST_DAGS_FOLDER - pytestmark = pytest.mark.db_test logger = logging.getLogger(__name__) diff --git a/tests/dag_processing/test_processor.py b/tests/dag_processing/test_processor.py index 40412a131b40b..439c1123f9954 100644 --- a/tests/dag_processing/test_processor.py +++ b/tests/dag_processing/test_processor.py @@ -39,6 +39,7 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.types import DagRunType + from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError from tests_common.test_utils.config import conf_vars, env_vars diff --git a/tests/dags/test_miscellaneous.py b/tests/dags/test_miscellaneous.py index e54ae1b458491..5bc216bd6dee0 100644 --- a/tests/dags/test_miscellaneous.py +++ b/tests/dags/test_miscellaneous.py @@ -23,6 +23,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator + from tests_common.test_utils.compat import BashOperator args = { diff --git a/tests/dags/test_sensor.py b/tests/dags/test_sensor.py index 5dc8fbe020901..a338b83d56c06 100644 --- a/tests/dags/test_sensor.py +++ b/tests/dags/test_sensor.py @@ -21,6 +21,7 @@ from airflow.decorators import task from airflow.models.dag import DAG from airflow.utils import timezone + from tests_common.test_utils.compat import DateTimeSensor with DAG( diff --git a/tests/decorators/test_bash.py b/tests/decorators/test_bash.py index dd2ce6223c53c..59613b45e9233 100644 --- a/tests/decorators/test_bash.py +++ b/tests/decorators/test_bash.py @@ -29,6 +29,7 @@ from airflow.models.renderedtifields import RenderedTaskInstanceFields from airflow.utils import timezone from airflow.utils.types import NOTSET + from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields DEFAULT_DATE = timezone.datetime(2023, 1, 1) diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py index 4307b29afb48b..0e030dcbdcb71 100644 --- a/tests/decorators/test_python.py +++ b/tests/decorators/test_python.py @@ -39,9 +39,9 @@ from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from airflow.utils.xcom import XCOM_RETURN_KEY -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests.operators.test_python import BasePythonTest +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/executors/test_executor_loader.py b/tests/executors/test_executor_loader.py index d6e783282d0ef..40a336bc580c3 100644 --- a/tests/executors/test_executor_loader.py +++ b/tests/executors/test_executor_loader.py @@ -29,6 +29,7 @@ from airflow.executors.local_executor import LocalExecutor from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/tests/integration/cli/commands/test_celery_command.py b/tests/integration/cli/commands/test_celery_command.py index bb6c097578a6c..ce38535ce46ae 100644 --- a/tests/integration/cli/commands/test_celery_command.py +++ b/tests/integration/cli/commands/test_celery_command.py @@ -25,6 +25,7 @@ from airflow.cli import cli_parser from airflow.cli.commands import celery_command from airflow.executors import executor_loader + from tests_common.test_utils.config import conf_vars diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 4dd5e2bdc94d9..7bbe3bd29b940 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -45,6 +45,7 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.state import State, TaskInstanceState + from tests_common.test_utils import db logger = logging.getLogger(__name__) diff --git a/tests/integration/security/test_kerberos.py b/tests/integration/security/test_kerberos.py index 7267b69442d78..ad21a9ddd0dcc 100644 --- a/tests/integration/security/test_kerberos.py +++ b/tests/integration/security/test_kerberos.py @@ -26,6 +26,7 @@ from airflow.security import kerberos from airflow.security.kerberos import renew_from_kt + from tests_common.test_utils.config import conf_vars diff --git a/tests/jobs/test_base_job.py b/tests/jobs/test_base_job.py index 058539868f1c0..8da90df73be14 100644 --- a/tests/jobs/test_base_job.py +++ b/tests/jobs/test_base_job.py @@ -32,10 +32,10 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State -from tests_common.test_utils.config import conf_vars from tests.listeners import lifecycle_listener from tests.utils.test_helpers import MockJobRunner, SchedulerJobRunner, TriggererJobRunner +from tests_common.test_utils.config import conf_vars if TYPE_CHECKING: from airflow.serialization.pydantic.job import JobPydantic diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py index 7d99c593511dd..574215e8c3434 100644 --- a/tests/jobs/test_local_task_job.py +++ b/tests/jobs/test_local_task_job.py @@ -52,6 +52,7 @@ from airflow.utils.state import State from airflow.utils.timeout import timeout from airflow.utils.types import DagRunType + from tests_common.test_utils import db from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 13ccfc74a0ffb..b2b7f5a5ed8f5 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -69,6 +69,11 @@ from airflow.utils.session import create_session, provide_session from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState from airflow.utils.types import DagRunType + +from tests.listeners import dag_listener +from tests.listeners.test_listeners import get_listener_manager +from tests.models import TEST_DAGS_FOLDER +from tests.utils.test_timezone import UTC from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars, env_vars @@ -86,11 +91,6 @@ from tests_common.test_utils.mock_executor import MockExecutor from tests_common.test_utils.mock_operators import CustomOperator -from tests.listeners import dag_listener -from tests.listeners.test_listeners import get_listener_manager -from tests.models import TEST_DAGS_FOLDER -from tests.utils.test_timezone import UTC - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/jobs/test_triggerer_job.py b/tests/jobs/test_triggerer_job.py index 3987aa86438fa..cc457687b0900 100644 --- a/tests/jobs/test_triggerer_job.py +++ b/tests/jobs/test_triggerer_job.py @@ -46,9 +46,9 @@ from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunType -from tests_common.test_utils.db import clear_db_dags, clear_db_runs from tests.core.test_logging_config import reset_logging +from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/jobs/test_triggerer_job_logging.py b/tests/jobs/test_triggerer_job_logging.py index fcca727641512..335f9e6b81cc2 100644 --- a/tests/jobs/test_triggerer_job_logging.py +++ b/tests/jobs/test_triggerer_job_logging.py @@ -30,6 +30,7 @@ from airflow.utils.log.file_task_handler import FileTaskHandler from airflow.utils.log.logging_mixin import RedirectStdHandler from airflow.utils.log.trigger_handler import DropTriggerLogsFilter, TriggererHandlerWrapper + from tests_common.test_utils.config import conf_vars diff --git a/tests/lineage/test_hook.py b/tests/lineage/test_hook.py index 6287b39fc8fa1..cfb446b8adee4 100644 --- a/tests/lineage/test_hook.py +++ b/tests/lineage/test_hook.py @@ -33,6 +33,7 @@ NoOpCollector, get_hook_lineage_collector, ) + from tests_common.test_utils.mock_plugins import mock_plugin_manager diff --git a/tests/lineage/test_lineage.py b/tests/lineage/test_lineage.py index bd5b2ebe0fc03..6a3e11b68b76e 100644 --- a/tests/lineage/test_lineage.py +++ b/tests/lineage/test_lineage.py @@ -30,6 +30,7 @@ from airflow.utils import timezone from airflow.utils.context import Context from airflow.utils.types import DagRunType + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/listeners/class_listener.py b/tests/listeners/class_listener.py index 5ae71733bf3bd..7dcda72c5e2b2 100644 --- a/tests/listeners/class_listener.py +++ b/tests/listeners/class_listener.py @@ -19,6 +19,7 @@ from airflow.listeners import hookimpl from airflow.utils.state import DagRunState, TaskInstanceState + from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS if AIRFLOW_V_2_10_PLUS: diff --git a/tests/listeners/test_dag_import_error_listener.py b/tests/listeners/test_dag_import_error_listener.py index e636de4218c53..5709ba19a8de0 100644 --- a/tests/listeners/test_dag_import_error_listener.py +++ b/tests/listeners/test_dag_import_error_listener.py @@ -31,6 +31,8 @@ from airflow.models import DagModel from airflow.models.errors import ParseImportError from airflow.utils import timezone + +from tests.listeners import dag_import_error_listener from tests_common.test_utils.config import conf_vars, env_vars from tests_common.test_utils.db import ( clear_db_dags, @@ -42,8 +44,6 @@ ) from tests_common.test_utils.mock_executor import MockExecutor -from tests.listeners import dag_import_error_listener - pytestmark = pytest.mark.db_test DEFAULT_DATE = timezone.datetime(2016, 1, 1) diff --git a/tests/models/test_backfill.py b/tests/models/test_backfill.py index fafcca478c397..50b544a6957f5 100644 --- a/tests/models/test_backfill.py +++ b/tests/models/test_backfill.py @@ -33,6 +33,7 @@ ) from airflow.operators.python import PythonOperator from airflow.utils.state import DagRunState + from tests_common.test_utils.db import ( clear_db_backfills, clear_db_dags, diff --git a/tests/models/test_base.py b/tests/models/test_base.py index 2a58dc05009f3..e55f7b0068d6a 100644 --- a/tests/models/test_base.py +++ b/tests/models/test_base.py @@ -19,6 +19,7 @@ import pytest from airflow.models.base import get_id_collation_args + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index b9edc2b56f5a4..31bb0c9dd66a7 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -49,9 +49,9 @@ from airflow.utils.template import literal from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType -from tests_common.test_utils.mock_operators import DeprecatedOperator, MockOperator from tests.models import DEFAULT_DATE +from tests_common.test_utils.mock_operators import DeprecatedOperator, MockOperator if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/tests/models/test_cleartasks.py b/tests/models/test_cleartasks.py index a58e308b01861..b018f2bf2b276 100644 --- a/tests/models/test_cleartasks.py +++ b/tests/models/test_cleartasks.py @@ -34,10 +34,10 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.types import DagRunType -from tests_common.test_utils import db -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests.models import DEFAULT_DATE +from tests_common.test_utils import db +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index e6f8042253ec0..61b4628175200 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -90,6 +90,14 @@ from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType from airflow.utils.weight_rule import WeightRule + +from tests.models import DEFAULT_DATE +from tests.plugins.priority_weight_strategy import ( + FactorPriorityWeightStrategy, + NotRegisteredPriorityWeightStrategy, + StaticTestPriorityWeightStrategy, + TestPriorityWeightStrategyPlugin, +) from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars @@ -103,14 +111,6 @@ from tests_common.test_utils.mock_plugins import mock_plugin_manager from tests_common.test_utils.timetables import cron_timetable, delta_timetable -from tests.models import DEFAULT_DATE -from tests.plugins.priority_weight_strategy import ( - FactorPriorityWeightStrategy, - NotRegisteredPriorityWeightStrategy, - StaticTestPriorityWeightStrategy, - TestPriorityWeightStrategyPlugin, -) - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py index 0973da449ef38..b35921facb7fb 100644 --- a/tests/models/test_dagbag.py +++ b/tests/models/test_dagbag.py @@ -44,12 +44,12 @@ from airflow.utils.dates import timezone as tz from airflow.utils.session import create_session from airflow.www.security_appless import ApplessAirflowSecurityManager -from tests_common.test_utils import db -from tests_common.test_utils.asserts import assert_queries_count -from tests_common.test_utils.config import conf_vars from tests import cluster_policies from tests.models import TEST_DAGS_FOLDER +from tests_common.test_utils import db +from tests_common.test_utils.asserts import assert_queries_count +from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/models/test_dagcode.py b/tests/models/test_dagcode.py index f0cec513c09d3..26b29ea8f9c02 100644 --- a/tests/models/test_dagcode.py +++ b/tests/models/test_dagcode.py @@ -30,6 +30,7 @@ # To move it to a shared module. from airflow.utils.file import open_maybe_zipped from airflow.utils.session import create_session + from tests_common.test_utils.db import clear_db_dag_code pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index 587d514cf807e..d779276b2b90a 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -46,13 +46,13 @@ from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType + +from tests.models import DEFAULT_DATE as _DEFAULT_DATE from tests_common.test_utils import db from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.mock_operators import MockOperator -from tests.models import DEFAULT_DATE as _DEFAULT_DATE - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_dagwarning.py b/tests/models/test_dagwarning.py index 95d12c005e4bb..be5fbf56e10f2 100644 --- a/tests/models/test_dagwarning.py +++ b/tests/models/test_dagwarning.py @@ -25,6 +25,7 @@ from airflow.models import DagModel from airflow.models.dagwarning import DagWarning + from tests_common.test_utils.db import clear_db_dags pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/models/test_mappedoperator.py b/tests/models/test_mappedoperator.py index 7180421a90bd6..1bb616989ce64 100644 --- a/tests/models/test_mappedoperator.py +++ b/tests/models/test_mappedoperator.py @@ -41,6 +41,8 @@ from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.trigger_rule import TriggerRule from airflow.utils.xcom import XCOM_RETURN_KEY + +from tests.models import DEFAULT_DATE from tests_common.test_utils.mapping import expand_mapped_task from tests_common.test_utils.mock_operators import ( MockOperator, @@ -48,8 +50,6 @@ NestedFields, ) -from tests.models import DEFAULT_DATE - pytestmark = pytest.mark.db_test if TYPE_CHECKING: diff --git a/tests/models/test_param.py b/tests/models/test_param.py index 7ade5bd21cebf..79fbbeaabb081 100644 --- a/tests/models/test_param.py +++ b/tests/models/test_param.py @@ -26,6 +26,7 @@ from airflow.serialization.serialized_objects import BaseSerialization from airflow.utils import timezone from airflow.utils.types import DagRunType + from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom diff --git a/tests/models/test_pool.py b/tests/models/test_pool.py index 698899be002e4..7711d828b6788 100644 --- a/tests/models/test_pool.py +++ b/tests/models/test_pool.py @@ -27,6 +27,7 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State + from tests_common.test_utils.db import ( clear_db_dags, clear_db_pools, diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index 6d160164ecff8..81421dca17116 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -37,6 +37,7 @@ from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.timezone import datetime + from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py index 2ed2d5090a2d6..cdff883760c92 100644 --- a/tests/models/test_serialized_dag.py +++ b/tests/models/test_serialized_dag.py @@ -36,6 +36,7 @@ from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 from airflow.utils.session import create_session + from tests_common.test_utils import db from tests_common.test_utils.asserts import assert_queries_count diff --git a/tests/models/test_skipmixin.py b/tests/models/test_skipmixin.py index e15f5521d1cd0..1efb3e2bb493b 100644 --- a/tests/models/test_skipmixin.py +++ b/tests/models/test_skipmixin.py @@ -31,6 +31,7 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType + from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 5753f51fcd2f8..53d8feceff1e1 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -98,14 +98,14 @@ from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.types import DagRunType from airflow.utils.xcom import XCOM_RETURN_KEY + +from tests.models import DEFAULT_DATE, TEST_DAGS_FOLDER from tests_common.test_utils import db from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_connections, clear_db_runs from tests_common.test_utils.mock_operators import MockOperator -from tests.models import DEFAULT_DATE, TEST_DAGS_FOLDER - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/models/test_timestamp.py b/tests/models/test_timestamp.py index 6bf9b9b768a60..e174629736bcf 100644 --- a/tests/models/test_timestamp.py +++ b/tests/models/test_timestamp.py @@ -25,6 +25,7 @@ from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import State + from tests_common.test_utils.db import clear_db_dags, clear_db_logs, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/models/test_trigger.py b/tests/models/test_trigger.py index 8a9c21c685bbb..065575e390041 100644 --- a/tests/models/test_trigger.py +++ b/tests/models/test_trigger.py @@ -42,6 +42,7 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.xcom import XCOM_RETURN_KEY + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/models/test_variable.py b/tests/models/test_variable.py index 29a5c94d9ae0d..79a7730bf3fde 100644 --- a/tests/models/test_variable.py +++ b/tests/models/test_variable.py @@ -27,6 +27,7 @@ from airflow.models import Variable, crypto, variable from airflow.secrets.cache import SecretCache from airflow.secrets.metastore import MetastoreBackend + from tests_common.test_utils import db from tests_common.test_utils.config import conf_vars diff --git a/tests/models/test_xcom.py b/tests/models/test_xcom.py index ee47ac79fab1e..87a56c430e6f7 100644 --- a/tests/models/test_xcom.py +++ b/tests/models/test_xcom.py @@ -35,6 +35,7 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.xcom import XCOM_RETURN_KEY + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import is_db_isolation_mode diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index 11613bcfec6b9..f38414ef30012 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -22,6 +22,7 @@ from airflow.operators.python import PythonOperator from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.types import NOTSET + from tests_common.test_utils.db import clear_db_dags, clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/operators/test_branch_operator.py b/tests/operators/test_branch_operator.py index 3442ff0a7e5e1..9df1b897b4270 100644 --- a/tests/operators/test_branch_operator.py +++ b/tests/operators/test_branch_operator.py @@ -28,6 +28,7 @@ from airflow.utils.state import State from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: diff --git a/tests/operators/test_email.py b/tests/operators/test_email.py index 8ef2634aad4d3..df772beb95fdd 100644 --- a/tests/operators/test_email.py +++ b/tests/operators/test_email.py @@ -24,6 +24,7 @@ from airflow.operators.email import EmailOperator from airflow.utils import timezone + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/operators/test_latest_only_operator.py b/tests/operators/test_latest_only_operator.py index 57bc2d27c1a3f..fbc2cd5b46d8a 100644 --- a/tests/operators/test_latest_only_operator.py +++ b/tests/operators/test_latest_only_operator.py @@ -30,6 +30,7 @@ from airflow.utils.state import State from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.db import clear_db_runs, clear_db_xcom diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index 5c91362723c1e..b759e9a600540 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -68,6 +68,7 @@ from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.trigger_rule import TriggerRule from airflow.utils.types import NOTSET, DagRunType + from tests_common.test_utils import AIRFLOW_MAIN_FOLDER from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.db import clear_db_runs diff --git a/tests/plugins/test_plugin.py b/tests/plugins/test_plugin.py index 8751d75cd20fd..ccf7ae50707c2 100644 --- a/tests/plugins/test_plugin.py +++ b/tests/plugins/test_plugin.py @@ -33,6 +33,9 @@ from airflow.task.priority_strategy import PriorityWeightStrategy from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.timetables.interval import CronDataIntervalTimetable + +from tests.listeners import empty_listener +from tests.listeners.class_listener import ClassBasedListener from tests_common.test_utils.mock_operators import ( AirflowLink, AirflowLink2, @@ -42,9 +45,6 @@ GoogleLink, ) -from tests.listeners import empty_listener -from tests.listeners.class_listener import ClassBasedListener - # Will show up under airflow.hooks.test_plugin.PluginHook class PluginHook(BaseHook): diff --git a/tests/plugins/test_plugins_manager.py b/tests/plugins/test_plugins_manager.py index 0c742dd70c67d..158e2040abf7a 100644 --- a/tests/plugins/test_plugins_manager.py +++ b/tests/plugins/test_plugins_manager.py @@ -33,6 +33,7 @@ from airflow.plugins_manager import AirflowPlugin from airflow.utils.module_loading import qualname from airflow.www import app as application + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.mock_plugins import mock_plugin_manager diff --git a/tests/secrets/test_cache.py b/tests/secrets/test_cache.py index 86ed57688f34d..a27dfde992bce 100644 --- a/tests/secrets/test_cache.py +++ b/tests/secrets/test_cache.py @@ -22,6 +22,7 @@ import pytest from airflow.secrets.cache import SecretCache + from tests_common.test_utils.config import conf_vars diff --git a/tests/security/test_kerberos.py b/tests/security/test_kerberos.py index 9f15c0296894c..1153ac61a7983 100644 --- a/tests/security/test_kerberos.py +++ b/tests/security/test_kerberos.py @@ -25,6 +25,7 @@ from airflow.security import kerberos from airflow.security.kerberos import get_kerberos_principle, renew_from_kt + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/sensors/test_base.py b/tests/sensors/test_base.py index f89cee879f9b8..9f3bd3f8fe84b 100644 --- a/tests/sensors/test_base.py +++ b/tests/sensors/test_base.py @@ -57,6 +57,7 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.timezone import datetime + from tests_common.test_utils import db pytestmark = pytest.mark.db_test diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index 2f17bfd38d133..bd4d827b3fcf9 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -51,12 +51,12 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType + +from tests.models import TEST_DAGS_FOLDER from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.db import clear_db_runs from tests_common.test_utils.mock_operators import MockOperator -from tests.models import TEST_DAGS_FOLDER - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index da6723800d54a..b54ded3a55c6a 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -80,6 +80,7 @@ from airflow.utils.operator_resources import Resources from airflow.utils.task_group import TaskGroup from airflow.utils.xcom import XCOM_RETURN_KEY + from tests_common.test_utils.compat import BaseOperatorLink from tests_common.test_utils.mock_operators import AirflowLink2, CustomOperator, GoogleLink, MockOperator from tests_common.test_utils.timetables import ( diff --git a/tests/serialization/test_pydantic_models.py b/tests/serialization/test_pydantic_models.py index 6ecf9c8d5d90d..ff17bfa9a5d5f 100644 --- a/tests/serialization/test_pydantic_models.py +++ b/tests/serialization/test_pydantic_models.py @@ -44,9 +44,9 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import AttributeRemoved, DagRunType -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/serialization/test_serde.py b/tests/serialization/test_serde.py index 829195476c3d6..e37464905d41b 100644 --- a/tests/serialization/test_serde.py +++ b/tests/serialization/test_serde.py @@ -43,6 +43,7 @@ serialize, ) from airflow.utils.module_loading import import_string, iter_namespace, qualname + from tests_common.test_utils.config import conf_vars diff --git a/tests/serialization/test_serialized_objects.py b/tests/serialization/test_serialized_objects.py index 56a31d4d38b78..1a20cc7975947 100644 --- a/tests/serialization/test_serialized_objects.py +++ b/tests/serialization/test_serialized_objects.py @@ -472,6 +472,7 @@ def test_all_pydantic_models_round_trip(): @pytest.mark.db_test def test_serialized_mapped_operator_unmap(dag_maker): from airflow.serialization.serialized_objects import SerializedDAG + from tests_common.test_utils.mock_operators import MockOperator with dag_maker(dag_id="dag") as dag: diff --git a/tests/task/task_runner/test_standard_task_runner.py b/tests/task/task_runner/test_standard_task_runner.py index 7e1c80c8f8d6f..5c1f88079c50f 100644 --- a/tests/task/task_runner/test_standard_task_runner.py +++ b/tests/task/task_runner/test_standard_task_runner.py @@ -40,11 +40,11 @@ from airflow.utils.platform import getuser from airflow.utils.state import State from airflow.utils.timeout import timeout -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_runs from tests.listeners import xcom_listener from tests.listeners.file_write_listener import FileWriteListener +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.db import clear_db_runs if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/ti_deps/deps/test_pool_slots_available_dep.py b/tests/ti_deps/deps/test_pool_slots_available_dep.py index 1cf2d1d2072fe..0933dc23a3db1 100644 --- a/tests/ti_deps/deps/test_pool_slots_available_dep.py +++ b/tests/ti_deps/deps/test_pool_slots_available_dep.py @@ -26,6 +26,7 @@ from airflow.ti_deps.deps.pool_slots_available_dep import PoolSlotsAvailableDep from airflow.utils.session import create_session from airflow.utils.state import TaskInstanceState + from tests_common.test_utils import db pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py b/tests/ti_deps/deps/test_prev_dagrun_dep.py index bb7ea047ac651..0e8b75d2fcbaf 100644 --- a/tests/ti_deps/deps/test_prev_dagrun_dep.py +++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py @@ -29,6 +29,7 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import convert_to_utc, datetime from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.db import clear_db_runs diff --git a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py index 02b291797956f..885f045ee37b4 100644 --- a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py +++ b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py @@ -30,6 +30,7 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State + from tests_common.test_utils import db pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/log/test_log_reader.py b/tests/utils/log/test_log_reader.py index 95d8471a3a35b..b760e69fd7e11 100644 --- a/tests/utils/log/test_log_reader.py +++ b/tests/utils/log/test_log_reader.py @@ -38,6 +38,7 @@ from airflow.utils.log.logging_mixin import ExternalLoggingMixin from airflow.utils.state import TaskInstanceState from airflow.utils.types import DagRunType + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs diff --git a/tests/utils/log/test_secrets_masker.py b/tests/utils/log/test_secrets_masker.py index ffe37fbfc4955..1f4642fa85b9d 100644 --- a/tests/utils/log/test_secrets_masker.py +++ b/tests/utils/log/test_secrets_masker.py @@ -38,6 +38,7 @@ should_hide_value_for_key, ) from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.enable_redact diff --git a/tests/utils/log/test_task_context_logger.py b/tests/utils/log/test_task_context_logger.py index 5d326a240f45e..7c73648535a22 100644 --- a/tests/utils/log/test_task_context_logger.py +++ b/tests/utils/log/test_task_context_logger.py @@ -24,6 +24,7 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.utils.log.task_context_logger import TaskContextLogger + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index dcfcd16421527..1d3412d361412 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -50,6 +50,7 @@ upgradedb, ) from airflow.utils.db_manager import RunDBManager + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py index ba7cc9476c933..53e242df089bf 100644 --- a/tests/utils/test_db_cleanup.py +++ b/tests/utils/test_db_cleanup.py @@ -48,6 +48,7 @@ run_cleanup, ) from airflow.utils.session import create_session + from tests_common.test_utils.db import ( clear_db_assets, clear_db_dags, diff --git a/tests/utils/test_db_manager.py b/tests/utils/test_db_manager.py index 001d89e1d4c59..b44262961f85c 100644 --- a/tests/utils/test_db_manager.py +++ b/tests/utils/test_db_manager.py @@ -25,6 +25,7 @@ from airflow.models import Base from airflow.utils.db import downgrade, initdb from airflow.utils.db_manager import BaseDBManager, RunDBManager + from tests_common.test_utils.config import conf_vars pytestmark = [pytest.mark.db_test] diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index 718d2d03b0832..d8b80c0ba3f4a 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -29,6 +29,7 @@ from airflow.utils import dot_renderer, timezone from airflow.utils.state import State from airflow.utils.task_group import TaskGroup + from tests_common.test_utils.compat import BashOperator from tests_common.test_utils.db import clear_db_dags diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index ac90c20ef6a75..625f0f4b1a5da 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -29,6 +29,7 @@ from airflow.configuration import conf from airflow.utils import email + from tests_common.test_utils.config import conf_vars EMAILS = ["test1@example.com", "test2@example.com"] diff --git a/tests/utils/test_file.py b/tests/utils/test_file.py index a7ca4bd5575f7..ed02ccaa2d4c7 100644 --- a/tests/utils/test_file.py +++ b/tests/utils/test_file.py @@ -26,9 +26,9 @@ from airflow.utils import file as file_utils from airflow.utils.file import correct_maybe_zipped, find_path_from_directory, open_maybe_zipped -from tests_common.test_utils.config import conf_vars from tests.models import TEST_DAGS_FOLDER +from tests_common.test_utils.config import conf_vars def might_contain_dag(file_path: str, zip_file: zipfile.ZipFile | None = None): diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index ded59b999f8bf..041dba2a90e4f 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -37,6 +37,7 @@ validate_key, ) from airflow.utils.types import NOTSET + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py index c1135d8fe4a18..190197e310de3 100644 --- a/tests/utils/test_log_handlers.py +++ b/tests/utils/test_log_handlers.py @@ -54,6 +54,7 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars diff --git a/tests/utils/test_net.py b/tests/utils/test_net.py index 50fdd207dec3f..46fefd4202632 100644 --- a/tests/utils/test_net.py +++ b/tests/utils/test_net.py @@ -24,6 +24,7 @@ from airflow.exceptions import AirflowConfigException from airflow.utils import net + from tests_common.test_utils.config import conf_vars diff --git a/tests/utils/test_serve_logs.py b/tests/utils/test_serve_logs.py index 0c6fb8d87591e..1e88e24ac39a1 100644 --- a/tests/utils/test_serve_logs.py +++ b/tests/utils/test_serve_logs.py @@ -28,6 +28,7 @@ from airflow.utils import timezone from airflow.utils.jwt_signer import JWTSigner from airflow.utils.serve_logs import create_app + from tests_common.test_utils.config import conf_vars if TYPE_CHECKING: diff --git a/tests/utils/test_sqlalchemy.py b/tests/utils/test_sqlalchemy.py index 49e4db42788a6..f32858c064d2c 100644 --- a/tests/utils/test_sqlalchemy.py +++ b/tests/utils/test_sqlalchemy.py @@ -42,6 +42,7 @@ ) from airflow.utils.state import State from airflow.utils.timezone import utcnow + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: diff --git a/tests/utils/test_state.py b/tests/utils/test_state.py index 12d857ee1999e..effd7d28dad42 100644 --- a/tests/utils/test_state.py +++ b/tests/utils/test_state.py @@ -25,9 +25,9 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index 6b2185bf791fc..5abedfe553866 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -38,9 +38,9 @@ from airflow.operators.python import PythonOperator from airflow.utils.dag_edges import dag_edges from airflow.utils.task_group import TASKGROUP_ARGS_EXPECTED_TYPES, TaskGroup, task_group_to_dict -from tests_common.test_utils.compat import BashOperator from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import BashOperator def make_task(name, type_="classic"): diff --git a/tests/utils/test_task_handler_with_custom_formatter.py b/tests/utils/test_task_handler_with_custom_formatter.py index 785cb34eaa013..103c3543e31e4 100644 --- a/tests/utils/test_task_handler_with_custom_formatter.py +++ b/tests/utils/test_task_handler_with_custom_formatter.py @@ -28,6 +28,7 @@ from airflow.utils.state import DagRunState from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_runs diff --git a/tests/utils/test_types.py b/tests/utils/test_types.py index d982730392076..151758163406f 100644 --- a/tests/utils/test_types.py +++ b/tests/utils/test_types.py @@ -25,9 +25,9 @@ from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.types import DagRunType -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests.models import DEFAULT_DATE +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/test_app.py b/tests/www/test_app.py index 20dd5eb05e7e0..cc9f24ca6a8e2 100644 --- a/tests/www/test_app.py +++ b/tests/www/test_app.py @@ -29,6 +29,7 @@ from airflow.exceptions import AirflowConfigException from airflow.www import app as application + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index 1ee0415b822a4..67e64d601fcfa 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -44,6 +44,7 @@ wrapped_markdown, ) from airflow.www.widgets import AirflowDateTimePickerROWidget, BS3TextAreaROWidget, BS3TextFieldROWidget + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: diff --git a/tests/www/views/conftest.py b/tests/www/views/conftest.py index e5bf5f947ac3b..3a82136b715fe 100644 --- a/tests/www/views/conftest.py +++ b/tests/www/views/conftest.py @@ -27,6 +27,7 @@ from airflow import settings from airflow.models import DagBag from airflow.www.app import create_app + from tests_common.test_utils.api_connexion_utils import delete_user from tests_common.test_utils.config import conf_vars from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules diff --git a/tests/www/views/test_anonymous_as_admin_role.py b/tests/www/views/test_anonymous_as_admin_role.py index f881a35f95594..5a90f8becc267 100644 --- a/tests/www/views/test_anonymous_as_admin_role.py +++ b/tests/www/views/test_anonymous_as_admin_role.py @@ -23,6 +23,7 @@ from airflow.models import Pool from airflow.utils.session import create_session + from tests_common.test_utils.www import check_content_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_session.py b/tests/www/views/test_session.py index 3bce5ebd9bf0d..d6f9f12454746 100644 --- a/tests/www/views/test_session.py +++ b/tests/www/views/test_session.py @@ -22,6 +22,7 @@ from airflow.exceptions import AirflowConfigException from airflow.www import app + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index 46d2ea7b194f9..fc2f0b7565747 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -42,6 +42,7 @@ get_task_stats_from_query, get_value_from_path, ) + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.mock_plugins import mock_plugin_manager @@ -329,6 +330,7 @@ def test_mark_task_instance_state(test_app): from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from airflow.www.views import Airflow + from tests_common.test_utils.db import clear_db_runs clear_db_runs() @@ -422,6 +424,7 @@ def test_mark_task_group_state(test_app): from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from airflow.www.views import Airflow + from tests_common.test_utils.db import clear_db_runs clear_db_runs() diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index 08e70b5727626..e99aeccb5b4ef 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -30,6 +30,8 @@ from airflow.utils.state import State from airflow.utils.types import DagRunType from airflow.www.views import FILTER_STATUS_COOKIE + +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user_scope from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.db import clear_db_runs from tests_common.test_utils.permissions import _resource_name @@ -39,8 +41,6 @@ client_with_login, ) -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user_scope - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_base.py b/tests/www/views/test_views_base.py index 3254441c74da8..3c1f00fcc5e98 100644 --- a/tests/www/views/test_views_base.py +++ b/tests/www/views/test_views_base.py @@ -27,6 +27,7 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.www import app as application + from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.config import conf_vars from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response diff --git a/tests/www/views/test_views_cluster_activity.py b/tests/www/views/test_views_cluster_activity.py index 64063e069db69..a8c2a175511de 100644 --- a/tests/www/views/test_views_cluster_activity.py +++ b/tests/www/views/test_views_cluster_activity.py @@ -26,6 +26,7 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType + from tests_common.test_utils.db import clear_db_runs pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_configuration.py b/tests/www/views/test_views_configuration.py index 4bd1f2fc4c3c8..20d9c7bf7ce2b 100644 --- a/tests/www/views/test_views_configuration.py +++ b/tests/www/views/test_views_configuration.py @@ -21,6 +21,7 @@ import pytest from airflow.configuration import conf + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 7b0229c315b96..a157e0aea0ca1 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -27,6 +27,7 @@ from airflow.models import Connection from airflow.utils.session import create_session from airflow.www.views import ConnectionFormWidget, ConnectionModelView + from tests_common.test_utils.www import ( _check_last_log, _check_last_log_masked_connection, diff --git a/tests/www/views/test_views_custom_user_views.py b/tests/www/views/test_views_custom_user_views.py index 3c00242308b75..e3f7f3e5cccd5 100644 --- a/tests/www/views/test_views_custom_user_views.py +++ b/tests/www/views/test_views_custom_user_views.py @@ -27,16 +27,16 @@ from airflow import settings from airflow.security import permissions from airflow.www import app as application -from tests_common.test_utils.www import ( - check_content_in_response, - check_content_not_in_response, - client_with_login, -) from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_role, ) +from tests_common.test_utils.www import ( + check_content_in_response, + check_content_not_in_response, + client_with_login, +) pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_dagrun.py b/tests/www/views/test_views_dagrun.py index 9a30e0a3511eb..df55cc952e8a2 100644 --- a/tests/www/views/test_views_dagrun.py +++ b/tests/www/views/test_views_dagrun.py @@ -24,12 +24,6 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.www.views import DagRunModelView -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.www import ( - check_content_in_response, - check_content_not_in_response, - client_with_login, -) from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, @@ -37,6 +31,12 @@ delete_user, ) from tests.www.views.test_views_tasks import _get_appbuilder_pk_string +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.www import ( + check_content_in_response, + check_content_not_in_response, + client_with_login, +) if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_dataset.py b/tests/www/views/test_views_dataset.py index bc40b9010c29c..b712a43da515e 100644 --- a/tests/www/views/test_views_dataset.py +++ b/tests/www/views/test_views_dataset.py @@ -24,6 +24,7 @@ from airflow.assets import Asset from airflow.models.asset import AssetActive, AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator + from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.db import clear_db_assets diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py index 730fc55b4548b..7e00192ab7297 100644 --- a/tests/www/views/test_views_decorators.py +++ b/tests/www/views/test_views_decorators.py @@ -23,6 +23,7 @@ from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.db import clear_db_runs, clear_db_variables from tests_common.test_utils.www import ( diff --git a/tests/www/views/test_views_extra_links.py b/tests/www/views/test_views_extra_links.py index bc06a09c8f95b..586d38f807914 100644 --- a/tests/www/views/test_views_extra_links.py +++ b/tests/www/views/test_views_extra_links.py @@ -28,6 +28,7 @@ from airflow.utils import timezone from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink from tests_common.test_utils.db import clear_db_runs from tests_common.test_utils.mock_operators import ( diff --git a/tests/www/views/test_views_grid.py b/tests/www/views/test_views_grid.py index d896520422493..edf878b9829e7 100644 --- a/tests/www/views/test_views_grid.py +++ b/tests/www/views/test_views_grid.py @@ -35,6 +35,7 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType from airflow.www.views import dag_to_grid + from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.db import clear_db_assets, clear_db_runs from tests_common.test_utils.mock_operators import MockOperator diff --git a/tests/www/views/test_views_home.py b/tests/www/views/test_views_home.py index b02099a668008..c7068f0191bc8 100644 --- a/tests/www/views/test_views_home.py +++ b/tests/www/views/test_views_home.py @@ -27,6 +27,8 @@ from airflow.utils.state import State from airflow.www.utils import UIAlert from airflow.www.views import FILTER_LASTRUN_COOKIE, FILTER_STATUS_COOKIE, FILTER_TAGS_COOKIE + +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user from tests_common.test_utils.db import clear_db_dags, clear_db_import_errors, clear_db_serialized_dags from tests_common.test_utils.permissions import _resource_name from tests_common.test_utils.www import ( @@ -35,8 +37,6 @@ client_with_login, ) -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user - pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_log.py b/tests/www/views/test_views_log.py index b0fdf6a01d10c..9c3a8d38ad2ae 100644 --- a/tests/www/views/test_views_log.py +++ b/tests/www/views/test_views_log.py @@ -40,6 +40,7 @@ from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType from airflow.www.app import create_app + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs diff --git a/tests/www/views/test_views_mount.py b/tests/www/views/test_views_mount.py index c20dc418d3a02..03115fa611047 100644 --- a/tests/www/views/test_views_mount.py +++ b/tests/www/views/test_views_mount.py @@ -22,6 +22,7 @@ import werkzeug.wrappers from airflow.www.app import create_app + from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_paused.py b/tests/www/views/test_views_paused.py index 59eee1868b70e..f38aee3fabb59 100644 --- a/tests/www/views/test_views_paused.py +++ b/tests/www/views/test_views_paused.py @@ -19,6 +19,7 @@ import pytest from airflow.models.log import Log + from tests_common.test_utils.db import clear_db_dags pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_pool.py b/tests/www/views/test_views_pool.py index 7b7b20d57c9cf..04c23a6e5eb3f 100644 --- a/tests/www/views/test_views_pool.py +++ b/tests/www/views/test_views_pool.py @@ -23,6 +23,7 @@ from airflow.models import Pool from airflow.utils.session import create_session + from tests_common.test_utils.www import check_content_in_response, check_content_not_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_rate_limit.py b/tests/www/views/test_views_rate_limit.py index eee9fed8dde00..9efbf0af64200 100644 --- a/tests/www/views/test_views_rate_limit.py +++ b/tests/www/views/test_views_rate_limit.py @@ -20,6 +20,7 @@ import pytest from airflow.www.app import create_app + from tests_common.test_utils.config import conf_vars from tests_common.test_utils.decorators import dont_initialize_flask_app_submodules from tests_common.test_utils.www import client_with_login diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index c23c300416a1f..822e657da2009 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -34,6 +34,7 @@ from airflow.utils.session import create_session from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator from tests_common.test_utils.db import ( clear_db_dags, diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index b1a57ef290528..034c6d5aab748 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -43,6 +43,12 @@ from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType from airflow.www.views import TaskInstanceModelView, _safe_parse_datetime + +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( + create_user, + delete_roles, + delete_user, +) from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_runs, clear_db_xcom @@ -52,12 +58,6 @@ client_with_login, ) -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( - create_user, - delete_roles, - delete_user, -) - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/www/views/test_views_trigger_dag.py b/tests/www/views/test_views_trigger_dag.py index f7e4d62599ea1..c6a936eebfaf6 100644 --- a/tests/www/views/test_views_trigger_dag.py +++ b/tests/www/views/test_views_trigger_dag.py @@ -32,10 +32,10 @@ from airflow.utils.json import WebEncoder from airflow.utils.session import create_session from airflow.utils.types import DagRunType -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.www import check_content_in_response from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_test_client +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.www import check_content_in_response pytestmark = pytest.mark.db_test diff --git a/tests/www/views/test_views_variable.py b/tests/www/views/test_views_variable.py index 2f1ec81fef1d6..aab164b74b75a 100644 --- a/tests/www/views/test_views_variable.py +++ b/tests/www/views/test_views_variable.py @@ -25,6 +25,8 @@ from airflow.models import Variable from airflow.security import permissions from airflow.utils.session import create_session + +from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user from tests_common.test_utils.www import ( _check_last_log, check_content_in_response, @@ -32,8 +34,6 @@ client_with_login, ) -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user - pytestmark = pytest.mark.db_test VARIABLE = { "key": "test_key", diff --git a/tests_common/pytest_plugin.py b/tests_common/pytest_plugin.py index bf34ae5be5f4b..f2dab17b2dddb 100644 --- a/tests_common/pytest_plugin.py +++ b/tests_common/pytest_plugin.py @@ -827,6 +827,7 @@ def create_dagrun(self, **kwargs): from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: @@ -924,6 +925,7 @@ def cleanup(self): from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskmap import TaskMap from airflow.utils.retries import run_with_db_retries + from tests_common.test_utils.compat import AssetEvent for attempt in run_with_db_retries(logger=self.log): diff --git a/tests_common/test_utils/api_connexion_utils.py b/tests_common/test_utils/api_connexion_utils.py index 23a273d93557f..3d940f2c86988 100644 --- a/tests_common/test_utils/api_connexion_utils.py +++ b/tests_common/test_utils/api_connexion_utils.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP + from tests_common.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): diff --git a/tests_common/test_utils/azure_system_helpers.py b/tests_common/test_utils/azure_system_helpers.py index 057124dea1f29..39b29cb9f3b14 100644 --- a/tests_common/test_utils/azure_system_helpers.py +++ b/tests_common/test_utils/azure_system_helpers.py @@ -28,6 +28,7 @@ from airflow.models import Connection from airflow.providers.microsoft.azure.hooks.fileshare import AzureFileShareHook from airflow.utils.process_utils import patch_environ + from tests_common.test_utils import AIRFLOW_MAIN_FOLDER from tests_common.test_utils.system_tests_class import SystemTest diff --git a/tests_common/test_utils/db.py b/tests_common/test_utils/db.py index c9df1f3970026..6a45cb072c8a2 100644 --- a/tests_common/test_utils/db.py +++ b/tests_common/test_utils/db.py @@ -43,6 +43,7 @@ from airflow.security.permissions import RESOURCE_DAG_PREFIX from airflow.utils.db import add_default_pool_if_not_exists, create_default_connections, reflect_tables from airflow.utils.session import create_session + from tests_common.test_utils.compat import ( AIRFLOW_V_2_10_PLUS, AssetDagRunQueue, @@ -61,6 +62,7 @@ def initial_db_init(): from airflow.utils import db from airflow.www.extensions.init_appbuilder import init_appbuilder from airflow.www.extensions.init_auth_manager import get_auth_manager + from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS db.resetdb() diff --git a/tests_common/test_utils/gcp_system_helpers.py b/tests_common/test_utils/gcp_system_helpers.py index 82b16da8490b7..504407dddd3aa 100644 --- a/tests_common/test_utils/gcp_system_helpers.py +++ b/tests_common/test_utils/gcp_system_helpers.py @@ -30,12 +30,12 @@ import airflow.providers.google from airflow.providers.google.cloud.utils.credentials_provider import provide_gcp_conn_and_credentials + +from providers.tests.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY, GCP_SECRET_MANAGER_KEY from tests_common.test_utils import AIRFLOW_MAIN_FOLDER from tests_common.test_utils.logging_command_executor import CommandExecutor from tests_common.test_utils.system_tests_class import SystemTest -from providers.tests.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY, GCP_SECRET_MANAGER_KEY - GCP_DIR = Path(airflow.providers.google.__file__).parent CLOUD_DAG_FOLDER = GCP_DIR.joinpath("cloud", "example_dags") MARKETING_DAG_FOLDER = GCP_DIR.joinpath("marketing_platform", "example_dags") diff --git a/tests_common/test_utils/mock_operators.py b/tests_common/test_utils/mock_operators.py index 7cc807a3a485b..4bf53551164cc 100644 --- a/tests_common/test_utils/mock_operators.py +++ b/tests_common/test_utils/mock_operators.py @@ -23,6 +23,7 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.xcom import XCom + from tests_common.test_utils.compat import BaseOperatorLink if TYPE_CHECKING: diff --git a/tests_common/test_utils/system_tests_class.py b/tests_common/test_utils/system_tests_class.py index 61f0b71d4c781..550cabee61e6b 100644 --- a/tests_common/test_utils/system_tests_class.py +++ b/tests_common/test_utils/system_tests_class.py @@ -28,6 +28,7 @@ from airflow.configuration import AIRFLOW_HOME, AirflowConfigParser, get_airflow_config from airflow.exceptions import AirflowException + from tests_common.test_utils import AIRFLOW_MAIN_FOLDER from tests_common.test_utils.logging_command_executor import get_executor From 2ffd0bf374fb950e101a878cd1c9563d1c49853d Mon Sep 17 00:00:00 2001 From: AutomationDev85 <96178949+AutomationDev85@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:47:31 +0200 Subject: [PATCH 151/199] Added task_instance_mutation_hook for mapped operator index 0 (#42661) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added task_instance_mutation_hook for mapped operator index 0 * Added unit test --------- Co-authored-by: Marco Küttelwesch --- airflow/models/abstractoperator.py | 2 ++ tests/models/test_mappedoperator.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 5e5d13d5dc266..45eb3c5fff189 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -654,6 +654,8 @@ def expand_mapped_task(self, run_id: str, *, session: Session) -> tuple[Sequence unmapped_ti.map_index = 0 self.log.debug("Updated in place to become %s", unmapped_ti) all_expanded_tis.append(unmapped_ti) + # execute hook for task instance map index 0 + task_instance_mutation_hook(unmapped_ti) session.flush() else: self.log.debug("Deleting the original task instance: %s", unmapped_ti) diff --git a/tests/models/test_mappedoperator.py b/tests/models/test_mappedoperator.py index 1bb616989ce64..c7bd11b966758 100644 --- a/tests/models/test_mappedoperator.py +++ b/tests/models/test_mappedoperator.py @@ -20,6 +20,7 @@ from collections import defaultdict from datetime import timedelta from typing import TYPE_CHECKING +from unittest import mock from unittest.mock import patch import pendulum @@ -730,6 +731,30 @@ def test_expand_mapped_task_instance_with_named_index( assert indices == expected_rendered_names +@pytest.mark.parametrize( + "create_mapped_task", + [ + pytest.param(_create_mapped_with_name_template_classic, id="classic"), + pytest.param(_create_mapped_with_name_template_taskflow, id="taskflow"), + ], +) +def test_expand_mapped_task_task_instance_mutation_hook(dag_maker, session, create_mapped_task) -> None: + """Test that the tast_instance_mutation_hook is called.""" + expected_map_index = [0, 1, 2] + + with dag_maker(session=session): + task1 = BaseOperator(task_id="op1") + mapped = MockOperator.partial(task_id="task_2").expand(arg2=task1.output) + + dr = dag_maker.create_dagrun() + + with mock.patch("airflow.settings.task_instance_mutation_hook") as mock_hook: + expand_mapped_task(mapped, dr.run_id, task1.task_id, length=len(expected_map_index), session=session) + + for index, call in enumerate(mock_hook.call_args_list): + assert call.args[0].map_index == expected_map_index[index] + + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "map_index, expected", From 96fbda92cf14e823667b034c24265472d122c63c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 16 Oct 2024 19:47:26 +0200 Subject: [PATCH 152/199] Fix selective checks when only pyproject.toml changes (#43088) In case only pyproject.toml changes we should automatically build CI image as we need to run full set of static checks - including mypy and other tools. This change triggers ci-image for such case and it also fixes a subtle bug where "pyproject_toml_changed" was always set to true (but nothing depended on it, because we are checking if build dependencies changed in pyproject.toml to trigger full build as most of the build logic and dependencies is in hatch_build.py The change first now attempts to see if the pyprject.toml changed at all (i.e. if pyprooject.toml is in the list of files that are changed in the commit) and only then reads the content of old and new version of pyproject.toml to compare them. This will make selective checks a bit faster as it won't run two git commands when pyproject.toml has not changed. Also tests_common are now added to "all sources" group which should trigger tests when only tests_common change. --- .../src/airflow_breeze/utils/selective_checks.py | 15 ++++++++++++++- dev/breeze/tests/test_selective_checks.py | 9 +++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 74a856f3693c6..3b16b51d59585 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -232,6 +232,7 @@ def __hash__(self): r"^task_sdk/src/", r"^task_sdk/tests/", r"^tests", + r"^tests_common", r"^kubernetes_tests", ], FileGroupForCi.SYSTEM_TEST_FILES: [ @@ -732,11 +733,21 @@ def needs_helm_tests(self) -> bool: @cached_property def run_tests(self) -> bool: + # we should run all test return self._should_be_run(FileGroupForCi.ALL_SOURCE_FILES) @cached_property def ci_image_build(self) -> bool: - return self.run_tests or self.docs_build or self.run_kubernetes_tests or self.needs_helm_tests + # in case pyproject.toml changed, CI image should be built - even if no build dependencies + # changes because some of our tests - those that need CI image might need to be run depending on + # changed rules for static checks that are part of the pyproject.toml file + return ( + self.run_tests + or self.docs_build + or self.run_kubernetes_tests + or self.needs_helm_tests + or self.pyproject_toml_changed + ) @cached_property def prod_image_build(self) -> bool: @@ -937,6 +948,8 @@ def pyproject_toml_changed(self) -> bool: if not self._commit_ref: get_console().print("[warning]Cannot determine pyproject.toml changes as commit is missing[/]") return False + if "pyproject.toml" not in self._files: + return False new_result = run_command( ["git", "show", f"{self._commit_ref}:pyproject.toml"], capture_output=True, diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ec0769c0a87fe..ec7f924d2d61f 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -131,6 +131,15 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="No tests on simple change", ) ), + ( + pytest.param( + ("pyproject.toml",), + { + "ci-image-build": "true", + }, + id="CI image build and when pyproject.toml change", + ) + ), ( pytest.param( ("airflow/api/file.py",), From dbf3c7f0c34cb6affd42698865a98afe5b87fee6 Mon Sep 17 00:00:00 2001 From: Josix Date: Thu, 17 Oct 2024 02:57:49 +0800 Subject: [PATCH 153/199] fix(providers/mongo): prevent applying lower method on boolean field (#43024) --- providers/src/airflow/providers/mongo/hooks/mongo.py | 6 +++--- providers/tests/mongo/hooks/test_mongo.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/providers/src/airflow/providers/mongo/hooks/mongo.py b/providers/src/airflow/providers/mongo/hooks/mongo.py index 63b981df9ef08..a15cc45491ddd 100644 --- a/providers/src/airflow/providers/mongo/hooks/mongo.py +++ b/providers/src/airflow/providers/mongo/hooks/mongo.py @@ -136,10 +136,10 @@ def __init__(self, mongo_conn_id: str = default_conn_name, *args, **kwargs) -> N self.client: MongoClient | None = None self.uri = self._create_uri() - self.allow_insecure = self.extras.pop("allow_insecure", "false").lower() == "true" + self.allow_insecure = str(self.extras.pop("allow_insecure", "false")).lower() == "true" self.ssl_enabled = ( - self.extras.get("ssl", "false").lower() == "true" - or self.extras.get("tls", "false").lower() == "true" + str(self.extras.get("ssl", "false")).lower() == "true" + or str(self.extras.get("tls", "false")).lower() == "true" ) if self.ssl_enabled and not self.allow_insecure: diff --git a/providers/tests/mongo/hooks/test_mongo.py b/providers/tests/mongo/hooks/test_mongo.py index 01e58f6081e37..5a284b69cc2af 100644 --- a/providers/tests/mongo/hooks/test_mongo.py +++ b/providers/tests/mongo/hooks/test_mongo.py @@ -63,6 +63,12 @@ def mongo_connections(): port=27017, extra='{"srv": true}', ), + Connection( + conn_id="mongo_default_with_allow_insecure_ssl_fields", + conn_type="mongo", + host="mongo", + extra='{"allow_insecure": false, "ssl": true}', + ), # Mongo establishes connection during initialization, so we need to have this connection Connection(conn_id="fake_connection", conn_type="mongo", host="mongo", port=27017), ] @@ -400,3 +406,9 @@ def test_context_manager(): assert ctx_hook.client is not None assert ctx_hook.client is None + + +def test_allow_insecure_and_ssl_enabled(): + hook = MongoHook(mongo_conn_id="mongo_default_with_allow_insecure_ssl_fields") + assert hook.allow_insecure is False + assert hook.ssl_enabled is True From 4d93ae517f7c0369511ea4f056e272a14dd504a3 Mon Sep 17 00:00:00 2001 From: GPK Date: Wed, 16 Oct 2024 20:27:58 +0100 Subject: [PATCH 154/199] add kubernetes_conn_id to templated fields (#42786) --- .../src/airflow/providers/cncf/kubernetes/operators/pod.py | 1 + .../providers/cncf/kubernetes/operators/spark_kubernetes.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py b/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py index 5f0462904447d..e51397447c3c7 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py @@ -259,6 +259,7 @@ class KubernetesPodOperator(BaseOperator): "cluster_context", "env_from", "node_selector", + "kubernetes_conn_id", ) template_fields_renderers = {"env_vars": "py"} diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py index 9bcf46d0d4f57..c3dd4755b9831 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +++ b/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py @@ -69,7 +69,7 @@ class SparkKubernetesOperator(KubernetesPodOperator): :param kubernetes_conn_id: the connection to Kubernetes cluster """ - template_fields = ["application_file", "namespace", "template_spec"] + template_fields = ["application_file", "namespace", "template_spec", "kubernetes_conn_id"] template_fields_renderers = {"template_spec": "py"} template_ext = ("yaml", "yml", "json") ui_color = "#f4a460" From 3a7b96b7f5f025e03a5a273a03dc1c25fcff3b93 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Thu, 17 Oct 2024 01:32:48 +0530 Subject: [PATCH 155/199] Add min version to cloudpickle (#43066) * pin cloudpickle >= 3.0.0 * pin cloudpickle >= 2.1.1 as apache-beam requires it to be ~=2.2.1 --- hatch_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 72f59c79ef9ff..6dc0e1c9c7c50 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -78,7 +78,8 @@ "cgroupspy>=0.2.2", ], "cloudpickle": [ - "cloudpickle", + # Latest version of apache-beam requires cloudpickle~=2.2.1 + "cloudpickle>=2.2.1", ], "github-enterprise": [ "apache-airflow[fab]", From 674554b543e834ea85461fb540edc19adc4f8ac3 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Wed, 16 Oct 2024 21:06:55 +0100 Subject: [PATCH 156/199] Initialize dashboard page with health (#43090) --- airflow/ui/src/App.tsx | 5 +- airflow/ui/src/layouts/Nav/Nav.tsx | 17 +---- airflow/ui/src/pages/DagsList/DagsList.tsx | 2 +- airflow/ui/src/pages/Dashboard/Dashboard.tsx | 31 +++++++++ airflow/ui/src/pages/Dashboard/Health.tsx | 67 +++++++++++++++++++ .../ui/src/pages/Dashboard/HealthSection.tsx | 59 ++++++++++++++++ airflow/ui/src/pages/Dashboard/HealthTag.tsx | 59 ++++++++++++++++ airflow/ui/src/pages/Dashboard/index.tsx | 20 ++++++ airflow/ui/src/utils/capitalize.ts | 23 +++++++ airflow/ui/src/utils/index.ts | 21 ++++++ 10 files changed, 286 insertions(+), 18 deletions(-) create mode 100644 airflow/ui/src/pages/Dashboard/Dashboard.tsx create mode 100644 airflow/ui/src/pages/Dashboard/Health.tsx create mode 100644 airflow/ui/src/pages/Dashboard/HealthSection.tsx create mode 100644 airflow/ui/src/pages/Dashboard/HealthTag.tsx create mode 100644 airflow/ui/src/pages/Dashboard/index.tsx create mode 100644 airflow/ui/src/utils/capitalize.ts create mode 100644 airflow/ui/src/utils/index.ts diff --git a/airflow/ui/src/App.tsx b/airflow/ui/src/App.tsx index 3c5e9d866f0c9..b8c73c35c7fc3 100644 --- a/airflow/ui/src/App.tsx +++ b/airflow/ui/src/App.tsx @@ -16,16 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -import { Navigate, Route, Routes } from "react-router-dom"; +import { Route, Routes } from "react-router-dom"; import { DagsList } from "src/pages/DagsList"; +import { Dashboard } from "src/pages/Dashboard"; import { BaseLayout } from "./layouts/BaseLayout"; export const App = () => ( } path="/"> - } index /> + } index /> } path="dags" /> diff --git a/airflow/ui/src/layouts/Nav/Nav.tsx b/airflow/ui/src/layouts/Nav/Nav.tsx index 84ab23497cf17..71b0843831ab6 100644 --- a/airflow/ui/src/layouts/Nav/Nav.tsx +++ b/airflow/ui/src/layouts/Nav/Nav.tsx @@ -16,14 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { - Box, - Flex, - Icon, - Link, - useColorModeValue, - VStack, -} from "@chakra-ui/react"; +import { Box, Flex, Icon, useColorModeValue, VStack } from "@chakra-ui/react"; import { motion } from "framer-motion"; import { FiBarChart2, @@ -68,12 +61,7 @@ export const Nav = () => { > - } - isDisabled - title="Home" - to="/" - /> + } title="Home" to="/" /> } title="DAGs" @@ -106,7 +94,6 @@ export const Nav = () => { } title="Return to legacy UI" to={import.meta.env.VITE_LEGACY_API_URL} diff --git a/airflow/ui/src/pages/DagsList/DagsList.tsx b/airflow/ui/src/pages/DagsList/DagsList.tsx index 623b8a3b4bacc..5e1324873296f 100644 --- a/airflow/ui/src/pages/DagsList/DagsList.tsx +++ b/airflow/ui/src/pages/DagsList/DagsList.tsx @@ -46,7 +46,7 @@ import { SearchParamsKeys, type SearchParamsKeysType, } from "src/constants/searchParams"; -import { pluralize } from "src/utils/pluralize"; +import { pluralize } from "src/utils"; import { DagCard } from "./DagCard"; import { DagsFilters } from "./DagsFilters"; diff --git a/airflow/ui/src/pages/Dashboard/Dashboard.tsx b/airflow/ui/src/pages/Dashboard/Dashboard.tsx new file mode 100644 index 0000000000000..f7ef15013bbeb --- /dev/null +++ b/airflow/ui/src/pages/Dashboard/Dashboard.tsx @@ -0,0 +1,31 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, Heading } from "@chakra-ui/react"; + +import { Health } from "./Health"; + +export const Dashboard = () => ( + + Welcome + + + + + +); diff --git a/airflow/ui/src/pages/Dashboard/Health.tsx b/airflow/ui/src/pages/Dashboard/Health.tsx new file mode 100644 index 0000000000000..dfad9d75c05c0 --- /dev/null +++ b/airflow/ui/src/pages/Dashboard/Health.tsx @@ -0,0 +1,67 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, Flex, Heading, HStack } from "@chakra-ui/react"; +import { FiClipboard } from "react-icons/fi"; + +import { useMonitorServiceGetHealth } from "openapi/queries"; +import { ErrorAlert } from "src/components/ErrorAlert"; + +import { HealthTag } from "./HealthTag"; + +export const Health = () => { + const { data, error, isLoading } = useMonitorServiceGetHealth(); + + return ( + + + + + Health + + + + + + + + {/* TODO add is_standalone to API to determine when to render this */} + + + + ); +}; diff --git a/airflow/ui/src/pages/Dashboard/HealthSection.tsx b/airflow/ui/src/pages/Dashboard/HealthSection.tsx new file mode 100644 index 0000000000000..85e545c6341d2 --- /dev/null +++ b/airflow/ui/src/pages/Dashboard/HealthSection.tsx @@ -0,0 +1,59 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Skeleton, Tag, TagLabel, Text, Tooltip } from "@chakra-ui/react"; + +export const HealthSection = ({ + isLoading, + latestHeartbeat, + status, + title, +}: { + readonly isLoading: boolean; + readonly latestHeartbeat?: null | string; + readonly status?: null | string; + readonly title: string; +}) => { + if (isLoading) { + return ; + } + + return ( + + Status: {status} + Last Heartbeat: {latestHeartbeat} + + } + shouldWrapChildren + > + + {title} + + + ); +}; diff --git a/airflow/ui/src/pages/Dashboard/HealthTag.tsx b/airflow/ui/src/pages/Dashboard/HealthTag.tsx new file mode 100644 index 0000000000000..335d2f91216b9 --- /dev/null +++ b/airflow/ui/src/pages/Dashboard/HealthTag.tsx @@ -0,0 +1,59 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Skeleton, Tag, TagLabel, Text, Tooltip } from "@chakra-ui/react"; + +import { capitalize } from "src/utils"; + +export const HealthTag = ({ + isLoading, + latestHeartbeat, + status, + title, +}: { + readonly isLoading: boolean; + readonly latestHeartbeat?: null | string; + readonly status?: null | string; + readonly title: string; +}) => { + if (isLoading) { + return ; + } + + return ( + + Status: {capitalize(status)} + Last Heartbeat: {latestHeartbeat} + + } + shouldWrapChildren + > + + {title} + + + ); +}; diff --git a/airflow/ui/src/pages/Dashboard/index.tsx b/airflow/ui/src/pages/Dashboard/index.tsx new file mode 100644 index 0000000000000..bf6b0036cdedf --- /dev/null +++ b/airflow/ui/src/pages/Dashboard/index.tsx @@ -0,0 +1,20 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { Dashboard } from "./Dashboard"; diff --git a/airflow/ui/src/utils/capitalize.ts b/airflow/ui/src/utils/capitalize.ts new file mode 100644 index 0000000000000..bd941c4667724 --- /dev/null +++ b/airflow/ui/src/utils/capitalize.ts @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// eslint-disable-next-line perfectionist/sort-union-types +export const capitalize = (string: string | null | undefined) => + // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions + string ? string.charAt(0).toUpperCase() + string.slice(1).toLowerCase() : ""; diff --git a/airflow/ui/src/utils/index.ts b/airflow/ui/src/utils/index.ts new file mode 100644 index 0000000000000..7083089c2b18c --- /dev/null +++ b/airflow/ui/src/utils/index.ts @@ -0,0 +1,21 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { capitalize } from "./capitalize"; +export { pluralize } from "./pluralize"; From 368c7807e8fb8833c7c9b69af3eee4890115372f Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Wed, 16 Oct 2024 22:32:14 +0200 Subject: [PATCH 157/199] Update constraints to broken canary (#43095) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 71f93248bddd8..746e77f97e328 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "pluggy==1.5.0", "smmap==5.0.1", "tomli==2.0.2; python_version < '3.11'", - "trove-classifiers==2024.10.13", + "trove-classifiers==2024.10.16", ] build-backend = "hatchling.build" From 106a542085e74f6dc08003cbb69973f6d5f4d6d8 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 16 Oct 2024 22:48:34 +0100 Subject: [PATCH 158/199] Fix flaky `test_get_dags` in FastAPI routes (#43100) --- .../core_api/routes/public/test_dags.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/api_fastapi/core_api/routes/public/test_dags.py b/tests/api_fastapi/core_api/routes/public/test_dags.py index edc350c27b84a..e365c4d5e47b0 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dags.py +++ b/tests/api_fastapi/core_api/routes/public/test_dags.py @@ -135,37 +135,37 @@ class TestGetDags(TestDagEndpoint): "query_params, expected_total_entries, expected_ids", [ # Filters - ({}, 2, [DAG1_ID, DAG2_ID]), - ({"limit": 1}, 2, [DAG1_ID]), - ({"offset": 1}, 2, [DAG2_ID]), - ({"tags": ["example"]}, 1, [DAG1_ID]), - ({"only_active": False}, 3, [DAG1_ID, DAG2_ID, DAG3_ID]), - ({"paused": True, "only_active": False}, 1, [DAG3_ID]), - ({"paused": False}, 2, [DAG1_ID, DAG2_ID]), - ({"owners": ["airflow"]}, 2, [DAG1_ID, DAG2_ID]), - ({"owners": ["test_owner"], "only_active": False}, 1, [DAG3_ID]), - ({"last_dag_run_state": "success", "only_active": False}, 1, [DAG3_ID]), - ({"last_dag_run_state": "failed", "only_active": False}, 1, [DAG1_ID]), + ({}, 2, {DAG1_ID, DAG2_ID}), + ({"limit": 1}, 2, {DAG1_ID}), + ({"offset": 1}, 2, {DAG2_ID}), + ({"tags": ["example"]}, 1, {DAG1_ID}), + ({"only_active": False}, 3, {DAG1_ID, DAG2_ID, DAG3_ID}), + ({"paused": True, "only_active": False}, 1, {DAG3_ID}), + ({"paused": False}, 2, {DAG1_ID, DAG2_ID}), + ({"owners": ["airflow"]}, 2, {DAG1_ID, DAG2_ID}), + ({"owners": ["test_owner"], "only_active": False}, 1, {DAG3_ID}), + ({"last_dag_run_state": "success", "only_active": False}, 1, {DAG3_ID}), + ({"last_dag_run_state": "failed", "only_active": False}, 1, {DAG1_ID}), # # Sort - ({"order_by": "-dag_id"}, 2, [DAG2_ID, DAG1_ID]), - ({"order_by": "-dag_display_name"}, 2, [DAG2_ID, DAG1_ID]), - ({"order_by": "dag_display_name"}, 2, [DAG1_ID, DAG2_ID]), - ({"order_by": "next_dagrun", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), - ({"order_by": "last_run_state", "only_active": False}, 3, [DAG1_ID, DAG3_ID, DAG2_ID]), - ({"order_by": "-last_run_state", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), + ({"order_by": "-dag_id"}, 2, {DAG2_ID, DAG1_ID}), + ({"order_by": "-dag_display_name"}, 2, {DAG2_ID, DAG1_ID}), + ({"order_by": "dag_display_name"}, 2, {DAG1_ID, DAG2_ID}), + ({"order_by": "next_dagrun", "only_active": False}, 3, {DAG3_ID, DAG1_ID, DAG2_ID}), + ({"order_by": "last_run_state", "only_active": False}, 3, {DAG1_ID, DAG3_ID, DAG2_ID}), + ({"order_by": "-last_run_state", "only_active": False}, 3, {DAG3_ID, DAG1_ID, DAG2_ID}), ( {"order_by": "last_run_start_date", "only_active": False}, 3, - [DAG1_ID, DAG3_ID, DAG2_ID], + {DAG1_ID, DAG3_ID, DAG2_ID}, ), ( {"order_by": "-last_run_start_date", "only_active": False}, 3, - [DAG3_ID, DAG1_ID, DAG2_ID], + {DAG3_ID, DAG1_ID, DAG2_ID}, ), # Search - ({"dag_id_pattern": "1"}, 1, [DAG1_ID]), - ({"dag_display_name_pattern": "test_dag2"}, 1, [DAG2_ID]), + ({"dag_id_pattern": "1"}, 1, {DAG1_ID}), + ({"dag_display_name_pattern": "test_dag2"}, 1, {DAG2_ID}), ], ) def test_get_dags(self, test_client, query_params, expected_total_entries, expected_ids): @@ -175,7 +175,7 @@ def test_get_dags(self, test_client, query_params, expected_total_entries, expec body = response.json() assert body["total_entries"] == expected_total_entries - assert [dag["dag_id"] for dag in body["dags"]] == expected_ids + assert set(dag["dag_id"] for dag in body["dags"]) == expected_ids class TestPatchDag(TestDagEndpoint): From cbe4a2f70374582290aae62895f74ad3271f5646 Mon Sep 17 00:00:00 2001 From: Gyandeep Singh Date: Thu, 17 Oct 2024 00:10:41 -0500 Subject: [PATCH 159/199] SSHHook expose auth_timeout parameter (#43048) * SSHHook expose auth_timeout parameter * add tests --- providers/src/airflow/providers/ssh/hooks/ssh.py | 4 ++++ providers/tests/ssh/hooks/test_ssh.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/providers/src/airflow/providers/ssh/hooks/ssh.py b/providers/src/airflow/providers/ssh/hooks/ssh.py index e5ce9ec27cfe5..ecd3990d06b9a 100644 --- a/providers/src/airflow/providers/ssh/hooks/ssh.py +++ b/providers/src/airflow/providers/ssh/hooks/ssh.py @@ -74,6 +74,7 @@ class SSHHook(BaseHook): iterable of algorithm identifiers, which will be disabled for the lifetime of the transport :param ciphers: list of ciphers to use in order of preference + :param auth_timeout: timeout (in seconds) for the attempt to authenticate with the remote_host """ # List of classes to try loading private keys as, ordered (roughly) by most common to least common @@ -121,6 +122,7 @@ def __init__( banner_timeout: float = 30.0, disabled_algorithms: dict | None = None, ciphers: list[str] | None = None, + auth_timeout: int | None = None, ) -> None: super().__init__() self.ssh_conn_id = ssh_conn_id @@ -138,6 +140,7 @@ def __init__( self.disabled_algorithms = disabled_algorithms self.ciphers = ciphers self.host_proxy_cmd = None + self.auth_timeout = auth_timeout # Default values, overridable from Connection self.compress = True @@ -332,6 +335,7 @@ def get_conn(self) -> paramiko.SSHClient: "sock": self.host_proxy, "look_for_keys": self.look_for_keys, "banner_timeout": self.banner_timeout, + "auth_timeout": self.auth_timeout, } if self.password: diff --git a/providers/tests/ssh/hooks/test_ssh.py b/providers/tests/ssh/hooks/test_ssh.py index 6b3e5dcdae116..ac40d01c6f68c 100644 --- a/providers/tests/ssh/hooks/test_ssh.py +++ b/providers/tests/ssh/hooks/test_ssh.py @@ -362,6 +362,7 @@ def test_ssh_connection_with_password(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") @@ -381,6 +382,7 @@ def test_ssh_connection_without_password(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.SSHTunnelForwarder") @@ -569,6 +571,7 @@ def test_ssh_connection_with_private_key_extra(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") @@ -592,6 +595,7 @@ def test_ssh_connection_with_private_key_passphrase_extra(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") @@ -654,6 +658,7 @@ def test_ssh_connection_with_conn_timeout(self, ssh_mock): password="password", conn_timeout=20, key_file="fake.file", + auth_timeout=10, ) with hook.get_conn(): @@ -668,6 +673,7 @@ def test_ssh_connection_with_conn_timeout(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=10, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") @@ -695,6 +701,7 @@ def test_ssh_connection_with_conn_timeout_and_timeout(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") @@ -718,6 +725,7 @@ def test_ssh_connection_with_timeout_extra(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") @@ -743,6 +751,7 @@ def test_ssh_connection_with_conn_timeout_extra(self, ssh_mock): port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") @@ -768,6 +777,7 @@ def test_ssh_connection_with_timeout_extra_and_conn_timeout_extra(self, ssh_mock port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @pytest.mark.parametrize( @@ -837,6 +847,7 @@ def test_ssh_connection_with_all_timeout_param_and_extra_combinations( port="port", sock=None, look_for_keys=True, + auth_timeout=None, ) @pytest.mark.parametrize( @@ -901,6 +912,7 @@ def test_ssh_with_extra_disabled_algorithms(self, ssh_mock): sock=None, look_for_keys=True, disabled_algorithms=TEST_DISABLED_ALGORITHMS, + auth_timeout=None, ) @mock.patch("airflow.providers.ssh.hooks.ssh.paramiko.SSHClient") From 76fda155a9a309384fa76a1c6d591608b3d9a26a Mon Sep 17 00:00:00 2001 From: WS Hoekstra Date: Thu, 17 Oct 2024 09:43:55 +0200 Subject: [PATCH 160/199] vertex ai training operators: add display_name to rendered fields (#43028) * vertex ai training operators: add display_name to rendered fields * fix validate-operators-init static checks --------- Co-authored-by: Walter Hoekstra Co-authored-by: Shahar Epstein <60007259+shahar1@users.noreply.github.com> --- .../cloud/operators/vertex_ai/auto_ml.py | 11 ++++++++++- .../vertex_ai/batch_prediction_job.py | 2 +- .../cloud/operators/vertex_ai/custom_job.py | 19 +++++++++++++++++++ .../vertex_ai/hyperparameter_tuning_job.py | 1 + 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py index ec9a6784f79ff..86b3ae017c080 100644 --- a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +++ b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py @@ -106,6 +106,8 @@ class CreateAutoMLForecastingTrainingJobOperator(AutoMLTrainingJobBaseOperator): "dataset_id", "region", "impersonation_chain", + "display_name", + "model_display_name", ) operator_extra_links = (VertexAIModelLink(), VertexAITrainingLink()) @@ -121,6 +123,8 @@ def __init__( forecast_horizon: int, data_granularity_unit: str, data_granularity_count: int, + display_name: str, + model_display_name: str | None = None, optimization_objective: str | None = None, column_specs: dict[str, str] | None = None, column_transformations: list[dict[str, dict[str, str]]] | None = None, @@ -143,7 +147,12 @@ def __init__( **kwargs, ) -> None: super().__init__( - region=region, impersonation_chain=impersonation_chain, parent_model=parent_model, **kwargs + display_name=display_name, + model_display_name=model_display_name, + region=region, + impersonation_chain=impersonation_chain, + parent_model=parent_model, + **kwargs, ) self.dataset_id = dataset_id self.target_column = target_column diff --git a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py index a57a072fb68de..60ddd747eec56 100644 --- a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +++ b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py @@ -163,7 +163,7 @@ class CreateBatchPredictionJobOperator(GoogleCloudBaseOperator): :param poll_interval: Interval size which defines how often job status is checked in deferrable mode. """ - template_fields = ("region", "project_id", "model_name", "impersonation_chain") + template_fields = ("region", "project_id", "model_name", "impersonation_chain", "job_display_name") operator_extra_links = (VertexAIBatchPredictionJobLink(),) def __init__( diff --git a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py index 854101a72e34d..33ab07d075396 100644 --- a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +++ b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py @@ -496,6 +496,8 @@ class CreateCustomContainerTrainingJobOperator(CustomTrainingJobBaseOperator): "parent_model", "dataset_id", "impersonation_chain", + "display_name", + "model_display_name", ) operator_extra_links = ( VertexAIModelLink(), @@ -507,6 +509,8 @@ def __init__( *, command: Sequence[str] = [], region: str, + display_name: str, + model_display_name: str | None = None, parent_model: str | None = None, impersonation_chain: str | Sequence[str] | None = None, dataset_id: str | None = None, @@ -515,6 +519,8 @@ def __init__( **kwargs, ) -> None: super().__init__( + display_name=display_name, + model_display_name=model_display_name, region=region, parent_model=parent_model, impersonation_chain=impersonation_chain, @@ -949,6 +955,8 @@ class CreateCustomPythonPackageTrainingJobOperator(CustomTrainingJobBaseOperator "region", "dataset_id", "impersonation_chain", + "display_name", + "model_display_name", ) operator_extra_links = (VertexAIModelLink(), VertexAITrainingLink()) @@ -958,6 +966,8 @@ def __init__( python_package_gcs_uri: str, python_module_name: str, region: str, + display_name: str, + model_display_name: str | None = None, parent_model: str | None = None, impersonation_chain: str | Sequence[str] | None = None, dataset_id: str | None = None, @@ -966,6 +976,8 @@ def __init__( **kwargs, ) -> None: super().__init__( + display_name=display_name, + model_display_name=model_display_name, region=region, parent_model=parent_model, impersonation_chain=impersonation_chain, @@ -1405,6 +1417,8 @@ class CreateCustomTrainingJobOperator(CustomTrainingJobBaseOperator): "requirements", "dataset_id", "impersonation_chain", + "display_name", + "model_display_name", ) operator_extra_links = ( VertexAIModelLink(), @@ -1417,6 +1431,8 @@ def __init__( script_path: str, requirements: Sequence[str] | None = None, region: str, + display_name: str, + model_display_name: str | None = None, parent_model: str | None = None, impersonation_chain: str | Sequence[str] | None = None, dataset_id: str | None = None, @@ -1425,6 +1441,8 @@ def __init__( **kwargs, ) -> None: super().__init__( + display_name=display_name, + model_display_name=model_display_name, region=region, parent_model=parent_model, impersonation_chain=impersonation_chain, @@ -1732,6 +1750,7 @@ class ListCustomTrainingJobOperator(GoogleCloudBaseOperator): "region", "project_id", "impersonation_chain", + "display_name", ] operator_extra_links = [ VertexAITrainingPipelinesLink(), diff --git a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py index 2da2bf33946b8..43229e033cc8f 100644 --- a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +++ b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py @@ -147,6 +147,7 @@ class CreateHyperparameterTuningJobOperator(GoogleCloudBaseOperator): "region", "project_id", "impersonation_chain", + "display_name", ] operator_extra_links = (VertexAITrainingLink(),) From 8af1dc73bd159a7f67af727ffae11f4dca9d63cd Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 17 Oct 2024 03:48:16 -0400 Subject: [PATCH 161/199] Print the key name when max_length is exceeded (#43061) * Print the key name when max_length is exceeded * Fix tests --- airflow/utils/helpers.py | 2 +- tests/utils/test_helpers.py | 2 +- tests/www/test_validators.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 8c08acc66be24..8b83e3b134960 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -50,7 +50,7 @@ def validate_key(k: str, max_length: int = 250): if not isinstance(k, str): raise TypeError(f"The key has to be a string and is {type(k)}:{k}") if len(k) > max_length: - raise AirflowException(f"The key has to be less than {max_length} characters") + raise AirflowException(f"The key: {k} has to be less than {max_length} characters") if not KEY_REGEX.match(k): raise AirflowException( f"The key {k!r} has to be made of alphanumeric characters, dashes, " diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index 041dba2a90e4f..ed1b1134d842a 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -203,7 +203,7 @@ def test_build_airflow_url_with_query(self): "characters, dashes, dots and underscores exclusively", AirflowException, ), - (" " * 251, "The key has to be less than 250 characters", AirflowException), + (" " * 251, f"The key: {' ' * 251} has to be less than 250 characters", AirflowException), ], ) def test_validate_key(self, key_id, message, exception): diff --git a/tests/www/test_validators.py b/tests/www/test_validators.py index e92cc9d80453d..02923f9e2afd8 100644 --- a/tests/www/test_validators.py +++ b/tests/www/test_validators.py @@ -151,7 +151,7 @@ def test_validation_fails_with_too_many_characters(self): with pytest.raises( validators.ValidationError, - match=r"The key has to be less than [0-9]+ characters", + match=r"The key: [x]+ has to be less than [0-9]+ characters", ): self._validate() From b799d40eeb66b4ef761950aac15c92cfdb52506c Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Thu, 17 Oct 2024 18:07:06 +0800 Subject: [PATCH 162/199] Revert "Fix flaky `test_get_dags` in FastAPI routes (#43100)" (#43108) --- .../core_api/routes/public/test_dags.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/api_fastapi/core_api/routes/public/test_dags.py b/tests/api_fastapi/core_api/routes/public/test_dags.py index e365c4d5e47b0..edc350c27b84a 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dags.py +++ b/tests/api_fastapi/core_api/routes/public/test_dags.py @@ -135,37 +135,37 @@ class TestGetDags(TestDagEndpoint): "query_params, expected_total_entries, expected_ids", [ # Filters - ({}, 2, {DAG1_ID, DAG2_ID}), - ({"limit": 1}, 2, {DAG1_ID}), - ({"offset": 1}, 2, {DAG2_ID}), - ({"tags": ["example"]}, 1, {DAG1_ID}), - ({"only_active": False}, 3, {DAG1_ID, DAG2_ID, DAG3_ID}), - ({"paused": True, "only_active": False}, 1, {DAG3_ID}), - ({"paused": False}, 2, {DAG1_ID, DAG2_ID}), - ({"owners": ["airflow"]}, 2, {DAG1_ID, DAG2_ID}), - ({"owners": ["test_owner"], "only_active": False}, 1, {DAG3_ID}), - ({"last_dag_run_state": "success", "only_active": False}, 1, {DAG3_ID}), - ({"last_dag_run_state": "failed", "only_active": False}, 1, {DAG1_ID}), + ({}, 2, [DAG1_ID, DAG2_ID]), + ({"limit": 1}, 2, [DAG1_ID]), + ({"offset": 1}, 2, [DAG2_ID]), + ({"tags": ["example"]}, 1, [DAG1_ID]), + ({"only_active": False}, 3, [DAG1_ID, DAG2_ID, DAG3_ID]), + ({"paused": True, "only_active": False}, 1, [DAG3_ID]), + ({"paused": False}, 2, [DAG1_ID, DAG2_ID]), + ({"owners": ["airflow"]}, 2, [DAG1_ID, DAG2_ID]), + ({"owners": ["test_owner"], "only_active": False}, 1, [DAG3_ID]), + ({"last_dag_run_state": "success", "only_active": False}, 1, [DAG3_ID]), + ({"last_dag_run_state": "failed", "only_active": False}, 1, [DAG1_ID]), # # Sort - ({"order_by": "-dag_id"}, 2, {DAG2_ID, DAG1_ID}), - ({"order_by": "-dag_display_name"}, 2, {DAG2_ID, DAG1_ID}), - ({"order_by": "dag_display_name"}, 2, {DAG1_ID, DAG2_ID}), - ({"order_by": "next_dagrun", "only_active": False}, 3, {DAG3_ID, DAG1_ID, DAG2_ID}), - ({"order_by": "last_run_state", "only_active": False}, 3, {DAG1_ID, DAG3_ID, DAG2_ID}), - ({"order_by": "-last_run_state", "only_active": False}, 3, {DAG3_ID, DAG1_ID, DAG2_ID}), + ({"order_by": "-dag_id"}, 2, [DAG2_ID, DAG1_ID]), + ({"order_by": "-dag_display_name"}, 2, [DAG2_ID, DAG1_ID]), + ({"order_by": "dag_display_name"}, 2, [DAG1_ID, DAG2_ID]), + ({"order_by": "next_dagrun", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), + ({"order_by": "last_run_state", "only_active": False}, 3, [DAG1_ID, DAG3_ID, DAG2_ID]), + ({"order_by": "-last_run_state", "only_active": False}, 3, [DAG3_ID, DAG1_ID, DAG2_ID]), ( {"order_by": "last_run_start_date", "only_active": False}, 3, - {DAG1_ID, DAG3_ID, DAG2_ID}, + [DAG1_ID, DAG3_ID, DAG2_ID], ), ( {"order_by": "-last_run_start_date", "only_active": False}, 3, - {DAG3_ID, DAG1_ID, DAG2_ID}, + [DAG3_ID, DAG1_ID, DAG2_ID], ), # Search - ({"dag_id_pattern": "1"}, 1, {DAG1_ID}), - ({"dag_display_name_pattern": "test_dag2"}, 1, {DAG2_ID}), + ({"dag_id_pattern": "1"}, 1, [DAG1_ID]), + ({"dag_display_name_pattern": "test_dag2"}, 1, [DAG2_ID]), ], ) def test_get_dags(self, test_client, query_params, expected_total_entries, expected_ids): @@ -175,7 +175,7 @@ def test_get_dags(self, test_client, query_params, expected_total_entries, expec body = response.json() assert body["total_entries"] == expected_total_entries - assert set(dag["dag_id"] for dag in body["dags"]) == expected_ids + assert [dag["dag_id"] for dag in body["dags"]] == expected_ids class TestPatchDag(TestDagEndpoint): From 79d5af9464afeb0222997ce1a99f99c8e423707a Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Thu, 17 Oct 2024 18:40:06 +0800 Subject: [PATCH 163/199] Ensure stable secondary ordering (#43085) --- airflow/api_fastapi/common/parameters.py | 30 ++++++++++-------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/airflow/api_fastapi/common/parameters.py b/airflow/api_fastapi/common/parameters.py index 07d8a76c79c3e..4aa8335905ca0 100644 --- a/airflow/api_fastapi/common/parameters.py +++ b/airflow/api_fastapi/common/parameters.py @@ -17,7 +17,6 @@ from __future__ import annotations -import importlib from abc import ABC, abstractmethod from datetime import datetime from typing import TYPE_CHECKING, Any, Callable, Generic, List, TypeVar @@ -195,32 +194,27 @@ def to_orm(self, select: Select) -> Select: # Reset default sorting select = select.order_by(None) + primary_key_column = self.get_primary_key_column() + if self.value[0] == "-": - return select.order_by(nullscheck, column.desc(), column.desc()) + return select.order_by(nullscheck, column.desc(), primary_key_column) else: - return select.order_by(nullscheck, column.asc(), column.asc()) - - def get_primary_key(self) -> str: - """Get the primary key of the model of SortParam object.""" - return inspect(self.model).primary_key[0].name + return select.order_by(nullscheck, column.asc(), primary_key_column) - @staticmethod - def get_primary_key_of_given_model_string(model_string: str) -> str: - """ - Get the primary key of given 'airflow.models' class as a string. The class should have driven be from 'airflow.models.base'. + def get_primary_key_column(self) -> Column: + """Get the primary key column of the model of SortParam object.""" + return inspect(self.model).primary_key[0] - :param model_string: The string representation of the model class. - :return: The primary key of the model class. - """ - dynamic_return_model = getattr(importlib.import_module("airflow.models"), model_string) - return inspect(dynamic_return_model).primary_key[0].name + def get_primary_key_string(self) -> str: + """Get the primary key string of the model of SortParam object.""" + return self.get_primary_key_column().name def depends(self, *args: Any, **kwargs: Any) -> Self: raise NotImplementedError("Use dynamic_depends, depends not implemented.") def dynamic_depends(self) -> Callable: - def inner(order_by: str = self.get_primary_key()) -> SortParam: - return self.set_value(self.get_primary_key() if order_by == "" else order_by) + def inner(order_by: str = self.get_primary_key_string()) -> SortParam: + return self.set_value(self.get_primary_key_string() if order_by == "" else order_by) return inner From 97431aaac417bac4a341d14c0a92f2c4eb389814 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 17 Oct 2024 11:44:47 +0100 Subject: [PATCH 164/199] Add simple "Task Execution" API server (#43015) closes https://github.com/apache/airflow/issues/43009 The Task Execution API is created as a separate FastAPI APP so that it allows us to hook CLI and selectively run the "Execution API" with the rest of Airflow API (UI/public). This is so that users can scale the APIs separately if needed as mentioned in #43009 --- airflow/api_fastapi/app.py | 22 +++--- airflow/api_fastapi/execution_api/__init__.py | 16 +++++ airflow/api_fastapi/execution_api/app.py | 34 +++++++++ .../execution_api/routes/__init__.py | 23 ++++++ .../execution_api/routes/health.py | 27 +++++++ airflow/api_fastapi/main.py | 7 +- airflow/cli/cli_config.py | 6 ++ airflow/cli/commands/fastapi_api_command.py | 12 +++- tests/api_fastapi/conftest.py | 11 +++ tests/api_fastapi/execution_api/__init__.py | 16 +++++ .../execution_api/routes/__init__.py | 16 +++++ .../execution_api/routes/test_health.py | 24 +++++++ tests/api_fastapi/test_app.py | 71 +++++++++++++++++++ .../cli/commands/test_fastapi_api_command.py | 47 +++++++++++- 14 files changed, 320 insertions(+), 12 deletions(-) create mode 100644 airflow/api_fastapi/execution_api/__init__.py create mode 100644 airflow/api_fastapi/execution_api/app.py create mode 100644 airflow/api_fastapi/execution_api/routes/__init__.py create mode 100644 airflow/api_fastapi/execution_api/routes/health.py create mode 100644 tests/api_fastapi/execution_api/__init__.py create mode 100644 tests/api_fastapi/execution_api/routes/__init__.py create mode 100644 tests/api_fastapi/execution_api/routes/test_health.py create mode 100644 tests/api_fastapi/test_app.py diff --git a/airflow/api_fastapi/app.py b/airflow/api_fastapi/app.py index ffe13bec0fa76..9c3e5a3bd7983 100644 --- a/airflow/api_fastapi/app.py +++ b/airflow/api_fastapi/app.py @@ -22,15 +22,18 @@ from fastapi.middleware.cors import CORSMiddleware from airflow.api_fastapi.core_api.app import init_dag_bag, init_plugins, init_views +from airflow.api_fastapi.execution_api.app import create_task_execution_api_app log = logging.getLogger(__name__) app: FastAPI | None = None -def create_app(apps: list[str] | None = None) -> FastAPI: +def create_app(apps: str = "all") -> FastAPI: from airflow.configuration import conf + apps_list = apps.split(",") if apps else ["all"] + app = FastAPI( title="Airflow API", description="Airflow API. All endpoints located under ``/public`` can be used safely, are stable and backward compatible. " @@ -38,11 +41,14 @@ def create_app(apps: list[str] | None = None) -> FastAPI: "depending on the need of the frontend. Users should not rely on those but use the public ones instead.", ) - init_dag_bag(app) - - init_views(app) + if "core" in apps_list or "all" in apps_list: + init_dag_bag(app) + init_views(app) + init_plugins(app) - init_plugins(app) + if "execution" in apps_list or "all" in apps_list: + task_exec_api_app = create_task_execution_api_app(app) + app.mount("/execution", task_exec_api_app) allow_origins = conf.getlist("api", "access_control_allow_origins") allow_methods = conf.getlist("api", "access_control_allow_methods") @@ -60,11 +66,11 @@ def create_app(apps: list[str] | None = None) -> FastAPI: return app -def cached_app(config=None, testing=False) -> FastAPI: - """Return cached instance of Airflow UI app.""" +def cached_app(config=None, testing=False, apps="all") -> FastAPI: + """Return cached instance of Airflow API app.""" global app if not app: - app = create_app() + app = create_app(apps=apps) return app diff --git a/airflow/api_fastapi/execution_api/__init__.py b/airflow/api_fastapi/execution_api/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/api_fastapi/execution_api/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/api_fastapi/execution_api/app.py b/airflow/api_fastapi/execution_api/app.py new file mode 100644 index 0000000000000..771b81c43ae06 --- /dev/null +++ b/airflow/api_fastapi/execution_api/app.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from fastapi import FastAPI + + +def create_task_execution_api_app(app: FastAPI) -> FastAPI: + """Create FastAPI app for task execution API.""" + from airflow.api_fastapi.execution_api.routes import execution_api_router + + task_exec_api_app = FastAPI( + title="Airflow Task Execution API", + description="The private Airflow Task Execution API.", + include_in_schema=False, + ) + + task_exec_api_app.include_router(execution_api_router) + return task_exec_api_app diff --git a/airflow/api_fastapi/execution_api/routes/__init__.py b/airflow/api_fastapi/execution_api/routes/__init__.py new file mode 100644 index 0000000000000..3d8761caefbcc --- /dev/null +++ b/airflow/api_fastapi/execution_api/routes/__init__.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.execution_api.routes.health import health_router + +execution_api_router = AirflowRouter() +execution_api_router.include_router(health_router) diff --git a/airflow/api_fastapi/execution_api/routes/health.py b/airflow/api_fastapi/execution_api/routes/health.py new file mode 100644 index 0000000000000..21ef586b8c76a --- /dev/null +++ b/airflow/api_fastapi/execution_api/routes/health.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from airflow.api_fastapi.common.router import AirflowRouter + +health_router = AirflowRouter(tags=["Task SDK"]) + + +@health_router.get("/health") +async def health() -> dict: + return {"status": "healthy"} diff --git a/airflow/api_fastapi/main.py b/airflow/api_fastapi/main.py index 9a204306efd4e..16419b9485ead 100644 --- a/airflow/api_fastapi/main.py +++ b/airflow/api_fastapi/main.py @@ -17,6 +17,11 @@ from __future__ import annotations +import os + from airflow.api_fastapi.app import cached_app -app = cached_app() +# There is no way to pass the apps to this file from Airflow CLI +# because fastapi dev command does not accept any additional arguments +# so environment variable is being used to pass it +app = cached_app(apps=os.environ.get("AIRFLOW_API_APPS", "all")) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 8fe239c27ba40..5d1fe9ba8e51e 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -769,6 +769,11 @@ def string_lower_type(val): ("-L", "--access-logformat"), help="The access log format for gunicorn logs", ) +ARG_FASTAPI_API_APPS = Arg( + ("--apps",), + help="Applications to run (comma-separated). Default is all. Options: core, execution, all", + default="all", +) # scheduler @@ -1936,6 +1941,7 @@ class GroupCommand(NamedTuple): ARG_FASTAPI_API_ACCESS_LOGFILE, ARG_FASTAPI_API_ERROR_LOGFILE, ARG_FASTAPI_API_ACCESS_LOGFORMAT, + ARG_FASTAPI_API_APPS, ARG_LOG_FILE, ARG_SSL_CERT, ARG_SSL_KEY, diff --git a/airflow/cli/commands/fastapi_api_command.py b/airflow/cli/commands/fastapi_api_command.py index c11da959ce910..33ef517197687 100644 --- a/airflow/cli/commands/fastapi_api_command.py +++ b/airflow/cli/commands/fastapi_api_command.py @@ -57,6 +57,7 @@ def fastapi_api(args): """Start Airflow FastAPI API.""" print(settings.HEADER) + apps = args.apps access_logfile = args.access_logfile or "-" error_logfile = args.error_logfile or "-" access_logformat = args.access_logformat @@ -81,16 +82,22 @@ def fastapi_api(args): str(args.hostname), ] + # There is no way to pass the apps to airflow/api_fastapi/main.py in the debug mode + # because fastapi dev command does not accept any additional arguments + # so environment variable is being used to pass it + os.environ["AIRFLOW_API_APPS"] = apps with subprocess.Popen( run_args, close_fds=True, ) as process: process.wait() + os.environ.pop("AIRFLOW_API_APPS") else: log.info( textwrap.dedent( f"""\ Running the Gunicorn Server with: + Apps: {apps} Workers: {num_workers} {worker_class} Host: {args.hostname}:{args.port} Timeout: {worker_timeout} @@ -101,6 +108,7 @@ def fastapi_api(args): ) pid_file, _, _, _ = setup_locations("fastapi-api", pid=args.pid) + run_args = [ sys.executable, "-m", @@ -135,7 +143,7 @@ def fastapi_api(args): if args.daemon: run_args += ["--daemon"] - run_args += ["airflow.api_fastapi.app:cached_app()"] + run_args += [f"airflow.api_fastapi.app:cached_app(apps='{apps}')"] # To prevent different workers creating the web app and # all writing to the database at the same time, we use the --preload option. @@ -192,7 +200,7 @@ def start_and_monitor_gunicorn(args): if args.daemon: # This makes possible errors get reported before daemonization os.environ["SKIP_DAGS_PARSING"] = "True" - create_app() + create_app(apps) os.environ.pop("SKIP_DAGS_PARSING") pid_file_path = Path(pid_file) diff --git a/tests/api_fastapi/conftest.py b/tests/api_fastapi/conftest.py index c5212272d7306..2ef3b14369a31 100644 --- a/tests/api_fastapi/conftest.py +++ b/tests/api_fastapi/conftest.py @@ -25,3 +25,14 @@ @pytest.fixture def test_client(): return TestClient(create_app()) + + +@pytest.fixture +def client(): + """This fixture is more flexible than test_client, as it allows to specify which apps to include.""" + + def create_test_client(apps="all"): + app = create_app(apps=apps) + return TestClient(app) + + return create_test_client diff --git a/tests/api_fastapi/execution_api/__init__.py b/tests/api_fastapi/execution_api/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/api_fastapi/execution_api/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/api_fastapi/execution_api/routes/__init__.py b/tests/api_fastapi/execution_api/routes/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/api_fastapi/execution_api/routes/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/api_fastapi/execution_api/routes/test_health.py b/tests/api_fastapi/execution_api/routes/test_health.py new file mode 100644 index 0000000000000..3d8b8dd97a254 --- /dev/null +++ b/tests/api_fastapi/execution_api/routes/test_health.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +def test_health(test_client): + response = test_client.get("/execution/health") + + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} diff --git a/tests/api_fastapi/test_app.py b/tests/api_fastapi/test_app.py new file mode 100644 index 0000000000000..3dddd827ff444 --- /dev/null +++ b/tests/api_fastapi/test_app.py @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + + +@mock.patch("airflow.api_fastapi.app.init_dag_bag") +@mock.patch("airflow.api_fastapi.app.init_views") +@mock.patch("airflow.api_fastapi.app.init_plugins") +@mock.patch("airflow.api_fastapi.app.create_task_execution_api_app") +def test_core_api_app( + mock_create_task_exec_api, mock_init_plugins, mock_init_views, mock_init_dag_bag, client +): + test_app = client(apps="core").app + + # Assert that core-related functions were called + mock_init_dag_bag.assert_called_once_with(test_app) + mock_init_views.assert_called_once_with(test_app) + mock_init_plugins.assert_called_once_with(test_app) + + # Assert that execution-related functions were NOT called + mock_create_task_exec_api.assert_not_called() + + +@mock.patch("airflow.api_fastapi.app.init_dag_bag") +@mock.patch("airflow.api_fastapi.app.init_views") +@mock.patch("airflow.api_fastapi.app.init_plugins") +@mock.patch("airflow.api_fastapi.app.create_task_execution_api_app") +def test_execution_api_app( + mock_create_task_exec_api, mock_init_plugins, mock_init_views, mock_init_dag_bag, client +): + test_app = client(apps="execution").app + + # Assert that execution-related functions were called + mock_create_task_exec_api.assert_called_once_with(test_app) + + # Assert that core-related functions were NOT called + mock_init_dag_bag.assert_not_called() + mock_init_views.assert_not_called() + mock_init_plugins.assert_not_called() + + +@mock.patch("airflow.api_fastapi.app.init_dag_bag") +@mock.patch("airflow.api_fastapi.app.init_views") +@mock.patch("airflow.api_fastapi.app.init_plugins") +@mock.patch("airflow.api_fastapi.app.create_task_execution_api_app") +def test_all_apps(mock_create_task_exec_api, mock_init_plugins, mock_init_views, mock_init_dag_bag, client): + test_app = client(apps="all").app + + # Assert that core-related functions were called + mock_init_dag_bag.assert_called_once_with(test_app) + mock_init_views.assert_called_once_with(test_app) + mock_init_plugins.assert_called_once_with(test_app) + + # Assert that execution-related functions were also called + mock_create_task_exec_api.assert_called_once_with(test_app) diff --git a/tests/cli/commands/test_fastapi_api_command.py b/tests/cli/commands/test_fastapi_api_command.py index b1a02b3b186e1..7f58f1ac482b3 100644 --- a/tests/cli/commands/test_fastapi_api_command.py +++ b/tests/cli/commands/test_fastapi_api_command.py @@ -122,6 +122,49 @@ def test_cli_fastapi_api_debug(self, app): close_fds=True, ) + def test_cli_fastapi_api_env_var_set_unset(self, app): + """ + Test that AIRFLOW_API_APPS is set and unset in the environment when + calling the airflow fastapi-api command + """ + with mock.patch("subprocess.Popen") as Popen, mock.patch.object( + fastapi_api_command, "GunicornMonitor" + ), mock.patch("os.environ", autospec=True) as mock_environ: + apps_value = "core,execution" + port = "9092" + hostname = "somehost" + + # Parse the command line arguments + args = self.parser.parse_args( + ["fastapi-api", "--port", port, "--hostname", hostname, "--apps", apps_value, "--debug"] + ) + + # Ensure AIRFLOW_API_APPS is not set initially + mock_environ.get.return_value = None + + # Call the fastapi_api command + fastapi_api_command.fastapi_api(args) + + # Assert that AIRFLOW_API_APPS was set in the environment before subprocess + mock_environ.__setitem__.assert_called_with("AIRFLOW_API_APPS", apps_value) + + # Simulate subprocess execution + Popen.assert_called_with( + [ + "fastapi", + "dev", + "airflow/api_fastapi/main.py", + "--port", + port, + "--host", + hostname, + ], + close_fds=True, + ) + + # Assert that AIRFLOW_API_APPS was unset after subprocess + mock_environ.pop.assert_called_with("AIRFLOW_API_APPS") + def test_cli_fastapi_api_args(self, ssl_cert_and_key): cert_path, key_path = ssl_cert_and_key @@ -139,6 +182,8 @@ def test_cli_fastapi_api_args(self, ssl_cert_and_key): str(cert_path), "--ssl-key", str(key_path), + "--apps", + "core", ] ) fastapi_api_command.fastapi_api(args) @@ -172,7 +217,7 @@ def test_cli_fastapi_api_args(self, ssl_cert_and_key): str(key_path), "--access-logformat", "custom_log_format", - "airflow.api_fastapi.app:cached_app()", + "airflow.api_fastapi.app:cached_app(apps='core')", "--preload", ], close_fds=True, From e8d12b82ba1aaa06ca0c03238b48b0554e9937f8 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Thu, 17 Oct 2024 16:47:53 +0530 Subject: [PATCH 165/199] Add ClientConnectorError to be a retryable error in databricks provider (#43091) closes #43080 --- .../databricks/hooks/databricks_base.py | 4 ++++ .../tests/databricks/hooks/test_databricks.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_base.py b/providers/src/airflow/providers/databricks/hooks/databricks_base.py index fd2e456642bdf..08a6eb8d40cbf 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_base.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_base.py @@ -34,6 +34,7 @@ import aiohttp import requests +from aiohttp.client_exceptions import ClientConnectorError from requests import PreparedRequest, exceptions as requests_exceptions from requests.auth import AuthBase, HTTPBasicAuth from requests.exceptions import JSONDecodeError @@ -678,6 +679,9 @@ def _retryable_error(exception: BaseException) -> bool: if exception.status >= 500 or exception.status == 429: return True + if isinstance(exception, ClientConnectorError): + return True + return False diff --git a/providers/tests/databricks/hooks/test_databricks.py b/providers/tests/databricks/hooks/test_databricks.py index 5d4dc3794af9e..bec238f70e39c 100644 --- a/providers/tests/databricks/hooks/test_databricks.py +++ b/providers/tests/databricks/hooks/test_databricks.py @@ -19,11 +19,13 @@ import itertools import json +import ssl import time from unittest import mock from unittest.mock import AsyncMock import aiohttp +import aiohttp.client_exceptions import azure.identity import azure.identity.aio import pytest @@ -1534,6 +1536,21 @@ async def test_init_async_session(self): assert isinstance(self.hook._session, aiohttp.ClientSession) assert self.hook._session is None + @pytest.mark.asyncio + @mock.patch("airflow.providers.databricks.hooks.databricks_base.aiohttp.ClientSession.get") + async def test_do_api_call_retries_with_client_connector_error(self, mock_get): + mock_get.side_effect = aiohttp.ClientConnectorError( + connection_key=None, + os_error=ssl.SSLError( + "SSL handshake is taking longer than 60.0 seconds: aborting the connection" + ), + ) + with mock.patch.object(self.hook.log, "error") as mock_errors: + async with self.hook: + with pytest.raises(AirflowException): + await self.hook._a_do_api_call(GET_RUN_ENDPOINT, {}) + assert mock_errors.call_count == DEFAULT_RETRY_NUMBER + @pytest.mark.asyncio @mock.patch("airflow.providers.databricks.hooks.databricks_base.aiohttp.ClientSession.get") async def test_do_api_call_retries_with_retryable_error(self, mock_get): From 46959defa5ffd3c5ac56e138a6acf5201bc4bae6 Mon Sep 17 00:00:00 2001 From: Bowrna Date: Thu, 17 Oct 2024 17:49:32 +0530 Subject: [PATCH 166/199] passing the filetype for SlackAPIFileOperator (#43069) --- providers/src/airflow/providers/slack/operators/slack.py | 1 + providers/tests/slack/operators/test_slack.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/providers/src/airflow/providers/slack/operators/slack.py b/providers/src/airflow/providers/slack/operators/slack.py index 883f8eb990888..64be693af67e4 100644 --- a/providers/src/airflow/providers/slack/operators/slack.py +++ b/providers/src/airflow/providers/slack/operators/slack.py @@ -265,6 +265,7 @@ def execute(self, context: Context): channels=self.channels, # For historical reason SlackAPIFileOperator use filename as reference to file file=self.filename, + filetype=self.filetype, content=self.content, initial_comment=self.initial_comment, title=self.title, diff --git a/providers/tests/slack/operators/test_slack.py b/providers/tests/slack/operators/test_slack.py index f276ee43159e6..f3b02839a3e55 100644 --- a/providers/tests/slack/operators/test_slack.py +++ b/providers/tests/slack/operators/test_slack.py @@ -253,6 +253,7 @@ def test_api_call_params_with_content_args(self, initial_comment, title, method_ channels="#test-channel", content="test-content", file=None, + filetype=None, initial_comment=initial_comment, title=title, ) @@ -282,6 +283,7 @@ def test_api_call_params_with_file_args(self, initial_comment, title, method_ver channels="C1234567890", content=None, file="/dev/null", + filetype=None, initial_comment=initial_comment, title=title, ) From ac1dbba9378f24d9fbd71903e529aca7f9e2dfa2 Mon Sep 17 00:00:00 2001 From: Sam Firke Date: Thu, 17 Oct 2024 08:43:50 -0400 Subject: [PATCH 167/199] chore(docs): add required import of BranchDayOfWeekOperator (#43053) --------- Co-authored-by: Shahar Epstein <60007259+shahar1@users.noreply.github.com> --- providers/src/airflow/providers/standard/operators/weekday.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/providers/src/airflow/providers/standard/operators/weekday.py b/providers/src/airflow/providers/standard/operators/weekday.py index f59a1da134888..0c74b19d795b4 100644 --- a/providers/src/airflow/providers/standard/operators/weekday.py +++ b/providers/src/airflow/providers/standard/operators/weekday.py @@ -39,6 +39,7 @@ class BranchDayOfWeekOperator(BaseBranchOperator): .. code-block:: python from airflow.operators.empty import EmptyOperator + from airflow.operators.weekday import BranchDayOfWeekOperator monday = EmptyOperator(task_id="monday") other_day = EmptyOperator(task_id="other_day") @@ -59,6 +60,7 @@ class BranchDayOfWeekOperator(BaseBranchOperator): # import WeekDay Enum from airflow.utils.weekday import WeekDay from airflow.operators.empty import EmptyOperator + from airflow.operators.weekday import BranchDayOfWeekOperator workday = EmptyOperator(task_id="workday") weekend = EmptyOperator(task_id="weekend") From 702135e65eb1dc9bca706616f925a2ca1e0388d9 Mon Sep 17 00:00:00 2001 From: yangyulely Date: Thu, 17 Oct 2024 21:49:25 +0800 Subject: [PATCH 168/199] Update unit_tests.rst links error (#43114) --- contributing-docs/testing/unit_tests.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index dc4e27f10d9fe..40f75663925e7 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -778,7 +778,7 @@ Running Unit tests Running Unit Tests from PyCharm IDE ................................... -To run unit tests from the PyCharm IDE, create the `local virtualenv <07_local_virtualenv.rst>`_, +To run unit tests from the PyCharm IDE, create the `local virtualenv <../07_local_virtualenv.rst>`_, select it as the default project's environment, then configure your test runner: .. image:: images/pycharm/configure_test_runner.png From f282eeab0a9a7f4460b5f976c998828cf01fce85 Mon Sep 17 00:00:00 2001 From: Eilon Goldberg Date: Thu, 17 Oct 2024 17:33:05 +0300 Subject: [PATCH 169/199] added MultipleFilesWebHdfsSensor (#43045) --- .../providers/apache/hdfs/sensors/web_hdfs.py | 32 ++++++++++++- .../apache/hdfs/sensors/test_web_hdfs.py | 48 ++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py b/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py index b09185325493e..d82da2a15697b 100644 --- a/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py +++ b/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py @@ -17,12 +17,14 @@ # under the License. from __future__ import annotations -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any, Sequence, Union from airflow.sensors.base import BaseSensorOperator if TYPE_CHECKING: from airflow.utils.context import Context + from hdfs.ext.kerberos import KerberosClient + from hdfs import InsecureClient class WebHdfsSensor(BaseSensorOperator): @@ -41,3 +43,31 @@ def poke(self, context: Context) -> bool: hook = WebHDFSHook(self.webhdfs_conn_id) self.log.info("Poking for file %s", self.filepath) return hook.check_for_path(hdfs_path=self.filepath) + + +class MultipleFilesWebHdfsSensor(BaseSensorOperator): + """Waits for multiple files in a folder to land in HDFS.""" + + template_fields: Sequence[str] = ("directory_path", "expected_filenames") + + def __init__(self, *, directory_path: str, expected_filenames: Sequence[str], + webhdfs_conn_id: str = "webhdfs_default", **kwargs: Any) -> None: + super().__init__(**kwargs) + self.directory_path = directory_path + self.expected_filenames = expected_filenames + self.webhdfs_conn_id = webhdfs_conn_id + + def poke(self, context: Context) -> bool: + from airflow.providers.apache.hdfs.hooks.webhdfs import WebHDFSHook + + hook = WebHDFSHook(self.webhdfs_conn_id) + conn: 'KerberosClient | InsecureClient' = hook.get_conn() + + actual_files = set(conn.list(self.directory_path)) + self.log.debug("Files Found in directory: %s", actual_files) + + missing_files = set(self.expected_filenames) - actual_files + if missing_files: + self.log.info("There are missing files: %s", missing_files) + return False + return True diff --git a/providers/tests/apache/hdfs/sensors/test_web_hdfs.py b/providers/tests/apache/hdfs/sensors/test_web_hdfs.py index ad1276814b7ab..4984140092214 100644 --- a/providers/tests/apache/hdfs/sensors/test_web_hdfs.py +++ b/providers/tests/apache/hdfs/sensors/test_web_hdfs.py @@ -17,12 +17,15 @@ # under the License. from __future__ import annotations +import os from unittest import mock -from airflow.providers.apache.hdfs.sensors.web_hdfs import WebHdfsSensor +from airflow.providers.apache.hdfs.sensors.web_hdfs import WebHdfsSensor, MultipleFilesWebHdfsSensor TEST_HDFS_CONN = "webhdfs_default" -TEST_HDFS_PATH = "hdfs://user/hive/warehouse/airflow.db/static_babynames" +TEST_HDFS_DIRECTORY = "hdfs://user/hive/warehouse/airflow.db" +TEST_HDFS_FILENAMES = ["static_babynames1", "static_babynames2", "static_babynames3"] +TEST_HDFS_PATH = os.path.join(TEST_HDFS_DIRECTORY, TEST_HDFS_FILENAMES[0]) class TestWebHdfsSensor: @@ -55,3 +58,44 @@ def test_poke_should_return_false_for_non_existing_table(self, mock_hook): mock_hook.return_value.check_for_path.assert_called_once_with(hdfs_path=TEST_HDFS_PATH) mock_hook.assert_called_once_with(TEST_HDFS_CONN) + + +class TestMultipleFilesWebHdfsSensor: + @mock.patch("airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook") + def test_poke(self, mock_hook, caplog): + mock_hook.return_value.get_conn.return_value.list.return_value = TEST_HDFS_FILENAMES + + sensor = MultipleFilesWebHdfsSensor( + task_id="test_task", + webhdfs_conn_id=TEST_HDFS_CONN, + directory_path=TEST_HDFS_DIRECTORY, + expected_filenames=TEST_HDFS_FILENAMES + ) + result = sensor.poke(dict()) + + assert result + assert "Files Found in directory: " in caplog.text + + mock_hook.return_value.get_conn.return_value.list.assert_called_once_with(TEST_HDFS_DIRECTORY) + mock_hook.return_value.get_conn.assert_called_once() + mock_hook.assert_called_once_with(TEST_HDFS_CONN) + + @mock.patch("airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook") + def test_poke_should_return_false_for_missing_file(self, mock_hook, caplog): + mock_hook.return_value.get_conn.return_value.list.return_value = TEST_HDFS_FILENAMES[0] + + sensor = MultipleFilesWebHdfsSensor( + task_id="test_task", + webhdfs_conn_id=TEST_HDFS_CONN, + directory_path=TEST_HDFS_DIRECTORY, + expected_filenames=TEST_HDFS_FILENAMES + ) + exists = sensor.poke(dict()) + + assert not exists + assert "Files Found in directory: " in caplog.text + assert "There are missing files: " in caplog.text + + mock_hook.return_value.get_conn.return_value.list.assert_called_once_with(TEST_HDFS_DIRECTORY) + mock_hook.return_value.get_conn.assert_called_once() + mock_hook.assert_called_once_with(TEST_HDFS_CONN) From 0de7b102cc7c6fd6ca73097a0ee4349f5529896c Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 17 Oct 2024 16:30:04 +0100 Subject: [PATCH 170/199] Bump to `0.4.23` to fix a uv bug (#43121) Now that https://github.com/astral-sh/uv/issues/8236 is fixed (a bug in uv 0.4.22), we can remove `--no-sources` (used in `uv pip install`) introduced in https://github.com/apache/airflow/pull/43056 --- Dockerfile | 2 +- Dockerfile.ci | 4 ++-- dev/breeze/doc/ci/02_images.md | 2 +- .../airflow_breeze/commands/release_management_commands.py | 2 +- scripts/in_container/install_airflow_and_providers.py | 2 -- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 129de7a3f61ff..4e3aa5ac9009a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.10.2" ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.22 +ARG AIRFLOW_UV_VERSION=0.4.23 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index cab5c91e2016d..86268bf9209c3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1278,7 +1278,7 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.22 +ARG AIRFLOW_UV_VERSION=0.4.23 ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller @@ -1302,7 +1302,7 @@ ARG AIRFLOW_VERSION="" ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.4.22 +ARG AIRFLOW_UV_VERSION=0.4.23 ARG AIRFLOW_USE_UV="true" ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index c83adcdc03e14..982f6ac1c200f 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -448,7 +448,7 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.0` | PIP version used. | -| `AIRFLOW_UV_VERSION` | `0.4.22` | UV version used. | +| `AIRFLOW_UV_VERSION` | `0.4.23` | UV version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index d42b3b7e95cb9..d5470f5a87881 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -230,7 +230,7 @@ class VersionedFile(NamedTuple): AIRFLOW_PIP_VERSION = "24.0" -AIRFLOW_UV_VERSION = "0.4.22" +AIRFLOW_UV_VERSION = "0.4.23" AIRFLOW_USE_UV = False WHEEL_VERSION = "0.36.2" GITPYTHON_VERSION = "3.1.40" diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index 909eb98c2af96..bc28155875848 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -484,7 +484,6 @@ def install_airflow_and_providers( "/usr/local/bin/uv", "pip", "install", - "--no-sources", "--python", "/usr/local/bin/python", installation_spec.airflow_package, @@ -507,7 +506,6 @@ def install_airflow_and_providers( "/usr/local/bin/uv", "pip", "install", - "--no-sources", "--python", "/usr/local/bin/python", ] From 90bbe1365b7fd430f29c130fe5937f3a190832c0 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Thu, 17 Oct 2024 17:02:43 +0100 Subject: [PATCH 171/199] Clean up some eslint rules to better fit what we do (#43093) --- airflow/ui/rules/perfectionist.js | 42 +++++++++---------- airflow/ui/rules/typescript.js | 19 ++------- airflow/ui/rules/unicorn.js | 38 ++++++++--------- .../ui/src/pages/Dashboard/HealthSection.tsx | 4 +- airflow/ui/src/pages/Dashboard/HealthTag.tsx | 4 +- airflow/ui/src/theme.ts | 2 - airflow/ui/src/utils/capitalize.ts | 1 - 7 files changed, 45 insertions(+), 65 deletions(-) diff --git a/airflow/ui/rules/perfectionist.js b/airflow/ui/rules/perfectionist.js index fc838f7fdc19a..40d48cc0a07d1 100644 --- a/airflow/ui/rules/perfectionist.js +++ b/airflow/ui/rules/perfectionist.js @@ -48,13 +48,6 @@ export const perfectionistRules = */ [`${perfectionistNamespace}/sort-array-includes`]: ERROR, - /** - * Enforce sorted attributes in Astro elements. - * - * @see [perfectionist/sort-astro-attributes](https://perfectionist.dev/rules/sort-astro-attributes) - */ - [`${perfectionistNamespace}/sort-astro-attributes`]: ERROR, - /** * Enforce sorted class members. * @@ -109,14 +102,12 @@ export const perfectionistRules = * * @see [perfectionist/sort-objects](https://perfectionist.dev/rules/sort-objects) */ - [`${perfectionistNamespace}/sort-objects`]: ERROR, - - /** - * Enforce sorted attributes in Svelte elements. - * - * @see [perfectionist/sort-svelte-attributes](https://perfectionist.dev/rules/sort-svelte-attributes) - */ - [`${perfectionistNamespace}/sort-svelte-attributes`]: ERROR, + [`${perfectionistNamespace}/sort-objects`]: [ + ERROR, + { + type: "natural", + }, + ], /** * Enforce sorted switch case statements. @@ -128,16 +119,21 @@ export const perfectionistRules = /** * Enforce sorted TypeScript union types. * - * @see [perfectionist/sort-union-types](https://perfectionist.dev/rules/sort-union-types) - */ - [`${perfectionistNamespace}/sort-union-types`]: ERROR, - - /** - * Enforce sorted attributes in Vue elements. + * nullish groups will leave null and undefined at the end * - * @see [perfectionist/sort-vue-attributes](https://perfectionist.dev/rules/sort-vue-attributes) + * @see [perfectionist/sort-union-types](https://perfectionist.dev/rules/sort-union-types) */ - [`${perfectionistNamespace}/sort-vue-attributes`]: ERROR, + [`${perfectionistNamespace}/sort-union-types`]: [ + ERROR, + { + groups: [ + ["conditional", "function", "import"], + ["intersection", "keyword", "named"], + ["literal", "object", "operator", "tuple", "union"], + "nullish", + ], + }, + ], ...off("sort-keys"), }, diff --git a/airflow/ui/rules/typescript.js b/airflow/ui/rules/typescript.js index c1964b0c9cb96..2856e89b5a7f1 100644 --- a/airflow/ui/rules/typescript.js +++ b/airflow/ui/rules/typescript.js @@ -1,3 +1,5 @@ +/* eslint-disable max-lines */ + /*! * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -1818,21 +1820,6 @@ export const typescriptRules = */ [`${typescriptNamespace}/triple-slash-reference`]: ERROR, - /** - * Bind stuff before using it. - * - * @example - * ```typescript - * // ❌ Incorrect - * const foo = classInstance.method; - * - * // ✅ Correct - * const foo = classInstance.method.bind(classInstance); - * ``` - * @see [@typescript-eslint/unbound-method](https://typescript-eslint.io/rules/unbound-method/) - */ - [`${typescriptNamespace}/unbound-method`]: ERROR, - /** * Unify signatures instead of overloading. * @@ -1881,7 +1868,7 @@ export const typescriptRules = "camelcase", "class-methods-use-this", "consistent-return", - "constructor-super", // eslint-disable-line max-lines + "constructor-super", "default-param-last", "dot-notation", "getter-return", diff --git a/airflow/ui/rules/unicorn.js b/airflow/ui/rules/unicorn.js index a11445c28586a..a53941d053c23 100644 --- a/airflow/ui/rules/unicorn.js +++ b/airflow/ui/rules/unicorn.js @@ -654,25 +654,6 @@ export const unicornRules = */ [`${unicornNamespace}/prefer-array-find`]: ERROR, - /** - * Prefer `Array#flatMap` over `Array#map(…).flat()`. - * - * @example - * ```typescript - * // ❌ Incorrect - * const foo = bar.map(unicorn).flat(); - * const foo = bar.map(unicorn).flat(1); - * - * // ✅ Correct - * const foo = bar.flatMap(unicorn); - * const foo = bar.map(unicorn).flat(2); - * const foo = bar.map(unicorn).foo().flat(); - * const foo = bar.flat().map(unicorn); - * ``` - * @see [unicorn/prefer-array-flat-map](https://github.com/sindresorhus/eslint-plugin-unicorn/blob/main/docs/rules/prefer-array-flat-map.md) - */ - [`${unicornNamespace}/prefer-array-flat-map`]: ERROR, - /** * Prefer `Array#flat` over legacy techniques to flatten arrays. * @@ -700,6 +681,25 @@ export const unicornRules = */ [`${unicornNamespace}/prefer-array-flat`]: ERROR, + /** + * Prefer `Array#flatMap` over `Array#map(…).flat()`. + * + * @example + * ```typescript + * // ❌ Incorrect + * const foo = bar.map(unicorn).flat(); + * const foo = bar.map(unicorn).flat(1); + * + * // ✅ Correct + * const foo = bar.flatMap(unicorn); + * const foo = bar.map(unicorn).flat(2); + * const foo = bar.map(unicorn).foo().flat(); + * const foo = bar.flat().map(unicorn); + * ``` + * @see [unicorn/prefer-array-flat-map](https://github.com/sindresorhus/eslint-plugin-unicorn/blob/main/docs/rules/prefer-array-flat-map.md) + */ + [`${unicornNamespace}/prefer-array-flat-map`]: ERROR, + /** * Prefer `Array#indexOf` and `Array#lastIndexOf` over `Array#findIndex` and `Array#findLastIndex` when looking for the index of an item. * diff --git a/airflow/ui/src/pages/Dashboard/HealthSection.tsx b/airflow/ui/src/pages/Dashboard/HealthSection.tsx index 85e545c6341d2..70c0468366c1e 100644 --- a/airflow/ui/src/pages/Dashboard/HealthSection.tsx +++ b/airflow/ui/src/pages/Dashboard/HealthSection.tsx @@ -25,8 +25,8 @@ export const HealthSection = ({ title, }: { readonly isLoading: boolean; - readonly latestHeartbeat?: null | string; - readonly status?: null | string; + readonly latestHeartbeat?: string | null; + readonly status?: string | null; readonly title: string; }) => { if (isLoading) { diff --git a/airflow/ui/src/pages/Dashboard/HealthTag.tsx b/airflow/ui/src/pages/Dashboard/HealthTag.tsx index 335d2f91216b9..7318f077da1d1 100644 --- a/airflow/ui/src/pages/Dashboard/HealthTag.tsx +++ b/airflow/ui/src/pages/Dashboard/HealthTag.tsx @@ -27,8 +27,8 @@ export const HealthTag = ({ title, }: { readonly isLoading: boolean; - readonly latestHeartbeat?: null | string; - readonly status?: null | string; + readonly latestHeartbeat?: string | null; + readonly status?: string | null; readonly title: string; }) => { if (isLoading) { diff --git a/airflow/ui/src/theme.ts b/airflow/ui/src/theme.ts index 0c7146643d55c..b348c54aed821 100644 --- a/airflow/ui/src/theme.ts +++ b/airflow/ui/src/theme.ts @@ -19,8 +19,6 @@ import { tableAnatomy } from "@chakra-ui/anatomy"; import { createMultiStyleConfigHelpers, extendTheme } from "@chakra-ui/react"; -// Chakra has bad types for this util, so is registered as unbound -// eslint-disable-next-line @typescript-eslint/unbound-method const { defineMultiStyleConfig, definePartsStyle } = createMultiStyleConfigHelpers(tableAnatomy.keys); diff --git a/airflow/ui/src/utils/capitalize.ts b/airflow/ui/src/utils/capitalize.ts index bd941c4667724..0b48e3b65403d 100644 --- a/airflow/ui/src/utils/capitalize.ts +++ b/airflow/ui/src/utils/capitalize.ts @@ -17,7 +17,6 @@ * under the License. */ -// eslint-disable-next-line perfectionist/sort-union-types export const capitalize = (string: string | null | undefined) => // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions string ? string.charAt(0).toUpperCase() + string.slice(1).toLowerCase() : ""; From 89cc08ba1bd2bd8f0ba55dcb177498b06194587d Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 17 Oct 2024 17:19:20 +0100 Subject: [PATCH 172/199] Fixed failing static checks & provider tests (#43122) --- .../providers/apache/hdfs/sensors/web_hdfs.py | 19 +++++++++++++------ .../apache/hdfs/sensors/test_web_hdfs.py | 11 ++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py b/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py index d82da2a15697b..14f622e6efedc 100644 --- a/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py +++ b/providers/src/airflow/providers/apache/hdfs/sensors/web_hdfs.py @@ -17,14 +17,15 @@ # under the License. from __future__ import annotations -from typing import TYPE_CHECKING, Any, Sequence, Union +from typing import TYPE_CHECKING, Any, Sequence from airflow.sensors.base import BaseSensorOperator if TYPE_CHECKING: - from airflow.utils.context import Context - from hdfs.ext.kerberos import KerberosClient from hdfs import InsecureClient + from hdfs.ext.kerberos import KerberosClient + + from airflow.utils.context import Context class WebHdfsSensor(BaseSensorOperator): @@ -50,8 +51,14 @@ class MultipleFilesWebHdfsSensor(BaseSensorOperator): template_fields: Sequence[str] = ("directory_path", "expected_filenames") - def __init__(self, *, directory_path: str, expected_filenames: Sequence[str], - webhdfs_conn_id: str = "webhdfs_default", **kwargs: Any) -> None: + def __init__( + self, + *, + directory_path: str, + expected_filenames: Sequence[str], + webhdfs_conn_id: str = "webhdfs_default", + **kwargs: Any, + ) -> None: super().__init__(**kwargs) self.directory_path = directory_path self.expected_filenames = expected_filenames @@ -61,7 +68,7 @@ def poke(self, context: Context) -> bool: from airflow.providers.apache.hdfs.hooks.webhdfs import WebHDFSHook hook = WebHDFSHook(self.webhdfs_conn_id) - conn: 'KerberosClient | InsecureClient' = hook.get_conn() + conn: KerberosClient | InsecureClient = hook.get_conn() actual_files = set(conn.list(self.directory_path)) self.log.debug("Files Found in directory: %s", actual_files) diff --git a/providers/tests/apache/hdfs/sensors/test_web_hdfs.py b/providers/tests/apache/hdfs/sensors/test_web_hdfs.py index 4984140092214..41ccb88e28e18 100644 --- a/providers/tests/apache/hdfs/sensors/test_web_hdfs.py +++ b/providers/tests/apache/hdfs/sensors/test_web_hdfs.py @@ -20,7 +20,7 @@ import os from unittest import mock -from airflow.providers.apache.hdfs.sensors.web_hdfs import WebHdfsSensor, MultipleFilesWebHdfsSensor +from airflow.providers.apache.hdfs.sensors.web_hdfs import MultipleFilesWebHdfsSensor, WebHdfsSensor TEST_HDFS_CONN = "webhdfs_default" TEST_HDFS_DIRECTORY = "hdfs://user/hive/warehouse/airflow.db" @@ -69,9 +69,11 @@ def test_poke(self, mock_hook, caplog): task_id="test_task", webhdfs_conn_id=TEST_HDFS_CONN, directory_path=TEST_HDFS_DIRECTORY, - expected_filenames=TEST_HDFS_FILENAMES + expected_filenames=TEST_HDFS_FILENAMES, ) - result = sensor.poke(dict()) + + with caplog.at_level("DEBUG", logger="airflow.task"): + result = sensor.poke(dict()) assert result assert "Files Found in directory: " in caplog.text @@ -88,12 +90,11 @@ def test_poke_should_return_false_for_missing_file(self, mock_hook, caplog): task_id="test_task", webhdfs_conn_id=TEST_HDFS_CONN, directory_path=TEST_HDFS_DIRECTORY, - expected_filenames=TEST_HDFS_FILENAMES + expected_filenames=TEST_HDFS_FILENAMES, ) exists = sensor.poke(dict()) assert not exists - assert "Files Found in directory: " in caplog.text assert "There are missing files: " in caplog.text mock_hook.return_value.get_conn.return_value.list.assert_called_once_with(TEST_HDFS_DIRECTORY) From 2d15c16b85711206067485ce19e9dce03aebad82 Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Thu, 17 Oct 2024 22:27:29 +0530 Subject: [PATCH 173/199] Add min version to python-ldap (#43104) * add min version to python-ldap * add min version to python-ldap --- hatch_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 6dc0e1c9c7c50..769e610bcdec9 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -98,7 +98,7 @@ "thrift-sasl>=0.2.0", ], "ldap": [ - "python-ldap", + "python-ldap>=3.4.4", ], "leveldb": [ "plyvel", From 40670b687d114ce987938036d182c84e537388df Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 18 Oct 2024 01:03:28 +0800 Subject: [PATCH 174/199] feat(providers/fab): Use asset in common provider (#43112) --- dev/breeze/tests/test_packages.py | 6 +++--- generated/provider_dependencies.json | 2 +- .../providers/fab/auth_manager/fab_auth_manager.py | 5 +---- .../fab/auth_manager/security_manager/override.py | 5 +---- providers/src/airflow/providers/fab/provider.yaml | 2 +- .../tests/fab/auth_manager/test_fab_auth_manager.py | 9 +-------- providers/tests/fab/auth_manager/test_security.py | 5 +---- 7 files changed, 9 insertions(+), 25 deletions(-) diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index e828f24cf1bb0..f0c470129b4a2 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -167,7 +167,7 @@ def test_get_documentation_package_path(): "fab", "", """ - "apache-airflow-providers-common-compat>=1.2.0", + "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow>=2.9.0", "flask-appbuilder==4.5.0", "flask-login>=0.6.2", @@ -181,7 +181,7 @@ def test_get_documentation_package_path(): "fab", "dev0", """ - "apache-airflow-providers-common-compat>=1.2.0.dev0", + "apache-airflow-providers-common-compat>=1.2.1.dev0", "apache-airflow>=2.9.0.dev0", "flask-appbuilder==4.5.0", "flask-login>=0.6.2", @@ -195,7 +195,7 @@ def test_get_documentation_package_path(): "fab", "beta0", """ - "apache-airflow-providers-common-compat>=1.2.0b0", + "apache-airflow-providers-common-compat>=1.2.1b0", "apache-airflow>=2.9.0b0", "flask-appbuilder==4.5.0", "flask-login>=0.6.2", diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index fb50939ac7fb8..3c4363864c309 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -569,7 +569,7 @@ }, "fab": { "deps": [ - "apache-airflow-providers-common-compat>=1.2.0", + "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow>=2.9.0", "flask-appbuilder==4.5.0", "flask-login>=0.6.2", diff --git a/providers/src/airflow/providers/fab/auth_manager/fab_auth_manager.py b/providers/src/airflow/providers/fab/auth_manager/fab_auth_manager.py index 425f2d6d2124f..5874c570b6fbd 100644 --- a/providers/src/airflow/providers/fab/auth_manager/fab_auth_manager.py +++ b/providers/src/airflow/providers/fab/auth_manager/fab_auth_manager.py @@ -96,10 +96,7 @@ from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride from airflow.security.permissions import RESOURCE_ASSET else: - try: - from airflow.security.permissions import RESOURCE_ASSET - except ImportError: - from airflow.security.permissions import RESOURCE_DATASET as RESOURCE_ASSET + from airflow.providers.common.compat.security.permissions import RESOURCE_ASSET _MAP_DAG_ACCESS_ENTITY_TO_FAB_RESOURCE_TYPE: dict[DagAccessEntity, tuple[str, ...]] = { diff --git a/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py b/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py index 640d406f81942..868c65deca5cf 100644 --- a/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py +++ b/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py @@ -117,10 +117,7 @@ from airflow.auth.managers.base_auth_manager import ResourceMethod from airflow.security.permissions import RESOURCE_ASSET else: - try: - from airflow.security.permissions import RESOURCE_ASSET - except ImportError: - from airflow.security.permissions import RESOURCE_DATASET as RESOURCE_ASSET + from airflow.providers.common.compat.security.permissions import RESOURCE_ASSET log = logging.getLogger(__name__) diff --git a/providers/src/airflow/providers/fab/provider.yaml b/providers/src/airflow/providers/fab/provider.yaml index c5e6f76cccf26..4b72c0ab9a5ee 100644 --- a/providers/src/airflow/providers/fab/provider.yaml +++ b/providers/src/airflow/providers/fab/provider.yaml @@ -48,7 +48,7 @@ versions: dependencies: - apache-airflow>=2.9.0 - - apache-airflow-providers-common-compat>=1.2.0 + - apache-airflow-providers-common-compat>=1.2.1 - flask>=2.2,<2.3 # We are tightly coupled with FAB version as we vendored-in part of FAB code related to security manager # This is done as part of preparation to removing FAB as dependency, but we are not ready for it yet diff --git a/providers/tests/fab/auth_manager/test_fab_auth_manager.py b/providers/tests/fab/auth_manager/test_fab_auth_manager.py index 0d111f0453ad4..91efb8428c654 100644 --- a/providers/tests/fab/auth_manager/test_fab_auth_manager.py +++ b/providers/tests/fab/auth_manager/test_fab_auth_manager.py @@ -38,6 +38,7 @@ from airflow.providers.fab.auth_manager.models import User from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride +from airflow.providers.common.compat.security.permissions import RESOURCE_ASSET from airflow.security.permissions import ( ACTION_CAN_ACCESS_MENU, ACTION_CAN_CREATE, @@ -61,14 +62,6 @@ if TYPE_CHECKING: from airflow.auth.managers.base_auth_manager import ResourceMethod - from airflow.security.permissions import RESOURCE_ASSET -else: - try: - from airflow.security.permissions import RESOURCE_ASSET - except ImportError: - from airflow.security.permissions import ( - RESOURCE_DATASET as RESOURCE_ASSET, - ) IS_AUTHORIZED_METHODS_SIMPLE = { diff --git a/providers/tests/fab/auth_manager/test_security.py b/providers/tests/fab/auth_manager/test_security.py index 632ea5cb1f4b2..1db881eaa7018 100644 --- a/providers/tests/fab/auth_manager/test_security.py +++ b/providers/tests/fab/auth_manager/test_security.py @@ -66,10 +66,7 @@ if TYPE_CHECKING: from airflow.security.permissions import RESOURCE_ASSET else: - try: - from airflow.security.permissions import RESOURCE_ASSET - except ImportError: - from airflow.security.permissions import RESOURCE_DATASET as RESOURCE_ASSET + from airflow.providers.common.compat.security.permissions import RESOURCE_ASSET pytestmark = pytest.mark.db_test From 17cf4081858b85626f637e7351deaafb00e64635 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 17 Oct 2024 18:20:10 +0100 Subject: [PATCH 175/199] Add `bugraoz93` in the triage team (#43124) --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index 7685192c06351..ce0d702834bfa 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -129,6 +129,7 @@ github: - vatsrahul1001 - cmarteepants - gopidesupavan + - bugraoz93 notifications: jobs: jobs@airflow.apache.org From 64b1d1b0b4ef74b21d7da5313570529bc9e7e641 Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 15 Aug 2024 17:25:16 -0600 Subject: [PATCH 176/199] add on_kill override to databricks workflow operator --- .../tests/databricks/operators/test_databricks_workflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/providers/tests/databricks/operators/test_databricks_workflow.py b/providers/tests/databricks/operators/test_databricks_workflow.py index fbc429ed1d9a8..019f996ab3a19 100644 --- a/providers/tests/databricks/operators/test_databricks_workflow.py +++ b/providers/tests/databricks/operators/test_databricks_workflow.py @@ -31,6 +31,7 @@ WorkflowRunMetadata, _CreateDatabricksWorkflowOperator, _flatten_node, + WorkflowRunMetadata, ) from airflow.utils import timezone @@ -59,6 +60,10 @@ def mock_task_group(): mock_group.group_id = "test_group" return mock_group +@pytest.fixture +def mock_workflow_run_metadata(): + return MagicMock(spec=WorkflowRunMetadata) + @pytest.fixture def mock_workflow_run_metadata(): From cafc23aeffb605f5037b50e544674ea76d9b9c49 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 30 Aug 2024 18:23:34 -0600 Subject: [PATCH 177/199] on_kill equivalent for DatabricksSqlOperator --- .../databricks/hooks/databricks_sql.py | 31 ++-- .../databricks/hooks/test_databricks_sql.py | 141 ++++-------------- 2 files changed, 46 insertions(+), 126 deletions(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 61975045ac61d..d3fd98c19c769 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -21,7 +21,7 @@ from collections import namedtuple from contextlib import closing from copy import copy -from datetime import timedelta +from datetime import timedelta, datetime from typing import ( TYPE_CHECKING, Any, @@ -37,12 +37,8 @@ from databricks import sql # type: ignore[attr-defined] -from airflow.exceptions import ( - AirflowException, - AirflowProviderDeprecationWarning, -) +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results -from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook if TYPE_CHECKING: @@ -196,11 +192,11 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., - execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: None = ..., split_statements: bool = ..., return_last: bool = ..., + execution_timeout: timedelta | None = ..., ) -> None: ... @overload @@ -208,11 +204,11 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., - execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: Callable[[Any], T] = ..., split_statements: bool = ..., return_last: bool = ..., + execution_timeout: timedelta | None = ..., ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: ... def run( @@ -241,8 +237,8 @@ def run( :param handler: The result handler which is called with the result of each statement. :param split_statements: Whether to split a single SQL string into statements and run separately :param return_last: Whether to return result for only last statement or for all after split - :param execution_timeout: max time allowed for the execution of this task instance, if it goes beyond - it will raise and fail. + :param execution_timeout: max time allowed for the execution of + this task instance, if it goes beyond it will raise and fail. :return: return only result of the LAST SQL expression if handler was provided unless return_last is set to False. """ @@ -263,25 +259,26 @@ def run( conn = None results = [] for sql_statement in sql_list: + # when using AAD tokens, it could expire if previous query run longer than token lifetime conn = self.get_conn() with closing(conn.cursor()) as cur: self.set_autocommit(conn, autocommit) with closing(conn.cursor()) as cur: + t = create_timeout_thread(cur, execution_timeout) - # TODO: adjust this to make testing easier + #todo; adjust this to make testing easier try: self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: - if t is None or t.is_alive(): - raise DatabricksSqlExecutionError( - f"Error running SQL statement: {sql_statement}. {str(e)}" + if t.is_alive(): + raise AirflowException(f"Error running statement: {sql_statement}. {str(e)}") + else: + raise AirflowException( + f"Timeout threshold exceeded for query: {sql_statement} was cancelled." ) - raise DatabricksSqlExecutionTimeout( - f"Timeout threshold exceeded for SQL statement: {sql_statement} was cancelled." - ) finally: if t is not None: t.cancel() diff --git a/providers/tests/databricks/hooks/test_databricks_sql.py b/providers/tests/databricks/hooks/test_databricks_sql.py index 824be94421d72..f87abfe5d4320 100644 --- a/providers/tests/databricks/hooks/test_databricks_sql.py +++ b/providers/tests/databricks/hooks/test_databricks_sql.py @@ -19,10 +19,11 @@ from __future__ import annotations import threading +import time from collections import namedtuple from datetime import timedelta from unittest import mock -from unittest.mock import PropertyMock, patch +from unittest.mock import patch, PropertyMock import pytest from databricks.sql.types import Row @@ -30,8 +31,9 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import fetch_all_handler -from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook, create_timeout_thread +from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook from airflow.utils.session import provide_session +from airflow.utils.timezone import datetime pytestmark = pytest.mark.db_test @@ -57,7 +59,6 @@ def create_connection(session): def databricks_hook(): return DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=True) - @pytest.fixture def mock_get_conn(): # Start the patcher @@ -99,16 +100,9 @@ def mock_get_requests(): mock_patch.stop() -@pytest.fixture -def mock_timer(): - with patch("threading.Timer") as mock_timer: - yield mock_timer - - def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: return [(field,) for field in fields] - # Serializable Row object similar to the one returned by the Hook SerializableRow = namedtuple("Row", ["id", "value"]) # type: ignore[name-match] @@ -308,40 +302,7 @@ def test_query( mock_get_conn, mock_get_requests, ): -<<<<<<< HEAD:tests/providers/databricks/hooks/test_databricks_sql.py - connections = [] - cursors = [] - for index in range(len(cursor_descriptions)): - conn = mock.MagicMock() - cur = mock.MagicMock( - rowcount=len(cursor_results[index]), - description=get_cursor_descriptions(cursor_descriptions[index]), - ) - cur.fetchall.return_value = cursor_results[index] - conn.cursor.return_value = cur - cursors.append(cur) - connections.append(conn) - mock_get_conn.side_effect = connections -======= - with ( - patch("airflow.providers.databricks.hooks.databricks_sql.DatabricksSqlHook.get_conn") as mock_conn, - patch("airflow.providers.databricks.hooks.databricks_base.requests") as mock_requests, - ): - mock_requests.codes.ok = 200 - mock_requests.get.return_value.json.return_value = { - "endpoints": [ - { - "id": "1264e5078741679a", - "name": "Test", - "odbc_params": { - "hostname": "xx.cloud.databricks.com", - "path": "/sql/1.0/endpoints/1264e5078741679a", - }, - } - ] - } - status_code_mock = mock.PropertyMock(return_value=200) - type(mock_requests.get.return_value).status_code = status_code_mock + connections = [] cursors = [] for index in range(len(cursor_descriptions)): @@ -354,30 +315,29 @@ def test_query( conn.cursor.return_value = cur cursors.append(cur) connections.append(conn) - mock_conn.side_effect = connections ->>>>>>> 857ca4c06c (Split providers out of the main "airflow/" tree into a UV workspace project (#42505)):providers/tests/databricks/hooks/test_databricks_sql.py + mock_get_conn.side_effect = connections - if not return_tuple: - with pytest.warns( - AirflowProviderDeprecationWarning, - match="""Returning a raw `databricks.sql.Row` object is deprecated. A namedtuple will be + if not return_tuple: + with pytest.warns( + AirflowProviderDeprecationWarning, + match="""Returning a raw `databricks.sql.Row` object is deprecated. A namedtuple will be returned instead in a future release of the databricks provider. Set `return_tuple=True` to enable this behavior.""", - ): + ): + databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) + else: databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) - else: - databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) - results = databricks_hook.run( - sql=sql, handler=fetch_all_handler, return_last=return_last, split_statements=split_statements - ) + results = databricks_hook.run( + sql=sql, handler=fetch_all_handler, return_last=return_last, split_statements=split_statements + ) - assert databricks_hook.descriptions == hook_descriptions - assert databricks_hook.last_description == hook_descriptions[-1] - assert results == hook_results + assert databricks_hook.descriptions == hook_descriptions + assert databricks_hook.last_description == hook_descriptions[-1] + assert results == hook_results - for index, cur in enumerate(cursors): - cur.execute.assert_has_calls([mock.call(cursor_calls[index])]) - cur.close.assert_called() + for index, cur in enumerate(cursors): + cur.execute.assert_has_calls([mock.call(cursor_calls[index])]) + cur.close.assert_called() @pytest.mark.parametrize( @@ -418,18 +378,14 @@ def test_execution_timeout_exceeded( mock_get_requests, sql="select * from test.test", execution_timeout=timedelta(microseconds=0), - cursor_descriptions=( - "id", - "value", - ), - cursor_results=( - Row(id=1, value=2), - Row(id=11, value=12), - ), + cursor_descriptions=["id", "value"], + cursor_results=[Row(id=1, value=2), Row(id=11, value=12)], ): with patch( - "airflow.providers.databricks.hooks.databricks_sql.create_timeout_thread" - ) as mock_create_timeout_thread, patch.object(DatabricksSqlHook, "_run_command") as mock_run_command: + 'airflow.providers.databricks.hooks.databricks_sql.create_timeout_thread' + ) as mock_create_timeout_thread, patch.object( + DatabricksSqlHook, '_run_command' + ) as mock_run_command: conn = mock.MagicMock() cur = mock.MagicMock( rowcount=len(cursor_results), @@ -437,9 +393,11 @@ def test_execution_timeout_exceeded( ) # Simulate a timeout - mock_create_timeout_thread.return_value = threading.Timer(cur, execution_timeout) + mock_create_timeout_thread.return_value = threading.Timer( + cur, execution_timeout + ) - mock_run_command.side_effect = Exception("Mocked exception") + mock_run_command.side_effect = Exception('Mocked exception') cur.fetchall.return_value = cursor_results conn.cursor.return_value = cur @@ -455,38 +413,3 @@ def test_execution_timeout_exceeded( assert "Timeout threshold exceeded" in str(exc_info.value) -def test_create_timeout_thread( - mock_get_conn, - mock_get_requests, - mock_timer, - cursor_descriptions=( - "id", - "value", - ), -): - cur = mock.MagicMock( - rowcount=1, - description=get_cursor_descriptions(cursor_descriptions), - ) - timeout = timedelta(seconds=1) - thread = create_timeout_thread(cur=cur, execution_timeout=timeout) - mock_timer.assert_called_once_with(timeout.total_seconds(), cur.connection.cancel) - assert thread is not None - - -def test_create_timeout_thread_no_timeout( - mock_get_conn, - mock_get_requests, - mock_timer, - cursor_descriptions=( - "id", - "value", - ), -): - cur = mock.MagicMock( - rowcount=1, - description=get_cursor_descriptions(cursor_descriptions), - ) - thread = create_timeout_thread(cur=cur, execution_timeout=None) - mock_timer.assert_not_called() - assert thread is None From 0cab9ab8a226251c19a4478a346a8a29872c0052 Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 5 Sep 2024 10:50:34 -0600 Subject: [PATCH 178/199] add tests for create_timeout_thread --- .../databricks/hooks/test_databricks_sql.py | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/providers/tests/databricks/hooks/test_databricks_sql.py b/providers/tests/databricks/hooks/test_databricks_sql.py index f87abfe5d4320..0467ad41eb58c 100644 --- a/providers/tests/databricks/hooks/test_databricks_sql.py +++ b/providers/tests/databricks/hooks/test_databricks_sql.py @@ -31,7 +31,7 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import fetch_all_handler -from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook +from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook, create_timeout_thread from airflow.utils.session import provide_session from airflow.utils.timezone import datetime @@ -99,6 +99,10 @@ def mock_get_requests(): # Stop the patcher after the test mock_patch.stop() +@pytest.fixture +def mock_timer(): + with patch("threading.Timer") as mock_timer: + yield mock_timer def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: return [(field,) for field in fields] @@ -413,3 +417,32 @@ def test_execution_timeout_exceeded( assert "Timeout threshold exceeded" in str(exc_info.value) +def test_create_timeout_thread( + mock_get_conn, + mock_get_requests, + mock_timer, + cursor_descriptions=["id", "value"], +): + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + timeout = timedelta(seconds=1) + thread = create_timeout_thread(cur=cur, execution_timeout=timeout) + mock_timer.assert_called_once_with(timeout.total_seconds(), cur.connection.cancel) + assert thread is not None + + +def test_create_timeout_thread_no_timeout( + mock_get_conn, + mock_get_requests, + mock_timer, + cursor_descriptions=["id", "value"], +): + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + thread = create_timeout_thread(cur=cur, execution_timeout=None) + mock_timer.assert_not_called() + assert thread is None From c3e7ecbda7045a7ae3e77430d8ced9f8a9dc1a03 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 6 Sep 2024 10:23:54 -0600 Subject: [PATCH 179/199] add note for on_kill in DatabricksCopyIntoOperator --- .../airflow/providers/databricks/operators/databricks_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/databricks/operators/databricks_sql.py b/providers/src/airflow/providers/databricks/operators/databricks_sql.py index 62e55c682de04..a3851f05e0408 100644 --- a/providers/src/airflow/providers/databricks/operators/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/operators/databricks_sql.py @@ -355,6 +355,6 @@ def execute(self, context: Context) -> Any: hook.run(sql) def on_kill(self) -> None: - # NB: on_kill isn't required for this operator since query cancelling gets + #NB: on_kill isn't required for this operator since query cancelling gets # handled in `DatabricksSqlHook.run()` method which is called in `execute()` ... From c29e5067a21adb2297e02c53b4c0af90524cdf0a Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 9 Sep 2024 10:13:38 -0600 Subject: [PATCH 180/199] chore: static checks --- .../databricks/hooks/databricks_sql.py | 12 +- .../databricks/operators/databricks_sql.py | 2 +- .../databricks/hooks/test_databricks_sql.py | 131 ++++++++++-------- .../operators/test_databricks_workflow.py | 2 +- 4 files changed, 77 insertions(+), 70 deletions(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index d3fd98c19c769..dc0af83476306 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -21,7 +21,7 @@ from collections import namedtuple from contextlib import closing from copy import copy -from datetime import timedelta, datetime +from datetime import timedelta from typing import ( TYPE_CHECKING, Any, @@ -192,11 +192,11 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., + execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: None = ..., split_statements: bool = ..., return_last: bool = ..., - execution_timeout: timedelta | None = ..., ) -> None: ... @overload @@ -204,11 +204,11 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., + execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: Callable[[Any], T] = ..., split_statements: bool = ..., return_last: bool = ..., - execution_timeout: timedelta | None = ..., ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: ... def run( @@ -259,21 +259,19 @@ def run( conn = None results = [] for sql_statement in sql_list: - # when using AAD tokens, it could expire if previous query run longer than token lifetime conn = self.get_conn() with closing(conn.cursor()) as cur: self.set_autocommit(conn, autocommit) with closing(conn.cursor()) as cur: - t = create_timeout_thread(cur, execution_timeout) - #todo; adjust this to make testing easier + # todo; adjust this to make testing easier try: self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: - if t.is_alive(): + if t is None or t.is_alive(): raise AirflowException(f"Error running statement: {sql_statement}. {str(e)}") else: raise AirflowException( diff --git a/providers/src/airflow/providers/databricks/operators/databricks_sql.py b/providers/src/airflow/providers/databricks/operators/databricks_sql.py index a3851f05e0408..62e55c682de04 100644 --- a/providers/src/airflow/providers/databricks/operators/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/operators/databricks_sql.py @@ -355,6 +355,6 @@ def execute(self, context: Context) -> Any: hook.run(sql) def on_kill(self) -> None: - #NB: on_kill isn't required for this operator since query cancelling gets + # NB: on_kill isn't required for this operator since query cancelling gets # handled in `DatabricksSqlHook.run()` method which is called in `execute()` ... diff --git a/providers/tests/databricks/hooks/test_databricks_sql.py b/providers/tests/databricks/hooks/test_databricks_sql.py index 0467ad41eb58c..0fdfba4776d07 100644 --- a/providers/tests/databricks/hooks/test_databricks_sql.py +++ b/providers/tests/databricks/hooks/test_databricks_sql.py @@ -19,11 +19,10 @@ from __future__ import annotations import threading -import time from collections import namedtuple from datetime import timedelta from unittest import mock -from unittest.mock import patch, PropertyMock +from unittest.mock import PropertyMock, patch import pytest from databricks.sql.types import Row @@ -33,7 +32,6 @@ from airflow.providers.common.sql.hooks.sql import fetch_all_handler from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook, create_timeout_thread from airflow.utils.session import provide_session -from airflow.utils.timezone import datetime pytestmark = pytest.mark.db_test @@ -59,6 +57,7 @@ def create_connection(session): def databricks_hook(): return DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=True) + @pytest.fixture def mock_get_conn(): # Start the patcher @@ -99,14 +98,17 @@ def mock_get_requests(): # Stop the patcher after the test mock_patch.stop() + @pytest.fixture def mock_timer(): with patch("threading.Timer") as mock_timer: yield mock_timer + def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: return [(field,) for field in fields] + # Serializable Row object similar to the one returned by the Hook SerializableRow = namedtuple("Row", ["id", "value"]) # type: ignore[name-match] @@ -306,42 +308,41 @@ def test_query( mock_get_conn, mock_get_requests, ): - - connections = [] - cursors = [] - for index in range(len(cursor_descriptions)): - conn = mock.MagicMock() - cur = mock.MagicMock( - rowcount=len(cursor_results[index]), - description=get_cursor_descriptions(cursor_descriptions[index]), - ) - cur.fetchall.return_value = cursor_results[index] - conn.cursor.return_value = cur - cursors.append(cur) - connections.append(conn) - mock_get_conn.side_effect = connections - - if not return_tuple: - with pytest.warns( - AirflowProviderDeprecationWarning, - match="""Returning a raw `databricks.sql.Row` object is deprecated. A namedtuple will be + connections = [] + cursors = [] + for index in range(len(cursor_descriptions)): + conn = mock.MagicMock() + cur = mock.MagicMock( + rowcount=len(cursor_results[index]), + description=get_cursor_descriptions(cursor_descriptions[index]), + ) + cur.fetchall.return_value = cursor_results[index] + conn.cursor.return_value = cur + cursors.append(cur) + connections.append(conn) + mock_get_conn.side_effect = connections + + if not return_tuple: + with pytest.warns( + AirflowProviderDeprecationWarning, + match="""Returning a raw `databricks.sql.Row` object is deprecated. A namedtuple will be returned instead in a future release of the databricks provider. Set `return_tuple=True` to enable this behavior.""", - ): - databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) - else: + ): databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) - results = databricks_hook.run( - sql=sql, handler=fetch_all_handler, return_last=return_last, split_statements=split_statements - ) + else: + databricks_hook = DatabricksSqlHook(sql_endpoint_name="Test", return_tuple=return_tuple) + results = databricks_hook.run( + sql=sql, handler=fetch_all_handler, return_last=return_last, split_statements=split_statements + ) - assert databricks_hook.descriptions == hook_descriptions - assert databricks_hook.last_description == hook_descriptions[-1] - assert results == hook_results + assert databricks_hook.descriptions == hook_descriptions + assert databricks_hook.last_description == hook_descriptions[-1] + assert results == hook_results - for index, cur in enumerate(cursors): - cur.execute.assert_has_calls([mock.call(cursor_calls[index])]) - cur.close.assert_called() + for index, cur in enumerate(cursors): + cur.execute.assert_has_calls([mock.call(cursor_calls[index])]) + cur.close.assert_called() @pytest.mark.parametrize( @@ -382,14 +383,18 @@ def test_execution_timeout_exceeded( mock_get_requests, sql="select * from test.test", execution_timeout=timedelta(microseconds=0), - cursor_descriptions=["id", "value"], - cursor_results=[Row(id=1, value=2), Row(id=11, value=12)], + cursor_descriptions=( + "id", + "value", + ), + cursor_results=( + Row(id=1, value=2), + Row(id=11, value=12), + ), ): with patch( - 'airflow.providers.databricks.hooks.databricks_sql.create_timeout_thread' - ) as mock_create_timeout_thread, patch.object( - DatabricksSqlHook, '_run_command' - ) as mock_run_command: + "airflow.providers.databricks.hooks.databricks_sql.create_timeout_thread" + ) as mock_create_timeout_thread, patch.object(DatabricksSqlHook, "_run_command") as mock_run_command: conn = mock.MagicMock() cur = mock.MagicMock( rowcount=len(cursor_results), @@ -397,11 +402,9 @@ def test_execution_timeout_exceeded( ) # Simulate a timeout - mock_create_timeout_thread.return_value = threading.Timer( - cur, execution_timeout - ) + mock_create_timeout_thread.return_value = threading.Timer(cur, execution_timeout) - mock_run_command.side_effect = Exception('Mocked exception') + mock_run_command.side_effect = Exception("Mocked exception") cur.fetchall.return_value = cursor_results conn.cursor.return_value = cur @@ -421,28 +424,34 @@ def test_create_timeout_thread( mock_get_conn, mock_get_requests, mock_timer, - cursor_descriptions=["id", "value"], + cursor_descriptions=( + "id", + "value", + ), ): - cur = mock.MagicMock( - rowcount=1, - description=get_cursor_descriptions(cursor_descriptions), - ) - timeout = timedelta(seconds=1) - thread = create_timeout_thread(cur=cur, execution_timeout=timeout) - mock_timer.assert_called_once_with(timeout.total_seconds(), cur.connection.cancel) - assert thread is not None + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + timeout = timedelta(seconds=1) + thread = create_timeout_thread(cur=cur, execution_timeout=timeout) + mock_timer.assert_called_once_with(timeout.total_seconds(), cur.connection.cancel) + assert thread is not None def test_create_timeout_thread_no_timeout( mock_get_conn, mock_get_requests, mock_timer, - cursor_descriptions=["id", "value"], + cursor_descriptions=( + "id", + "value", + ), ): - cur = mock.MagicMock( - rowcount=1, - description=get_cursor_descriptions(cursor_descriptions), - ) - thread = create_timeout_thread(cur=cur, execution_timeout=None) - mock_timer.assert_not_called() - assert thread is None + cur = mock.MagicMock( + rowcount=1, + description=get_cursor_descriptions(cursor_descriptions), + ) + thread = create_timeout_thread(cur=cur, execution_timeout=None) + mock_timer.assert_not_called() + assert thread is None diff --git a/providers/tests/databricks/operators/test_databricks_workflow.py b/providers/tests/databricks/operators/test_databricks_workflow.py index 019f996ab3a19..547aee904f7d9 100644 --- a/providers/tests/databricks/operators/test_databricks_workflow.py +++ b/providers/tests/databricks/operators/test_databricks_workflow.py @@ -31,7 +31,6 @@ WorkflowRunMetadata, _CreateDatabricksWorkflowOperator, _flatten_node, - WorkflowRunMetadata, ) from airflow.utils import timezone @@ -60,6 +59,7 @@ def mock_task_group(): mock_group.group_id = "test_group" return mock_group + @pytest.fixture def mock_workflow_run_metadata(): return MagicMock(spec=WorkflowRunMetadata) From 104983caa7ce33eb6ce04ba5968d77b71de652a6 Mon Sep 17 00:00:00 2001 From: Lorin Date: Wed, 2 Oct 2024 08:37:34 -0600 Subject: [PATCH 181/199] remove changes for databricks_workflow.py for PR isolated to databricks_sql --- .../operators/databricks_workflow.py | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/providers/src/airflow/providers/databricks/operators/databricks_workflow.py b/providers/src/airflow/providers/databricks/operators/databricks_workflow.py index 6df8e2d025cea..15333dc69118b 100644 --- a/providers/src/airflow/providers/databricks/operators/databricks_workflow.py +++ b/providers/src/airflow/providers/databricks/operators/databricks_workflow.py @@ -52,7 +52,7 @@ class WorkflowRunMetadata: """ conn_id: str - job_id: int + job_id: str run_id: int @@ -116,7 +116,6 @@ def __init__( self.notebook_params = notebook_params or {} self.tasks_to_convert = tasks_to_convert or [] self.relevant_upstreams = [task_id] - self.workflow_run_metadata: WorkflowRunMetadata | None = None super().__init__(task_id=task_id, **kwargs) def _get_hook(self, caller: str) -> DatabricksHook: @@ -213,36 +212,12 @@ def execute(self, context: Context) -> Any: self._wait_for_job_to_start(run_id) - self.workflow_run_metadata = WorkflowRunMetadata( - self.databricks_conn_id, - job_id, - run_id, - ) - return { "conn_id": self.databricks_conn_id, "job_id": job_id, "run_id": run_id, } - def on_kill(self) -> None: - if self.workflow_run_metadata: - run_id = self.workflow_run_metadata.run_id - job_id = self.workflow_run_metadata.job_id - - self._hook.cancel_run(run_id) - self.log.info( - "Run: %(run_id)s of job_id: %(job_id)s was requested to be cancelled.", - {"run_id": run_id, "job_id": job_id}, - ) - else: - self.log.error( - """ - Error: Workflow Run metadata is not populated, so the run was not canceled. This could be due - to the workflow not being started or an error in the workflow creation process. - """ - ) - class DatabricksWorkflowTaskGroup(TaskGroup): """ From 105462631c98becb7e9278eccfbf421b3b3c94ab Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 3 Oct 2024 15:03:36 -0600 Subject: [PATCH 182/199] add indent to execution_timeout param in docstring of DatabricksSqlHook.run --- .../src/airflow/providers/databricks/hooks/databricks_sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index dc0af83476306..d959705401d81 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -237,8 +237,8 @@ def run( :param handler: The result handler which is called with the result of each statement. :param split_statements: Whether to split a single SQL string into statements and run separately :param return_last: Whether to return result for only last statement or for all after split - :param execution_timeout: max time allowed for the execution of - this task instance, if it goes beyond it will raise and fail. + :param execution_timeout: max time allowed for the execution of this task instance, if it goes beyond + it will raise and fail. :return: return only result of the LAST SQL expression if handler was provided unless return_last is set to False. """ From 408d8fba30c9b39b3502bcbf8307aa83becdd29a Mon Sep 17 00:00:00 2001 From: Lorin Dawson <22798188+R7L208@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:36:02 -0600 Subject: [PATCH 183/199] Update airflow/providers/databricks/hooks/databricks_sql.py Co-authored-by: Wei Lee --- .../src/airflow/providers/databricks/hooks/databricks_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index d959705401d81..9df20a8166a42 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -267,7 +267,7 @@ def run( with closing(conn.cursor()) as cur: t = create_timeout_thread(cur, execution_timeout) - # todo; adjust this to make testing easier + # TODO: adjust this to make testing easier try: self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: From 30ecf43e48a9358a2e153eb5107088f27c9cd3e0 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 11:40:45 -0600 Subject: [PATCH 184/199] add new exceptions --- airflow/exceptions.py | 5 +++++ .../airflow/providers/databricks/hooks/databricks_sql.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index ccf62ca5e8178..bf492b79a8f97 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -421,6 +421,11 @@ def __repr__(self) -> str: class TaskDeferralError(AirflowException): """Raised when a task failed during deferral for some reason.""" +class AirflowTaskExecutionError(AirflowException): + """Raised when there is an error in task execution.""" + +class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): + """Raised when a task execution times out.""" # The try/except handling is needed after we moved all k8s classes to cncf.kubernetes provider # These two exceptions are used internally by Kubernetes Executor but also by PodGenerator, so we need diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 9df20a8166a42..a9d9933cba0a8 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -37,7 +37,7 @@ from databricks import sql # type: ignore[attr-defined] -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook @@ -272,9 +272,11 @@ def run( self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: if t is None or t.is_alive(): - raise AirflowException(f"Error running statement: {sql_statement}. {str(e)}") + raise AirflowTaskExecutionError( + f"Error running statement: {sql_statement}. {str(e)}" + ) else: - raise AirflowException( + raise AirflowTaskExecutionTimeout( f"Timeout threshold exceeded for query: {sql_statement} was cancelled." ) finally: From 930794dd0dda9f468803e4769d9d5a6ec6c2bffa Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 12:28:20 -0600 Subject: [PATCH 185/199] add new exceptions --- airflow/exceptions.py | 3 +++ .../airflow/providers/databricks/hooks/databricks_sql.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index bf492b79a8f97..7a7b576781aaa 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -421,12 +421,15 @@ def __repr__(self) -> str: class TaskDeferralError(AirflowException): """Raised when a task failed during deferral for some reason.""" + class AirflowTaskExecutionError(AirflowException): """Raised when there is an error in task execution.""" + class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): """Raised when a task execution times out.""" + # The try/except handling is needed after we moved all k8s classes to cncf.kubernetes provider # These two exceptions are used internally by Kubernetes Executor but also by PodGenerator, so we need # to leave them here in case older version of cncf.kubernetes provider is used to run KubernetesPodOperator diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index a9d9933cba0a8..0308c307c7b93 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -37,7 +37,12 @@ from databricks import sql # type: ignore[attr-defined] -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowTaskExecutionError, AirflowTaskExecutionTimeout +from airflow.exceptions import ( + AirflowException, + AirflowProviderDeprecationWarning, + AirflowTaskExecutionError, + AirflowTaskExecutionTimeout, +) from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook From e662230d7df54f9880c39c917dd4f62840814d18 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 12:46:42 -0600 Subject: [PATCH 186/199] fix typing warning --- .../src/airflow/providers/databricks/hooks/databricks_sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 0308c307c7b93..13f67a26a14e0 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -33,6 +33,7 @@ TypeVar, cast, overload, + Union, ) from databricks import sql # type: ignore[attr-defined] @@ -56,7 +57,7 @@ T = TypeVar("T") -def create_timeout_thread(cur, execution_timeout: timedelta | None) -> threading.Timer | None: +def create_timeout_thread(cur, execution_timeout: Union[timedelta, None]) -> Union[threading.Timer, None]: if execution_timeout is not None: seconds_to_timeout = execution_timeout.total_seconds() t = threading.Timer(seconds_to_timeout, cur.connection.cancel) From 894466cf031f4c325e89a6acc4fbd9154284108e Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 17:02:56 -0600 Subject: [PATCH 187/199] fix ruff error --- .../src/airflow/providers/databricks/hooks/databricks_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 13f67a26a14e0..23ef712bd1f6e 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -57,7 +57,7 @@ T = TypeVar("T") -def create_timeout_thread(cur, execution_timeout: Union[timedelta, None]) -> Union[threading.Timer, None]: +def create_timeout_thread(cur, execution_timeout: timedelta | None) -> threading.Timer | None: if execution_timeout is not None: seconds_to_timeout = execution_timeout.total_seconds() t = threading.Timer(seconds_to_timeout, cur.connection.cancel) From 0ae987d8f4d5a22f2ca452cfb70d33ae607b58be Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 4 Oct 2024 17:24:36 -0600 Subject: [PATCH 188/199] fix ruff error --- .../src/airflow/providers/databricks/hooks/databricks_sql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 23ef712bd1f6e..0308c307c7b93 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -33,7 +33,6 @@ TypeVar, cast, overload, - Union, ) from databricks import sql # type: ignore[attr-defined] From d2f37a104f51dfbc0bf4c9e517e1907eeb518083 Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 09:46:53 -0600 Subject: [PATCH 189/199] move new exceptions to Databricks provider --- airflow/exceptions.py | 8 -------- .../src/airflow/providers/databricks/exceptions.py | 11 ++++------- .../providers/databricks/hooks/databricks_sql.py | 3 +-- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 7a7b576781aaa..ccf62ca5e8178 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -422,14 +422,6 @@ class TaskDeferralError(AirflowException): """Raised when a task failed during deferral for some reason.""" -class AirflowTaskExecutionError(AirflowException): - """Raised when there is an error in task execution.""" - - -class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): - """Raised when a task execution times out.""" - - # The try/except handling is needed after we moved all k8s classes to cncf.kubernetes provider # These two exceptions are used internally by Kubernetes Executor but also by PodGenerator, so we need # to leave them here in case older version of cncf.kubernetes provider is used to run KubernetesPodOperator diff --git a/providers/src/airflow/providers/databricks/exceptions.py b/providers/src/airflow/providers/databricks/exceptions.py index 0488c975ed150..5dc885aeabf9d 100644 --- a/providers/src/airflow/providers/databricks/exceptions.py +++ b/providers/src/airflow/providers/databricks/exceptions.py @@ -19,14 +19,11 @@ # to be marked in an ERROR state """Exceptions used by Databricks Provider.""" -from __future__ import annotations - from airflow.exceptions import AirflowException - -class DatabricksSqlExecutionError(AirflowException): - """Raised when there is an error in sql execution.""" +class AirflowTaskExecutionError(AirflowException): + """Raised when there is an error in task execution.""" -class DatabricksSqlExecutionTimeout(DatabricksSqlExecutionError): - """Raised when a sql execution times out.""" +class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): + """Raised when a task execution times out.""" diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 0308c307c7b93..7ebbbe4144ff7 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -40,9 +40,8 @@ from airflow.exceptions import ( AirflowException, AirflowProviderDeprecationWarning, - AirflowTaskExecutionError, - AirflowTaskExecutionTimeout, ) +from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook From 9d5a3cf76218b526458aa441c117efec3c6bb2c0 Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 10:19:00 -0600 Subject: [PATCH 190/199] ruff formatting --- providers/src/airflow/providers/databricks/exceptions.py | 3 +++ .../src/airflow/providers/databricks/hooks/databricks_sql.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/databricks/exceptions.py b/providers/src/airflow/providers/databricks/exceptions.py index 5dc885aeabf9d..aac6556fe113e 100644 --- a/providers/src/airflow/providers/databricks/exceptions.py +++ b/providers/src/airflow/providers/databricks/exceptions.py @@ -19,8 +19,11 @@ # to be marked in an ERROR state """Exceptions used by Databricks Provider.""" +from __future__ import annotations + from airflow.exceptions import AirflowException + class AirflowTaskExecutionError(AirflowException): """Raised when there is an error in task execution.""" diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 7ebbbe4144ff7..e093e1d133c2d 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -41,8 +41,8 @@ AirflowException, AirflowProviderDeprecationWarning, ) -from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results +from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook if TYPE_CHECKING: From 2d64d32598051a6b2c595753c735fc1b19a9108f Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 12:22:46 -0600 Subject: [PATCH 191/199] add tests for new exceptions --- providers/tests/databricks/test_exceptions.py | 33 ++++--------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/providers/tests/databricks/test_exceptions.py b/providers/tests/databricks/test_exceptions.py index 17c42af474694..747314e0301e5 100644 --- a/providers/tests/databricks/test_exceptions.py +++ b/providers/tests/databricks/test_exceptions.py @@ -1,33 +1,14 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - import pytest - -from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout +from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout -def test_databricks_sql_execution_error(): +def test_airflow_task_execution_error(): """Test if AirflowTaskExecutionError can be raised correctly.""" - with pytest.raises(DatabricksSqlExecutionError, match="Task execution failed"): - raise DatabricksSqlExecutionError("Task execution failed") + with pytest.raises(AirflowTaskExecutionError, match="Task execution failed"): + raise AirflowTaskExecutionError("Task execution failed") -def test_databricks_sql_execution_timeout(): +def test_airflow_task_execution_timeout(): """Test if AirflowTaskExecutionTimeout can be raised correctly.""" - with pytest.raises(DatabricksSqlExecutionTimeout, match="Task execution timed out"): - raise DatabricksSqlExecutionTimeout("Task execution timed out") + with pytest.raises(AirflowTaskExecutionTimeout, match="Task execution timed out"): + raise AirflowTaskExecutionTimeout("Task execution timed out") From 75d6995d195070490a7df712896977f40fdb411e Mon Sep 17 00:00:00 2001 From: Lorin Date: Mon, 7 Oct 2024 14:53:24 -0600 Subject: [PATCH 192/199] chore - static checks --- providers/tests/databricks/test_exceptions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/providers/tests/databricks/test_exceptions.py b/providers/tests/databricks/test_exceptions.py index 747314e0301e5..71b36dff2bb96 100644 --- a/providers/tests/databricks/test_exceptions.py +++ b/providers/tests/databricks/test_exceptions.py @@ -1,4 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + import pytest + from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout From abe980c181713dbcc391dee695acca64b2b01960 Mon Sep 17 00:00:00 2001 From: Lorin Dawson <22798188+R7L208@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:51:21 -0600 Subject: [PATCH 193/199] Update airflow/providers/databricks/hooks/databricks_sql.py Co-authored-by: Wei Lee --- .../airflow/providers/databricks/hooks/databricks_sql.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index e093e1d133c2d..dca0c7c452231 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -279,10 +279,9 @@ def run( raise AirflowTaskExecutionError( f"Error running statement: {sql_statement}. {str(e)}" ) - else: - raise AirflowTaskExecutionTimeout( - f"Timeout threshold exceeded for query: {sql_statement} was cancelled." - ) + raise AirflowTaskExecutionTimeout( + f"Timeout threshold exceeded for query: {sql_statement} was cancelled." + ) finally: if t is not None: t.cancel() From e649f1a899ef73e07a212beab9a051cc6c84e0a2 Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 17 Oct 2024 13:35:26 -0600 Subject: [PATCH 194/199] make new exceptions databricks specific --- .../src/airflow/providers/databricks/exceptions.py | 8 ++++---- .../providers/databricks/hooks/databricks_sql.py | 10 +++++----- providers/tests/databricks/test_exceptions.py | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/providers/src/airflow/providers/databricks/exceptions.py b/providers/src/airflow/providers/databricks/exceptions.py index aac6556fe113e..0488c975ed150 100644 --- a/providers/src/airflow/providers/databricks/exceptions.py +++ b/providers/src/airflow/providers/databricks/exceptions.py @@ -24,9 +24,9 @@ from airflow.exceptions import AirflowException -class AirflowTaskExecutionError(AirflowException): - """Raised when there is an error in task execution.""" +class DatabricksSqlExecutionError(AirflowException): + """Raised when there is an error in sql execution.""" -class AirflowTaskExecutionTimeout(AirflowTaskExecutionError): - """Raised when a task execution times out.""" +class DatabricksSqlExecutionTimeout(DatabricksSqlExecutionError): + """Raised when a sql execution times out.""" diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index dca0c7c452231..61975045ac61d 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -42,7 +42,7 @@ AirflowProviderDeprecationWarning, ) from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results -from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout +from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook if TYPE_CHECKING: @@ -276,11 +276,11 @@ def run( self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined] except Exception as e: if t is None or t.is_alive(): - raise AirflowTaskExecutionError( - f"Error running statement: {sql_statement}. {str(e)}" + raise DatabricksSqlExecutionError( + f"Error running SQL statement: {sql_statement}. {str(e)}" ) - raise AirflowTaskExecutionTimeout( - f"Timeout threshold exceeded for query: {sql_statement} was cancelled." + raise DatabricksSqlExecutionTimeout( + f"Timeout threshold exceeded for SQL statement: {sql_statement} was cancelled." ) finally: if t is not None: diff --git a/providers/tests/databricks/test_exceptions.py b/providers/tests/databricks/test_exceptions.py index 71b36dff2bb96..17c42af474694 100644 --- a/providers/tests/databricks/test_exceptions.py +++ b/providers/tests/databricks/test_exceptions.py @@ -18,16 +18,16 @@ import pytest -from airflow.providers.databricks.exceptions import AirflowTaskExecutionError, AirflowTaskExecutionTimeout +from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout -def test_airflow_task_execution_error(): +def test_databricks_sql_execution_error(): """Test if AirflowTaskExecutionError can be raised correctly.""" - with pytest.raises(AirflowTaskExecutionError, match="Task execution failed"): - raise AirflowTaskExecutionError("Task execution failed") + with pytest.raises(DatabricksSqlExecutionError, match="Task execution failed"): + raise DatabricksSqlExecutionError("Task execution failed") -def test_airflow_task_execution_timeout(): +def test_databricks_sql_execution_timeout(): """Test if AirflowTaskExecutionTimeout can be raised correctly.""" - with pytest.raises(AirflowTaskExecutionTimeout, match="Task execution timed out"): - raise AirflowTaskExecutionTimeout("Task execution timed out") + with pytest.raises(DatabricksSqlExecutionTimeout, match="Task execution timed out"): + raise DatabricksSqlExecutionTimeout("Task execution timed out") From d18ba697c73ccbfc5c852cd609aeb171ef0bf411 Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 17 Oct 2024 15:20:43 -0600 Subject: [PATCH 195/199] accidentally removed necessary changes when resolving conflicts --- .../operators/databricks_workflow.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/databricks/operators/databricks_workflow.py b/providers/src/airflow/providers/databricks/operators/databricks_workflow.py index 15333dc69118b..6df8e2d025cea 100644 --- a/providers/src/airflow/providers/databricks/operators/databricks_workflow.py +++ b/providers/src/airflow/providers/databricks/operators/databricks_workflow.py @@ -52,7 +52,7 @@ class WorkflowRunMetadata: """ conn_id: str - job_id: str + job_id: int run_id: int @@ -116,6 +116,7 @@ def __init__( self.notebook_params = notebook_params or {} self.tasks_to_convert = tasks_to_convert or [] self.relevant_upstreams = [task_id] + self.workflow_run_metadata: WorkflowRunMetadata | None = None super().__init__(task_id=task_id, **kwargs) def _get_hook(self, caller: str) -> DatabricksHook: @@ -212,12 +213,36 @@ def execute(self, context: Context) -> Any: self._wait_for_job_to_start(run_id) + self.workflow_run_metadata = WorkflowRunMetadata( + self.databricks_conn_id, + job_id, + run_id, + ) + return { "conn_id": self.databricks_conn_id, "job_id": job_id, "run_id": run_id, } + def on_kill(self) -> None: + if self.workflow_run_metadata: + run_id = self.workflow_run_metadata.run_id + job_id = self.workflow_run_metadata.job_id + + self._hook.cancel_run(run_id) + self.log.info( + "Run: %(run_id)s of job_id: %(job_id)s was requested to be cancelled.", + {"run_id": run_id, "job_id": job_id}, + ) + else: + self.log.error( + """ + Error: Workflow Run metadata is not populated, so the run was not canceled. This could be due + to the workflow not being started or an error in the workflow creation process. + """ + ) + class DatabricksWorkflowTaskGroup(TaskGroup): """ From 900cb3470a4b21f98c05a873f6c01c015959d815 Mon Sep 17 00:00:00 2001 From: Lorin Date: Thu, 17 Oct 2024 16:35:27 -0600 Subject: [PATCH 196/199] remove redefinition of mock_workflow_run_metadata --- .../tests/databricks/operators/test_databricks_workflow.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/providers/tests/databricks/operators/test_databricks_workflow.py b/providers/tests/databricks/operators/test_databricks_workflow.py index 547aee904f7d9..fbc429ed1d9a8 100644 --- a/providers/tests/databricks/operators/test_databricks_workflow.py +++ b/providers/tests/databricks/operators/test_databricks_workflow.py @@ -65,11 +65,6 @@ def mock_workflow_run_metadata(): return MagicMock(spec=WorkflowRunMetadata) -@pytest.fixture -def mock_workflow_run_metadata(): - return MagicMock(spec=WorkflowRunMetadata) - - def test_flatten_node(): """Test that _flatten_node returns a flat list of operators.""" task_group = MagicMock(spec=DatabricksWorkflowTaskGroup) From 8fc1f79446865db144976eb6e2e5cc700bd4227b Mon Sep 17 00:00:00 2001 From: Lorin Dawson <22798188+R7L208@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:31:11 -0600 Subject: [PATCH 197/199] Update providers/tests/databricks/hooks/test_databricks_sql.py Co-authored-by: Wei Lee --- providers/tests/databricks/hooks/test_databricks_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/tests/databricks/hooks/test_databricks_sql.py b/providers/tests/databricks/hooks/test_databricks_sql.py index 0fdfba4776d07..aea7da9d719e6 100644 --- a/providers/tests/databricks/hooks/test_databricks_sql.py +++ b/providers/tests/databricks/hooks/test_databricks_sql.py @@ -314,7 +314,7 @@ def test_query( conn = mock.MagicMock() cur = mock.MagicMock( rowcount=len(cursor_results[index]), - description=get_cursor_descriptions(cursor_descriptions[index]), + description=get_cursor_descriptions(cursor_description), ) cur.fetchall.return_value = cursor_results[index] conn.cursor.return_value = cur From 6bd88bd44c812db6bc106563625541a2c290bea8 Mon Sep 17 00:00:00 2001 From: Lorin Dawson <22798188+R7L208@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:31:20 -0600 Subject: [PATCH 198/199] Update providers/tests/databricks/hooks/test_databricks_sql.py Co-authored-by: Wei Lee --- providers/tests/databricks/hooks/test_databricks_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/tests/databricks/hooks/test_databricks_sql.py b/providers/tests/databricks/hooks/test_databricks_sql.py index aea7da9d719e6..3eb628d46c2b2 100644 --- a/providers/tests/databricks/hooks/test_databricks_sql.py +++ b/providers/tests/databricks/hooks/test_databricks_sql.py @@ -310,7 +310,7 @@ def test_query( ): connections = [] cursors = [] - for index in range(len(cursor_descriptions)): + for index, cursor_description in enumerate(cursor_descriptions): conn = mock.MagicMock() cur = mock.MagicMock( rowcount=len(cursor_results[index]), From bcf4b687eff7b3252f9e6413fc0fe17b5d818d20 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 18 Oct 2024 11:00:53 -0600 Subject: [PATCH 199/199] make execution_timeout last arg and keyword arg --- .../providers/databricks/hooks/databricks_sql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py index 61975045ac61d..6d4f679b2eed9 100644 --- a/providers/src/airflow/providers/databricks/hooks/databricks_sql.py +++ b/providers/src/airflow/providers/databricks/hooks/databricks_sql.py @@ -196,11 +196,11 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., - execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: None = ..., split_statements: bool = ..., return_last: bool = ..., + execution_timeout: timedelta | None = None, ) -> None: ... @overload @@ -208,22 +208,22 @@ def run( self, sql: str | Iterable[str], autocommit: bool = ..., - execution_timeout: timedelta | None = ..., parameters: Iterable | Mapping[str, Any] | None = ..., handler: Callable[[Any], T] = ..., split_statements: bool = ..., return_last: bool = ..., + execution_timeout: timedelta | None = None, ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: ... def run( self, sql: str | Iterable[str], autocommit: bool = False, - execution_timeout: timedelta | None = None, parameters: Iterable | Mapping[str, Any] | None = None, handler: Callable[[Any], T] | None = None, split_statements: bool = True, return_last: bool = True, + execution_timeout: timedelta | None = None, ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: """ Run a command or a list of commands. @@ -241,10 +241,10 @@ def run( :param handler: The result handler which is called with the result of each statement. :param split_statements: Whether to split a single SQL string into statements and run separately :param return_last: Whether to return result for only last statement or for all after split - :param execution_timeout: max time allowed for the execution of this task instance, if it goes beyond - it will raise and fail. :return: return only result of the LAST SQL expression if handler was provided unless return_last is set to False. + :param execution_timeout: max time allowed for the execution of this task instance, if it goes beyond + it will raise and fail. """ self.descriptions = [] if isinstance(sql, str):