From dda99d819fc1843ff0cf45d9ef2b4d93ba2e470e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 22 Mar 2022 17:56:03 +0100 Subject: [PATCH 01/17] ISSUE #150 * Improve `BrokerSubscriberBuilder`. --- .../networks/brokers/subscribers/__init__.py | 1 + .../minos/networks/brokers/subscribers/abc.py | 86 ++++++++++++++++++- .../subscribers/idempotent/__init__.py | 1 + .../idempotent/detectors/__init__.py | 1 + .../subscribers/idempotent/detectors/abc.py | 12 ++- .../idempotent/detectors/memory.py | 15 ++++ .../subscribers/idempotent/detectors/pg.py | 24 ++++++ .../brokers/subscribers/idempotent/impl.py | 3 + .../networks/brokers/subscribers/memory.py | 9 +- .../brokers/subscribers/queued/impl.py | 10 +-- .../brokers/subscribers/queued/queues/abc.py | 5 +- .../subscribers/queued/queues/memory.py | 3 + .../brokers/subscribers/queued/queues/pg.py | 3 + .../test_brokers/test_subscribers/test_abc.py | 15 +--- .../test_subscribers/test_queued/test_impl.py | 3 - .../minos/plugins/kafka/subscriber.py | 13 +-- 16 files changed, 165 insertions(+), 39 deletions(-) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py index 9726686d0..63364dcd4 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py @@ -4,6 +4,7 @@ ) from .idempotent import ( BrokerSubscriberDuplicateDetector, + BrokerSubscriberDuplicateDetectorBuilder, IdempotentBrokerSubscriber, InMemoryBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetector, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index ceff83283..a563a16b6 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -12,12 +12,15 @@ Iterable, ) from typing import ( + TYPE_CHECKING, + Any, Optional, ) from minos.common import ( BuildableMixin, Builder, + Config, Injectable, ) @@ -25,6 +28,16 @@ BrokerMessage, ) +if TYPE_CHECKING: + from .idempotent import ( + BrokerSubscriberDuplicateDetectorBuilder, + IdempotentBrokerSubscriber, + ) + from .queued import ( + BrokerSubscriberQueueBuilder, + QueuedBrokerSubscriber, + ) + logger = logging.getLogger(__name__) @@ -66,9 +79,52 @@ async def _receive(self) -> BrokerMessage: @Injectable("broker_subscriber_builder") -class BrokerSubscriberBuilder(Builder[BrokerSubscriber], ABC): +class BrokerSubscriberBuilder(Builder[BrokerSubscriber]): """Broker Subscriber Builder class.""" + impl_cls: type[BrokerSubscriber] + idempotent_cls: type[IdempotentBrokerSubscriber] + queued_cls: type[QueuedBrokerSubscriber] + + def __init__( + self, + *args, + idempotent_builder: Optional[BrokerSubscriberDuplicateDetectorBuilder] = None, + queue_builder: Optional[BrokerSubscriberQueueBuilder] = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.duplicate_detector_builder = idempotent_builder + self.queue_builder = queue_builder + + def with_config(self, config: Config): + """Set config. + + :param config: The config to be set. + :return: This method return the builder instance. + """ + broker_config = config.get_interface_by_name("broker") + broker_subscriber_config = broker_config.get("subscriber", None) + if broker_subscriber_config is not None and broker_subscriber_config.get("idempotent", None) is not None: + self.duplicate_detector_builder = ( + broker_subscriber_config.get("idempotent").get_builder().new().with_config(config) + ) + if broker_subscriber_config is not None and broker_subscriber_config.get("queue", None) is not None: + self.queue_builder = broker_subscriber_config.get("queue").get_builder().new().with_config(config) + return super().with_config(config) + + def with_kwargs(self, kwargs: dict[str, Any]): + """Set kwargs. + + :param kwargs: The kwargs to be set. + :return: This method return the builder instance. + """ + if self.duplicate_detector_builder is not None: + self.duplicate_detector_builder.with_kwargs(kwargs) + if self.queue_builder is not None: + self.queue_builder.with_kwargs(kwargs) + return super().with_kwargs(kwargs) + def with_group_id(self, group_id: Optional[str]): """Set group_id. @@ -93,8 +149,36 @@ def with_topics(self, topics: Iterable[str]): :param topics: The topics to be set. :return: This method return the builder instance. """ + topics = set(topics) self.kwargs["topics"] = set(topics) + if self.queue_builder is not None: + self.queue_builder.with_topics(topics) return self + def build(self) -> BrokerSubscriber: + """Build the instance. + + :return: A ``QueuedBrokerSubscriber`` instance. + """ + impl = self.impl_cls(**self.kwargs) + + if self.duplicate_detector_builder is not None: + from .idempotent import ( + IdempotentBrokerSubscriber, + ) + + duplicate_detector = self.duplicate_detector_builder.build() + impl = IdempotentBrokerSubscriber(impl=impl, duplicate_detector=duplicate_detector, **self.kwargs) + + if self.queue_builder is not None: + from .queued import ( + QueuedBrokerSubscriber, + ) + + queue = self.queue_builder.build() + impl = QueuedBrokerSubscriber(impl=impl, queue=queue, **self.kwargs) + + return impl + BrokerSubscriber.set_builder(BrokerSubscriberBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py index 3cdac8c73..b8f819724 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py @@ -1,5 +1,6 @@ from .detectors import ( BrokerSubscriberDuplicateDetector, + BrokerSubscriberDuplicateDetectorBuilder, InMemoryBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py index 5bb630ea7..3c0371535 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py @@ -1,5 +1,6 @@ from .abc import ( BrokerSubscriberDuplicateDetector, + BrokerSubscriberDuplicateDetectorBuilder, ) from .memory import ( InMemoryBrokerSubscriberDuplicateDetector, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py index f4d0bd4ff..e80b1a858 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py @@ -11,7 +11,8 @@ ) from minos.common import ( - SetupMixin, + BuildableMixin, + Builder, ) from ....messages import ( @@ -19,7 +20,7 @@ ) -class BrokerSubscriberDuplicateDetector(ABC, SetupMixin): +class BrokerSubscriberDuplicateDetector(BuildableMixin, ABC): """Broker Subscriber Duplicate Detector class.""" async def is_valid(self, message: BrokerMessage) -> bool: @@ -33,3 +34,10 @@ async def is_valid(self, message: BrokerMessage) -> bool: @abstractmethod async def _is_valid(self, topic: str, uuid: UUID) -> bool: raise NotImplementedError + + +class BrokerSubscriberDuplicateDetectorBuilder(Builder[BrokerSubscriberDuplicateDetector], ABC): + """TODO""" + + +BrokerSubscriberDuplicateDetector.set_builder(BrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py index 62b605b35..c9f70ba71 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py @@ -7,6 +7,7 @@ from .abc import ( BrokerSubscriberDuplicateDetector, + BrokerSubscriberDuplicateDetectorBuilder, ) @@ -32,3 +33,17 @@ async def _is_valid(self, topic: str, uuid: UUID) -> bool: self._seen.add((topic, uuid)) return True return False + + +class InMemoryBrokerSubscriberDuplicateDetectorBuilder(BrokerSubscriberDuplicateDetectorBuilder): + """In Memory Broker Subscriber Queue Builder class.""" + + def build(self) -> InMemoryBrokerSubscriberDuplicateDetector: + """Build the instance. + + :return: An ``InMemoryBrokerSubscriberQueue`` instance. + """ + return InMemoryBrokerSubscriberDuplicateDetector(**self.kwargs) + + +InMemoryBrokerSubscriberDuplicateDetector.set_builder(InMemoryBrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py index 76f253962..9d1c3015e 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py @@ -23,6 +23,7 @@ from .abc import ( BrokerSubscriberDuplicateDetector, + BrokerSubscriberDuplicateDetectorBuilder, ) @@ -71,6 +72,29 @@ async def _is_valid(self, topic: str, uuid: UUID) -> bool: return False +class PostgreSqlBrokerSubscriberDuplicateDetectorBuilder(BrokerSubscriberDuplicateDetectorBuilder): + """TODO""" + + def with_config(self, config: Config): + """Set config. + + :param config: The config to be set. + :return: This method return the builder instance. + """ + self.kwargs |= config.get_database_by_name("broker") + return super().with_config(config) + + def build(self) -> PostgreSqlBrokerSubscriberDuplicateDetector: + """Build the instance. + + :return: A ``BrokerSubscriberQueue`` instance. + """ + return PostgreSqlBrokerSubscriberDuplicateDetector(**self.kwargs) + + +PostgreSqlBrokerSubscriberDuplicateDetector.set_builder(PostgreSqlBrokerSubscriberDuplicateDetectorBuilder) + + class PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory: """PostgreSql Broker Subscriber Duplicate Detector Query Factory class.""" diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py index 63a10cd00..19f8b327f 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py @@ -37,3 +37,6 @@ async def _receive(self) -> BrokerMessage: while message is _sentinel or not (await self.duplicate_detector.is_valid(message)): message = await self.impl.receive() return message + + +IdempotentBrokerSubscriber.set_builder(None) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py index d0cdc3a85..86606c7ea 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py @@ -44,6 +44,8 @@ async def _receive(self) -> BrokerMessage: class InMemoryBrokerSubscriberBuilder(BrokerSubscriberBuilder): """In Memory Broker Subscriber Builder class.""" + impl_cls = InMemoryBrokerSubscriber + def with_messages(self, messages: Iterable[BrokerMessage]) -> InMemoryBrokerSubscriberBuilder: """Set messages. @@ -52,10 +54,3 @@ def with_messages(self, messages: Iterable[BrokerMessage]) -> InMemoryBrokerSubs """ self.kwargs["messages"] = messages return self - - def build(self) -> BrokerSubscriber: - """Build the instance. - - :return: An ``InMemoryBrokerSubscriber`` instance. - """ - return InMemoryBrokerSubscriber(**self.kwargs) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py index 607365431..eb7438fcc 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py @@ -99,7 +99,7 @@ def with_config(self, config: Config) -> BrokerSubscriberBuilder: """ self.impl_builder.with_config(config) self.queue_builder.with_config(config) - return super().with_config(config) + return self def with_kwargs(self, kwargs: dict[str, Any]) -> BrokerSubscriberBuilder: """Set kwargs. @@ -109,7 +109,7 @@ def with_kwargs(self, kwargs: dict[str, Any]) -> BrokerSubscriberBuilder: """ self.impl_builder.with_kwargs(kwargs) self.queue_builder.with_kwargs(kwargs) - return super().with_kwargs(kwargs) + return self def with_topics(self, topics: Iterable[str]) -> BrokerSubscriberBuilder: """Set topics. @@ -120,7 +120,7 @@ def with_topics(self, topics: Iterable[str]) -> BrokerSubscriberBuilder: topics = set(topics) self.impl_builder.with_topics(topics) self.queue_builder.with_topics(topics) - return super().with_topics(topics) + return self def with_group_id(self, group_id: Optional[str]) -> BrokerSubscriberBuilder: """Set group_id. @@ -129,7 +129,7 @@ def with_group_id(self, group_id: Optional[str]) -> BrokerSubscriberBuilder: :return: This method return the builder instance. """ self.impl_builder.with_group_id(group_id) - return super().with_group_id(group_id) + return self def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> BrokerSubscriberBuilder: """Set remove_topics_on_destroy. @@ -138,7 +138,7 @@ def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> Broke :return: This method return the builder instance. """ self.impl_builder.with_remove_topics_on_destroy(remove_topics_on_destroy) - return super().with_remove_topics_on_destroy(remove_topics_on_destroy) + return self def build(self) -> BrokerSubscriber: """Build the instance. diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py index 55bc70553..82508d88c 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py @@ -14,6 +14,7 @@ ) from minos.common import ( + BuildableMixin, Builder, ) @@ -24,7 +25,7 @@ logger = logging.getLogger(__name__) -class BrokerSubscriberQueue(BrokerQueue, ABC): +class BrokerSubscriberQueue(BrokerQueue, BuildableMixin, ABC): """Broker Subscriber Queue class.""" def __init__(self, topics: Iterable[str], **kwargs): @@ -56,4 +57,6 @@ def with_topics(self: B, topics: Iterable[str]) -> B: return self +BrokerSubscriberQueue.set_builder(BrokerSubscriberQueueBuilder) + B = TypeVar("B", bound=BrokerSubscriberQueueBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py index fe0b900a7..82db7a979 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py @@ -28,3 +28,6 @@ def build(self) -> BrokerSubscriberQueue: :return: An ``InMemoryBrokerSubscriberQueue`` instance. """ return InMemoryBrokerSubscriberQueue(**self.kwargs) + + +InMemoryBrokerSubscriberQueue.set_builder(InMemoryBrokerSubscriberQueueBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py index 0253fcbaf..d0f952431 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py @@ -146,3 +146,6 @@ def build(self) -> PostgreSqlBrokerSubscriberQueue: :return: A ``BrokerSubscriberQueue`` instance. """ return PostgreSqlBrokerSubscriberQueue(**self.kwargs) + + +PostgreSqlBrokerSubscriberQueue.set_builder(PostgreSqlBrokerSubscriberQueueBuilder) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py index a8919f37a..7c0e7ccea 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py @@ -18,17 +18,15 @@ ) -class _BrokerSubscriberBuilder(BrokerSubscriberBuilder): - def build(self) -> BrokerSubscriber: - """For testing purposes.""" - return _BrokerSubscriber(**self.kwargs) - - class _BrokerSubscriber(BrokerSubscriber): async def _receive(self) -> BrokerMessage: """For testing purposes.""" +class _BrokerSubscriberBuilder(BrokerSubscriberBuilder): + impl_cls = _BrokerSubscriber + + class TestBrokerSubscriber(unittest.IsolatedAsyncioTestCase): def test_abstract(self): self.assertTrue(issubclass(BrokerSubscriber, (ABC, SetupMixin))) @@ -70,11 +68,6 @@ async def test_aiter(self): class TestBrokerSubscriberBuilder(unittest.TestCase): - def test_abstract(self): - self.assertTrue(issubclass(BrokerSubscriberBuilder, (ABC, SetupMixin))) - # noinspection PyUnresolvedReferences - self.assertEqual({"build"}, BrokerSubscriberBuilder.__abstractmethods__) - def test_with_group_id(self): builder = _BrokerSubscriberBuilder().with_group_id("foobar") self.assertIsInstance(builder, _BrokerSubscriberBuilder) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py index 8eec8e3fd..83beeffb8 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py @@ -139,7 +139,6 @@ def test_with_group_id(self): builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_group_id("foobar") self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) - self.assertEqual({"group_id": "foobar"}, builder.kwargs) self.assertEqual([call("foobar")], impl_mock.call_args_list) @@ -149,7 +148,6 @@ def test_with_remove_topics_on_destroy(self): builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_remove_topics_on_destroy(False) self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) - self.assertEqual({"remove_topics_on_destroy": False}, builder.kwargs) self.assertEqual([call(False)], impl_mock.call_args_list) @@ -161,7 +159,6 @@ def test_with_topics(self): builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_topics({"one", "two"}) self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) - self.assertEqual({"topics": {"one", "two"}}, builder.kwargs) self.assertEqual([call({"one", "two"})], impl_mock.call_args_list) self.assertEqual([call({"one", "two"})], queue_mock.call_args_list) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py index 7373d110f..764d33469 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py @@ -153,7 +153,9 @@ def client(self) -> AIOKafkaConsumer: class KafkaBrokerSubscriberBuilder(BrokerSubscriberBuilder): """Kafka Broker Subscriber Builder class.""" - def with_config(self, config: Config) -> BrokerSubscriberBuilder: + impl_cls = KafkaBrokerSubscriber + + def with_config(self, config: Config) -> KafkaBrokerSubscriberBuilder: """Set config. :param config: The config to be set. @@ -167,14 +169,7 @@ def with_config(self, config: Config) -> BrokerSubscriberBuilder: "broker_host": common_config["host"], "broker_port": common_config["port"], } - return self - - def build(self) -> BrokerSubscriber: - """Build the instance. - - :return: A ``KafkaBrokerSubscriber`` instance. - """ - return KafkaBrokerSubscriber(**self.kwargs) + return super().with_config(config) KafkaBrokerSubscriber.set_builder(KafkaBrokerSubscriberBuilder) From 2cac8261e6d65155ceafff8dc35bf6563eb25e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 08:41:16 +0100 Subject: [PATCH 02/17] ISSUE #150 * Fix bug related with not `broker` interface on config. --- .../minos/networks/brokers/subscribers/abc.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index a563a16b6..029eb7e9f 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -11,6 +11,9 @@ AsyncIterator, Iterable, ) +from contextlib import ( + suppress, +) from typing import ( TYPE_CHECKING, Any, @@ -22,6 +25,7 @@ Builder, Config, Injectable, + MinosConfigException, ) from ..messages import ( @@ -103,14 +107,15 @@ def with_config(self, config: Config): :param config: The config to be set. :return: This method return the builder instance. """ - broker_config = config.get_interface_by_name("broker") - broker_subscriber_config = broker_config.get("subscriber", None) - if broker_subscriber_config is not None and broker_subscriber_config.get("idempotent", None) is not None: - self.duplicate_detector_builder = ( - broker_subscriber_config.get("idempotent").get_builder().new().with_config(config) - ) - if broker_subscriber_config is not None and broker_subscriber_config.get("queue", None) is not None: - self.queue_builder = broker_subscriber_config.get("queue").get_builder().new().with_config(config) + with suppress(MinosConfigException): + broker_config = config.get_interface_by_name("broker") + broker_subscriber_config = broker_config.get("subscriber", None) + if broker_subscriber_config is not None and broker_subscriber_config.get("idempotent", None) is not None: + self.duplicate_detector_builder = ( + broker_subscriber_config.get("idempotent").get_builder().new().with_config(config) + ) + if broker_subscriber_config is not None and broker_subscriber_config.get("queue", None) is not None: + self.queue_builder = broker_subscriber_config.get("queue").get_builder().new().with_config(config) return super().with_config(config) def with_kwargs(self, kwargs: dict[str, Any]): From f589924e7da098fc369fe71ba42235afae463cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 10:29:10 +0100 Subject: [PATCH 03/17] ISSUE #150 * Refactor `Builder` class. --- .../minos/common/builders.py | 45 ++++-- .../tests/test_common/test_builders.py | 39 ++---- .../minos-microservice-common/tests/utils.py | 6 - .../minos/networks/__init__.py | 1 + .../minos/networks/brokers/__init__.py | 1 + .../networks/brokers/subscribers/__init__.py | 2 +- .../minos/networks/brokers/subscribers/abc.py | 131 ++++++++++++++---- .../subscribers/idempotent/__init__.py | 2 +- .../idempotent/detectors/__init__.py | 2 +- .../subscribers/idempotent/detectors/abc.py | 8 -- .../idempotent/detectors/memory.py | 15 -- .../subscribers/idempotent/detectors/pg.py | 15 +- .../brokers/subscribers/idempotent/impl.py | 6 +- .../networks/brokers/subscribers/memory.py | 7 +- .../brokers/subscribers/queued/impl.py | 19 ++- .../brokers/subscribers/queued/queues/abc.py | 6 +- .../subscribers/queued/queues/memory.py | 9 +- .../brokers/subscribers/queued/queues/pg.py | 9 +- .../minos/networks/utils.py | 5 +- .../test_brokers/test_subscribers/test_abc.py | 22 ++- .../test_queued/test_queues/test_abc.py | 24 +--- .../tests/test_networks/test_utils.py | 6 +- .../minos/plugins/kafka/subscriber.py | 4 +- 23 files changed, 201 insertions(+), 183 deletions(-) diff --git a/packages/core/minos-microservice-common/minos/common/builders.py b/packages/core/minos-microservice-common/minos/common/builders.py index 1902e65aa..c2621f3cb 100644 --- a/packages/core/minos-microservice-common/minos/common/builders.py +++ b/packages/core/minos-microservice-common/minos/common/builders.py @@ -4,12 +4,13 @@ from abc import ( ABC, - abstractmethod, ) from typing import ( Any, Generic, + Optional, TypeVar, + get_args, ) from .config import ( @@ -19,22 +20,41 @@ SetupMixin, ) -Instance = TypeVar("Instance") +Instance = TypeVar("Instance", bound=type) class Builder(SetupMixin, ABC, Generic[Instance]): """Builder class.""" - def __init__(self, *args, **kwargs): + def __init__(self, instance_cls: Optional[type[Instance]] = None, *args, **kwargs): + super().__init__(*args, **kwargs) + if instance_cls is None: + instance_cls = self._get_cls() + self.kwargs = dict() + self.instance_cls = instance_cls + + def _get_cls(self) -> Optional[type]: + # noinspection PyUnresolvedReferences + bases = self.__orig_bases__ + + try: + instance_cls = get_args(next((base for base in bases if len(get_args(base))), None))[0] + except Exception: + return None + + if not isinstance(instance_cls, type): + return None + + return instance_cls def copy(self: type[B]) -> B: """Get a copy of the instance. :return: A ``BrokerSubscriberBuilder`` instance. """ - return self.new().with_kwargs(self.kwargs) + return self.new().with_cls(self.instance_cls).with_kwargs(self.kwargs) @classmethod def new(cls: type[B]) -> B: @@ -44,6 +64,15 @@ def new(cls: type[B]) -> B: """ return cls() + def with_cls(self: B, cls: type) -> B: + """TODO + + :param cls: TODO + :return: TODO + """ + self.instance_cls = cls + return self + def with_kwargs(self: B, kwargs: dict[str, Any]) -> B: """Set kwargs. @@ -62,12 +91,12 @@ def with_config(self: B, config: Config) -> B: """ return self - @abstractmethod def build(self) -> Instance: """Build the instance. :return: A ``BrokerSubscriber`` instance. """ + return self.instance_cls(**self.kwargs) Ins = TypeVar("Ins", bound="BuildableMixin") @@ -76,11 +105,11 @@ def build(self) -> Instance: class BuildableMixin(SetupMixin): """Buildable Mixin class.""" - _builder_cls: type[Builder[Ins]] + _builder_cls: type[Builder[Ins]] = Builder @classmethod - def _from_config(cls: type[Ins], config: Config, **kwargs) -> Ins: - return cls.get_builder().new().with_config(config).with_kwargs(kwargs).build() + def _from_config(cls, config: Config, **kwargs): + return cls.get_builder().new().with_cls(cls).with_config(config).with_kwargs(kwargs).build() @classmethod def set_builder(cls: type[Ins], builder: type[Builder[Ins]]) -> None: diff --git a/packages/core/minos-microservice-common/tests/test_common/test_builders.py b/packages/core/minos-microservice-common/tests/test_common/test_builders.py index a8918f1e4..1cf34af71 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_builders.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_builders.py @@ -1,59 +1,42 @@ import unittest -from abc import ( - ABC, -) -from typing import ( - Any, -) from minos.common import ( Builder, Config, - SetupMixin, ) from tests.utils import ( CONFIG_FILE_PATH, ) -class _Builder(Builder[dict[str, Any]]): - def build(self) -> dict[str, Any]: - """For testing purposes.""" - return self.kwargs - - class TestBuilder(unittest.TestCase): - def test_abstract(self): - self.assertTrue(issubclass(Builder, (ABC, SetupMixin))) - # noinspection PyUnresolvedReferences - self.assertEqual({"build"}, Builder.__abstractmethods__) - def test_new(self): - builder = _Builder.new() - self.assertIsInstance(builder, _Builder) + builder = Builder.new().with_cls(dict) + self.assertIsInstance(builder, Builder) self.assertEqual(dict(), builder.kwargs) def test_copy(self): - base = _Builder.new().with_kwargs({"one": "two"}) + base = Builder.new().with_cls(dict).with_kwargs({"one": "two"}) builder = base.copy() self.assertNotEqual(id(base), id(builder)) - self.assertIsInstance(builder, _Builder) + self.assertIsInstance(builder, Builder) self.assertEqual({"one": "two"}, builder.kwargs) + self.assertEqual(dict, builder.instance_cls) def test_with_kwargs(self): - builder = _Builder().with_kwargs({"foo": "bar"}) - self.assertIsInstance(builder, _Builder) + builder = Builder().with_cls(dict).with_kwargs({"foo": "bar"}) + self.assertIsInstance(builder, Builder) self.assertEqual({"foo": "bar"}, builder.kwargs) def test_with_config(self): config = Config(CONFIG_FILE_PATH) - builder = _Builder().with_config(config) - self.assertIsInstance(builder, _Builder) + builder = Builder().with_cls(dict).with_config(config) + self.assertIsInstance(builder, Builder) self.assertEqual(dict(), builder.kwargs) def test_build(self): - builder = _Builder().with_kwargs({"one": "two"}) - self.assertIsInstance(builder, _Builder) + builder = Builder().with_cls(dict).with_kwargs({"one": "two"}) + self.assertIsInstance(builder, Builder) self.assertEqual({"one": "two"}, builder.build()) diff --git a/packages/core/minos-microservice-common/tests/utils.py b/packages/core/minos-microservice-common/tests/utils.py index 5356ed0db..d8eac9c62 100644 --- a/packages/core/minos-microservice-common/tests/utils.py +++ b/packages/core/minos-microservice-common/tests/utils.py @@ -132,9 +132,6 @@ class FakeBrokerPublisher(BuildableMixin): class FakeBrokerPublisherBuilder(Builder[FakeBrokerPublisher]): """For testing purposes.""" - def build(self) -> FakeBrokerPublisher: - return FakeBrokerPublisher() - FakeBrokerPublisher.set_builder(FakeBrokerPublisherBuilder) @@ -147,9 +144,6 @@ class FakeBrokerSubscriber(BuildableMixin): class FakeBrokerSubscriberBuilder(Builder[FakeBrokerSubscriber]): """For testing purposes.""" - def build(self) -> FakeBrokerSubscriber: - return FakeBrokerSubscriber() - FakeBrokerSubscriber.set_builder(FakeBrokerSubscriberBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/__init__.py b/packages/core/minos-microservice-networks/minos/networks/__init__.py index 427ae6ad5..9fc9f6c33 100644 --- a/packages/core/minos-microservice-networks/minos/networks/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/__init__.py @@ -39,6 +39,7 @@ PostgreSqlBrokerPublisherQueueQueryFactory, PostgreSqlBrokerQueue, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, PostgreSqlBrokerSubscriberQueue, PostgreSqlBrokerSubscriberQueueBuilder, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py index 6681a48fd..87d63f13c 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py @@ -50,6 +50,7 @@ InMemoryBrokerSubscriberQueue, InMemoryBrokerSubscriberQueueBuilder, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, PostgreSqlBrokerSubscriberQueue, PostgreSqlBrokerSubscriberQueueBuilder, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py index 63364dcd4..3b4bbdf56 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py @@ -4,10 +4,10 @@ ) from .idempotent import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, IdempotentBrokerSubscriber, InMemoryBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, ) from .memory import ( diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index 029eb7e9f..577b5cd13 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -11,13 +11,13 @@ AsyncIterator, Iterable, ) -from contextlib import ( - suppress, -) from typing import ( TYPE_CHECKING, Any, + Generic, Optional, + TypeVar, + Union, ) from minos.common import ( @@ -34,7 +34,6 @@ if TYPE_CHECKING: from .idempotent import ( - BrokerSubscriberDuplicateDetectorBuilder, IdempotentBrokerSubscriber, ) from .queued import ( @@ -82,42 +81,116 @@ async def _receive(self) -> BrokerMessage: raise NotImplementedError +BrokerSubscriberCls = TypeVar("BrokerSubscriberCls", bound=BrokerSubscriber) + + @Injectable("broker_subscriber_builder") -class BrokerSubscriberBuilder(Builder[BrokerSubscriber]): +class BrokerSubscriberBuilder(Builder[BrokerSubscriberCls], Generic[BrokerSubscriberCls]): """Broker Subscriber Builder class.""" - impl_cls: type[BrokerSubscriber] - idempotent_cls: type[IdempotentBrokerSubscriber] - queued_cls: type[QueuedBrokerSubscriber] - def __init__( self, *args, - idempotent_builder: Optional[BrokerSubscriberDuplicateDetectorBuilder] = None, + idempotent_builder: Optional[Builder] = None, queue_builder: Optional[BrokerSubscriberQueueBuilder] = None, + idempotent_cls: Optional[type[IdempotentBrokerSubscriber]] = None, + queued_cls: Optional[type[QueuedBrokerSubscriber]] = None, **kwargs, ): super().__init__(*args, **kwargs) + + if idempotent_cls is None: + from .idempotent import ( + IdempotentBrokerSubscriber, + ) + + idempotent_cls = IdempotentBrokerSubscriber + + if queued_cls is None: + from .queued import ( + QueuedBrokerSubscriber, + ) + + queued_cls = QueuedBrokerSubscriber + self.duplicate_detector_builder = idempotent_builder self.queue_builder = queue_builder + self.idempotent_cls = idempotent_cls + self.queued_cls = queued_cls + + def with_idempotent_cls(self, idempotent_cls: type[IdempotentBrokerSubscriber]): + """TODO + + :param idempotent_cls: TODO + :return: TODO + """ + self.idempotent_cls = idempotent_cls + + return self + + def with_queued_cls(self, queued_cls: type[QueuedBrokerSubscriber]): + """TODO + + :param queued_cls: TODO + :return: TODO + """ + self.queued_cls = queued_cls + + return self + def with_config(self, config: Config): """Set config. :param config: The config to be set. :return: This method return the builder instance. """ - with suppress(MinosConfigException): - broker_config = config.get_interface_by_name("broker") - broker_subscriber_config = broker_config.get("subscriber", None) - if broker_subscriber_config is not None and broker_subscriber_config.get("idempotent", None) is not None: - self.duplicate_detector_builder = ( - broker_subscriber_config.get("idempotent").get_builder().new().with_config(config) - ) - if broker_subscriber_config is not None and broker_subscriber_config.get("queue", None) is not None: - self.queue_builder = broker_subscriber_config.get("queue").get_builder().new().with_config(config) + self._with_builders_from_config(config) + + if self.duplicate_detector_builder is not None: + self.duplicate_detector_builder.with_config(config) + if self.queue_builder is not None: + self.queue_builder.with_config(config) return super().with_config(config) + def _with_builders_from_config(self, config): + try: + broker_config = config.get_interface_by_name("broker") + except MinosConfigException: + return + + broker_subscriber_config = broker_config["subscriber"] + + if "idempotent" in broker_subscriber_config: + self.with_duplicate_detector(broker_subscriber_config["idempotent"]) + + if "queue" in broker_subscriber_config: + self.with_queue(broker_subscriber_config["queue"]) + + def with_duplicate_detector( + self, duplicate_detector: Union[IdempotentBrokerSubscriber, Builder[IdempotentBrokerSubscriber]] + ): + """TODO + + :param duplicate_detector: TODO + :return: TODO + """ + if not isinstance(duplicate_detector, Builder): + duplicate_detector = duplicate_detector.get_builder() + self.duplicate_detector_builder = duplicate_detector.new() + return self + + def with_queue(self, queue: Union[QueuedBrokerSubscriber, BrokerSubscriberQueueBuilder]): + """TODO + + :param queue: TODO + :return: TODO + """ + if not isinstance(queue, Builder): + queue = queue.get_builder() + self.queue_builder = queue.new() + return self + def with_kwargs(self, kwargs: dict[str, Any]): """Set kwargs. @@ -126,8 +199,10 @@ def with_kwargs(self, kwargs: dict[str, Any]): """ if self.duplicate_detector_builder is not None: self.duplicate_detector_builder.with_kwargs(kwargs) + if self.queue_builder is not None: self.queue_builder.with_kwargs(kwargs) + return super().with_kwargs(kwargs) def with_group_id(self, group_id: Optional[str]): @@ -137,6 +212,7 @@ def with_group_id(self, group_id: Optional[str]): :return: This method return the builder instance. """ self.kwargs["group_id"] = group_id + return self def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool): @@ -146,6 +222,7 @@ def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool): :return: This method return the builder instance. """ self.kwargs["remove_topics_on_destroy"] = remove_topics_on_destroy + return self def with_topics(self, topics: Iterable[str]): @@ -156,8 +233,10 @@ def with_topics(self, topics: Iterable[str]): """ topics = set(topics) self.kwargs["topics"] = set(topics) + if self.queue_builder is not None: self.queue_builder.with_topics(topics) + return self def build(self) -> BrokerSubscriber: @@ -165,23 +244,15 @@ def build(self) -> BrokerSubscriber: :return: A ``QueuedBrokerSubscriber`` instance. """ - impl = self.impl_cls(**self.kwargs) + impl = super().build() if self.duplicate_detector_builder is not None: - from .idempotent import ( - IdempotentBrokerSubscriber, - ) - duplicate_detector = self.duplicate_detector_builder.build() - impl = IdempotentBrokerSubscriber(impl=impl, duplicate_detector=duplicate_detector, **self.kwargs) + impl = self.idempotent_cls(impl=impl, duplicate_detector=duplicate_detector, **self.kwargs) if self.queue_builder is not None: - from .queued import ( - QueuedBrokerSubscriber, - ) - queue = self.queue_builder.build() - impl = QueuedBrokerSubscriber(impl=impl, queue=queue, **self.kwargs) + impl = self.queued_cls(impl=impl, queue=queue, **self.kwargs) return impl diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py index b8f819724..10ac806c8 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py @@ -1,8 +1,8 @@ from .detectors import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, InMemoryBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, ) from .impl import ( diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py index 3c0371535..9a9e01451 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py @@ -1,11 +1,11 @@ from .abc import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, ) from .memory import ( InMemoryBrokerSubscriberDuplicateDetector, ) from .pg import ( PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, ) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py index e80b1a858..154d5c3fa 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py @@ -12,7 +12,6 @@ from minos.common import ( BuildableMixin, - Builder, ) from ....messages import ( @@ -34,10 +33,3 @@ async def is_valid(self, message: BrokerMessage) -> bool: @abstractmethod async def _is_valid(self, topic: str, uuid: UUID) -> bool: raise NotImplementedError - - -class BrokerSubscriberDuplicateDetectorBuilder(Builder[BrokerSubscriberDuplicateDetector], ABC): - """TODO""" - - -BrokerSubscriberDuplicateDetector.set_builder(BrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py index c9f70ba71..62b605b35 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py @@ -7,7 +7,6 @@ from .abc import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, ) @@ -33,17 +32,3 @@ async def _is_valid(self, topic: str, uuid: UUID) -> bool: self._seen.add((topic, uuid)) return True return False - - -class InMemoryBrokerSubscriberDuplicateDetectorBuilder(BrokerSubscriberDuplicateDetectorBuilder): - """In Memory Broker Subscriber Queue Builder class.""" - - def build(self) -> InMemoryBrokerSubscriberDuplicateDetector: - """Build the instance. - - :return: An ``InMemoryBrokerSubscriberQueue`` instance. - """ - return InMemoryBrokerSubscriberDuplicateDetector(**self.kwargs) - - -InMemoryBrokerSubscriberDuplicateDetector.set_builder(InMemoryBrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py index 9d1c3015e..c8d9fc17e 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py @@ -17,13 +17,13 @@ ) from minos.common import ( + Builder, Config, PostgreSqlMinosDatabase, ) from .abc import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, ) @@ -38,10 +38,6 @@ def __init__( super().__init__(*args, **kwargs) self._query_factory = query_factory - @classmethod - def _from_config(cls, config: Config, **kwargs) -> PostgreSqlBrokerSubscriberDuplicateDetector: - return cls(**config.get_database_by_name("broker"), **kwargs) - async def _setup(self) -> None: await super()._setup() await self._create_table() @@ -72,7 +68,7 @@ async def _is_valid(self, topic: str, uuid: UUID) -> bool: return False -class PostgreSqlBrokerSubscriberDuplicateDetectorBuilder(BrokerSubscriberDuplicateDetectorBuilder): +class PostgreSqlBrokerSubscriberDuplicateDetectorBuilder(Builder[PostgreSqlBrokerSubscriberDuplicateDetector]): """TODO""" def with_config(self, config: Config): @@ -84,13 +80,6 @@ def with_config(self, config: Config): self.kwargs |= config.get_database_by_name("broker") return super().with_config(config) - def build(self) -> PostgreSqlBrokerSubscriberDuplicateDetector: - """Build the instance. - - :return: A ``BrokerSubscriberQueue`` instance. - """ - return PostgreSqlBrokerSubscriberDuplicateDetector(**self.kwargs) - PostgreSqlBrokerSubscriberDuplicateDetector.set_builder(PostgreSqlBrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py index 19f8b327f..93d3511f9 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py @@ -1,3 +1,7 @@ +from minos.common import ( + Builder, +) + from ...messages import ( BrokerMessage, ) @@ -39,4 +43,4 @@ async def _receive(self) -> BrokerMessage: return message -IdempotentBrokerSubscriber.set_builder(None) +IdempotentBrokerSubscriber.set_builder(Builder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py index 86606c7ea..135da099d 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py @@ -41,11 +41,9 @@ async def _receive(self) -> BrokerMessage: return await self._queue.get() -class InMemoryBrokerSubscriberBuilder(BrokerSubscriberBuilder): +class InMemoryBrokerSubscriberBuilder(BrokerSubscriberBuilder[InMemoryBrokerSubscriber]): """In Memory Broker Subscriber Builder class.""" - impl_cls = InMemoryBrokerSubscriber - def with_messages(self, messages: Iterable[BrokerMessage]) -> InMemoryBrokerSubscriberBuilder: """Set messages. @@ -54,3 +52,6 @@ def with_messages(self, messages: Iterable[BrokerMessage]) -> InMemoryBrokerSubs """ self.kwargs["messages"] = messages return self + + +InMemoryBrokerSubscriber.set_builder(InMemoryBrokerSubscriberBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py index eb7438fcc..4dc98d56e 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py @@ -1,3 +1,7 @@ +from __future__ import ( + annotations, +) + from asyncio import ( CancelledError, TimeoutError, @@ -18,6 +22,7 @@ ) from minos.common import ( + Builder, Config, ) @@ -81,7 +86,7 @@ def _receive(self) -> Awaitable[BrokerMessage]: return self.queue.dequeue() -class QueuedBrokerSubscriberBuilder(BrokerSubscriberBuilder): +class QueuedBrokerSubscriberBuilder(Builder[QueuedBrokerSubscriber]): """Queued Broker Subscriber Publisher class.""" def __init__( @@ -91,7 +96,7 @@ def __init__( self.impl_builder = impl_builder self.queue_builder = queue_builder - def with_config(self, config: Config) -> BrokerSubscriberBuilder: + def with_config(self, config: Config) -> QueuedBrokerSubscriberBuilder: """Set config. :param config: The config to be set. @@ -101,7 +106,7 @@ def with_config(self, config: Config) -> BrokerSubscriberBuilder: self.queue_builder.with_config(config) return self - def with_kwargs(self, kwargs: dict[str, Any]) -> BrokerSubscriberBuilder: + def with_kwargs(self, kwargs: dict[str, Any]) -> QueuedBrokerSubscriberBuilder: """Set kwargs. :param kwargs: The kwargs to be set. @@ -111,7 +116,7 @@ def with_kwargs(self, kwargs: dict[str, Any]) -> BrokerSubscriberBuilder: self.queue_builder.with_kwargs(kwargs) return self - def with_topics(self, topics: Iterable[str]) -> BrokerSubscriberBuilder: + def with_topics(self, topics: Iterable[str]) -> QueuedBrokerSubscriberBuilder: """Set topics. :param topics: The topics to be set. @@ -122,7 +127,7 @@ def with_topics(self, topics: Iterable[str]) -> BrokerSubscriberBuilder: self.queue_builder.with_topics(topics) return self - def with_group_id(self, group_id: Optional[str]) -> BrokerSubscriberBuilder: + def with_group_id(self, group_id: Optional[str]) -> QueuedBrokerSubscriberBuilder: """Set group_id. :param group_id: The group_id to be set. @@ -131,7 +136,7 @@ def with_group_id(self, group_id: Optional[str]) -> BrokerSubscriberBuilder: self.impl_builder.with_group_id(group_id) return self - def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> BrokerSubscriberBuilder: + def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> QueuedBrokerSubscriberBuilder: """Set remove_topics_on_destroy. :param remove_topics_on_destroy: The remove_topics_on_destroy flag to be set. @@ -140,7 +145,7 @@ def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> Broke self.impl_builder.with_remove_topics_on_destroy(remove_topics_on_destroy) return self - def build(self) -> BrokerSubscriber: + def build(self) -> QueuedBrokerSubscriber: """Build the instance. :return: A ``QueuedBrokerSubscriber`` instance. diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py index 82508d88c..2d88c61be 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py @@ -10,6 +10,7 @@ Iterable, ) from typing import ( + Generic, TypeVar, ) @@ -44,7 +45,10 @@ def topics(self) -> set[str]: return self._topics -class BrokerSubscriberQueueBuilder(Builder[BrokerSubscriberQueue], ABC): +BrokerSubscriberQueueCls = TypeVar("BrokerSubscriberQueueCls", bound=BrokerSubscriberQueue) + + +class BrokerSubscriberQueueBuilder(Builder[BrokerSubscriberQueueCls], Generic[BrokerSubscriberQueueCls]): """Broker Subscriber Queue Builder class.""" def with_topics(self: B, topics: Iterable[str]) -> B: diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py index 82db7a979..1d8e5c474 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py @@ -19,15 +19,8 @@ class InMemoryBrokerSubscriberQueue(InMemoryBrokerQueue, BrokerSubscriberQueue): """In Memory Broker Subscriber Queue class.""" -class InMemoryBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder): +class InMemoryBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder[InMemoryBrokerSubscriberQueue]): """In Memory Broker Subscriber Queue Builder class.""" - def build(self) -> BrokerSubscriberQueue: - """Build the instance. - - :return: An ``InMemoryBrokerSubscriberQueue`` instance. - """ - return InMemoryBrokerSubscriberQueue(**self.kwargs) - InMemoryBrokerSubscriberQueue.set_builder(InMemoryBrokerSubscriberQueueBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py index d0f952431..da5003067 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py @@ -127,7 +127,7 @@ def build_select_not_processed(self) -> SQL: ) -class PostgreSqlBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder): +class PostgreSqlBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder[PostgreSqlBrokerSubscriberQueue]): """PostgreSql Broker Subscriber Queue Builder class.""" def with_config(self, config: Config): @@ -140,12 +140,5 @@ def with_config(self, config: Config): self.kwargs |= config.get_interface_by_name("broker")["common"]["queue"] return super().with_config(config) - def build(self) -> PostgreSqlBrokerSubscriberQueue: - """Build the instance. - - :return: A ``BrokerSubscriberQueue`` instance. - """ - return PostgreSqlBrokerSubscriberQueue(**self.kwargs) - PostgreSqlBrokerSubscriberQueue.set_builder(PostgreSqlBrokerSubscriberQueueBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/utils.py b/packages/core/minos-microservice-networks/minos/networks/utils.py index f4a111ee4..d2984217d 100644 --- a/packages/core/minos-microservice-networks/minos/networks/utils.py +++ b/packages/core/minos-microservice-networks/minos/networks/utils.py @@ -5,9 +5,6 @@ import re import socket import warnings -from abc import ( - ABC, -) from asyncio import ( QueueEmpty, ) @@ -59,7 +56,7 @@ async def consume_queue(queue, max_count: int) -> None: break -class Builder(CommonBuilder, ABC): +class Builder(CommonBuilder): """Builder class.""" def __init__(self, *args, **kwargs): diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py index 7c0e7ccea..90541b7fd 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py @@ -19,14 +19,12 @@ class _BrokerSubscriber(BrokerSubscriber): + """For testing purposes.""" + async def _receive(self) -> BrokerMessage: """For testing purposes.""" -class _BrokerSubscriberBuilder(BrokerSubscriberBuilder): - impl_cls = _BrokerSubscriber - - class TestBrokerSubscriber(unittest.IsolatedAsyncioTestCase): def test_abstract(self): self.assertTrue(issubclass(BrokerSubscriber, (ABC, SetupMixin))) @@ -69,23 +67,23 @@ async def test_aiter(self): class TestBrokerSubscriberBuilder(unittest.TestCase): def test_with_group_id(self): - builder = _BrokerSubscriberBuilder().with_group_id("foobar") - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_group_id("foobar") + self.assertIsInstance(builder, BrokerSubscriberBuilder) self.assertEqual({"group_id": "foobar"}, builder.kwargs) def test_with_remove_topics_on_destroy(self): - builder = _BrokerSubscriberBuilder().with_remove_topics_on_destroy(False) - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_remove_topics_on_destroy(False) + self.assertIsInstance(builder, BrokerSubscriberBuilder) self.assertEqual({"remove_topics_on_destroy": False}, builder.kwargs) def test_with_topics(self): - builder = _BrokerSubscriberBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_topics({"one", "two"}) + self.assertIsInstance(builder, BrokerSubscriberBuilder) self.assertEqual({"topics": {"one", "two"}}, builder.kwargs) def test_build(self): - builder = _BrokerSubscriberBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_cls(_BrokerSubscriber).with_topics({"one", "two"}) + self.assertIsInstance(builder, BrokerSubscriberBuilder) subscriber = builder.build() self.assertIsInstance(subscriber, _BrokerSubscriber) self.assertEqual({"one", "two"}, subscriber.topics) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py index 4ac0151fe..2ebcd2f80 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py @@ -3,9 +3,6 @@ ABC, ) -from minos.common import ( - SetupMixin, -) from minos.networks import ( BrokerMessage, BrokerQueue, @@ -24,14 +21,6 @@ async def _dequeue(self) -> BrokerMessage: """For testing purposes.""" -class _BrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder): - """For testing purposes.""" - - def build(self) -> BrokerSubscriberQueue: - """For testing purposes.""" - return _BrokerSubscriberQueue(**self.kwargs) - - class TestBrokerSubscriberQueue(unittest.IsolatedAsyncioTestCase): def setUp(self) -> None: self.topics = {"foo", "bar"} @@ -51,19 +40,14 @@ def test_topics_raises(self): class TestBrokerSubscriberBuilder(unittest.TestCase): - def test_abstract(self): - self.assertTrue(issubclass(BrokerSubscriberQueueBuilder, (ABC, SetupMixin))) - # noinspection PyUnresolvedReferences - self.assertEqual({"build"}, BrokerSubscriberQueueBuilder.__abstractmethods__) - def test_with_topics(self): - builder = _BrokerSubscriberQueueBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberQueueBuilder) + builder = BrokerSubscriberQueueBuilder().with_topics({"one", "two"}) + self.assertIsInstance(builder, BrokerSubscriberQueueBuilder) self.assertEqual({"topics": {"one", "two"}}, builder.kwargs) def test_build(self): - builder = _BrokerSubscriberQueueBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberQueueBuilder) + builder = BrokerSubscriberQueueBuilder().with_topics({"one", "two"}).with_cls(_BrokerSubscriberQueue) + self.assertIsInstance(builder, BrokerSubscriberQueueBuilder) subscriber = builder.build() self.assertIsInstance(subscriber, _BrokerSubscriberQueue) self.assertEqual({"one", "two"}, subscriber.topics) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py b/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py index 41eb5c0b5..5e4cc21fd 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py @@ -37,13 +37,9 @@ def test_is_subclass(self): self.assertTrue(issubclass(Builder, CommonBuilder)) def test_warnings(self): - class _Builder(Builder): - def build(self) -> None: - """For testing purpose""" - with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - builder = _Builder() + builder = Builder() self.assertIsInstance(builder, CommonBuilder) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py index 764d33469..fca2844ed 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py @@ -150,11 +150,9 @@ def client(self) -> AIOKafkaConsumer: ) -class KafkaBrokerSubscriberBuilder(BrokerSubscriberBuilder): +class KafkaBrokerSubscriberBuilder(BrokerSubscriberBuilder[KafkaBrokerSubscriber]): """Kafka Broker Subscriber Builder class.""" - impl_cls = KafkaBrokerSubscriber - def with_config(self, config: Config) -> KafkaBrokerSubscriberBuilder: """Set config. From 41e3a522cd6bee4607d5c96199c09cb0e95064d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 10:29:10 +0100 Subject: [PATCH 04/17] ISSUE #150 * Deprecate `QueuedBrokerSubscriberBuilder` class. --- .../minos/common/builders.py | 45 ++++-- .../tests/test_common/test_builders.py | 39 ++---- .../minos-microservice-common/tests/utils.py | 6 - .../minos/networks/__init__.py | 1 + .../minos/networks/brokers/__init__.py | 1 + .../networks/brokers/subscribers/__init__.py | 2 +- .../minos/networks/brokers/subscribers/abc.py | 131 ++++++++++++++---- .../subscribers/idempotent/__init__.py | 2 +- .../idempotent/detectors/__init__.py | 2 +- .../subscribers/idempotent/detectors/abc.py | 8 -- .../idempotent/detectors/memory.py | 15 -- .../subscribers/idempotent/detectors/pg.py | 15 +- .../brokers/subscribers/idempotent/impl.py | 6 +- .../networks/brokers/subscribers/memory.py | 7 +- .../brokers/subscribers/queued/impl.py | 24 +++- .../brokers/subscribers/queued/queues/abc.py | 6 +- .../subscribers/queued/queues/memory.py | 9 +- .../brokers/subscribers/queued/queues/pg.py | 9 +- .../minos/networks/utils.py | 5 +- .../test_brokers/test_subscribers/test_abc.py | 22 ++- .../test_queued/test_queues/test_abc.py | 24 +--- .../tests/test_networks/test_utils.py | 6 +- .../minos/plugins/kafka/subscriber.py | 4 +- 23 files changed, 206 insertions(+), 183 deletions(-) diff --git a/packages/core/minos-microservice-common/minos/common/builders.py b/packages/core/minos-microservice-common/minos/common/builders.py index 1902e65aa..c2621f3cb 100644 --- a/packages/core/minos-microservice-common/minos/common/builders.py +++ b/packages/core/minos-microservice-common/minos/common/builders.py @@ -4,12 +4,13 @@ from abc import ( ABC, - abstractmethod, ) from typing import ( Any, Generic, + Optional, TypeVar, + get_args, ) from .config import ( @@ -19,22 +20,41 @@ SetupMixin, ) -Instance = TypeVar("Instance") +Instance = TypeVar("Instance", bound=type) class Builder(SetupMixin, ABC, Generic[Instance]): """Builder class.""" - def __init__(self, *args, **kwargs): + def __init__(self, instance_cls: Optional[type[Instance]] = None, *args, **kwargs): + super().__init__(*args, **kwargs) + if instance_cls is None: + instance_cls = self._get_cls() + self.kwargs = dict() + self.instance_cls = instance_cls + + def _get_cls(self) -> Optional[type]: + # noinspection PyUnresolvedReferences + bases = self.__orig_bases__ + + try: + instance_cls = get_args(next((base for base in bases if len(get_args(base))), None))[0] + except Exception: + return None + + if not isinstance(instance_cls, type): + return None + + return instance_cls def copy(self: type[B]) -> B: """Get a copy of the instance. :return: A ``BrokerSubscriberBuilder`` instance. """ - return self.new().with_kwargs(self.kwargs) + return self.new().with_cls(self.instance_cls).with_kwargs(self.kwargs) @classmethod def new(cls: type[B]) -> B: @@ -44,6 +64,15 @@ def new(cls: type[B]) -> B: """ return cls() + def with_cls(self: B, cls: type) -> B: + """TODO + + :param cls: TODO + :return: TODO + """ + self.instance_cls = cls + return self + def with_kwargs(self: B, kwargs: dict[str, Any]) -> B: """Set kwargs. @@ -62,12 +91,12 @@ def with_config(self: B, config: Config) -> B: """ return self - @abstractmethod def build(self) -> Instance: """Build the instance. :return: A ``BrokerSubscriber`` instance. """ + return self.instance_cls(**self.kwargs) Ins = TypeVar("Ins", bound="BuildableMixin") @@ -76,11 +105,11 @@ def build(self) -> Instance: class BuildableMixin(SetupMixin): """Buildable Mixin class.""" - _builder_cls: type[Builder[Ins]] + _builder_cls: type[Builder[Ins]] = Builder @classmethod - def _from_config(cls: type[Ins], config: Config, **kwargs) -> Ins: - return cls.get_builder().new().with_config(config).with_kwargs(kwargs).build() + def _from_config(cls, config: Config, **kwargs): + return cls.get_builder().new().with_cls(cls).with_config(config).with_kwargs(kwargs).build() @classmethod def set_builder(cls: type[Ins], builder: type[Builder[Ins]]) -> None: diff --git a/packages/core/minos-microservice-common/tests/test_common/test_builders.py b/packages/core/minos-microservice-common/tests/test_common/test_builders.py index a8918f1e4..1cf34af71 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_builders.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_builders.py @@ -1,59 +1,42 @@ import unittest -from abc import ( - ABC, -) -from typing import ( - Any, -) from minos.common import ( Builder, Config, - SetupMixin, ) from tests.utils import ( CONFIG_FILE_PATH, ) -class _Builder(Builder[dict[str, Any]]): - def build(self) -> dict[str, Any]: - """For testing purposes.""" - return self.kwargs - - class TestBuilder(unittest.TestCase): - def test_abstract(self): - self.assertTrue(issubclass(Builder, (ABC, SetupMixin))) - # noinspection PyUnresolvedReferences - self.assertEqual({"build"}, Builder.__abstractmethods__) - def test_new(self): - builder = _Builder.new() - self.assertIsInstance(builder, _Builder) + builder = Builder.new().with_cls(dict) + self.assertIsInstance(builder, Builder) self.assertEqual(dict(), builder.kwargs) def test_copy(self): - base = _Builder.new().with_kwargs({"one": "two"}) + base = Builder.new().with_cls(dict).with_kwargs({"one": "two"}) builder = base.copy() self.assertNotEqual(id(base), id(builder)) - self.assertIsInstance(builder, _Builder) + self.assertIsInstance(builder, Builder) self.assertEqual({"one": "two"}, builder.kwargs) + self.assertEqual(dict, builder.instance_cls) def test_with_kwargs(self): - builder = _Builder().with_kwargs({"foo": "bar"}) - self.assertIsInstance(builder, _Builder) + builder = Builder().with_cls(dict).with_kwargs({"foo": "bar"}) + self.assertIsInstance(builder, Builder) self.assertEqual({"foo": "bar"}, builder.kwargs) def test_with_config(self): config = Config(CONFIG_FILE_PATH) - builder = _Builder().with_config(config) - self.assertIsInstance(builder, _Builder) + builder = Builder().with_cls(dict).with_config(config) + self.assertIsInstance(builder, Builder) self.assertEqual(dict(), builder.kwargs) def test_build(self): - builder = _Builder().with_kwargs({"one": "two"}) - self.assertIsInstance(builder, _Builder) + builder = Builder().with_cls(dict).with_kwargs({"one": "two"}) + self.assertIsInstance(builder, Builder) self.assertEqual({"one": "two"}, builder.build()) diff --git a/packages/core/minos-microservice-common/tests/utils.py b/packages/core/minos-microservice-common/tests/utils.py index 5356ed0db..d8eac9c62 100644 --- a/packages/core/minos-microservice-common/tests/utils.py +++ b/packages/core/minos-microservice-common/tests/utils.py @@ -132,9 +132,6 @@ class FakeBrokerPublisher(BuildableMixin): class FakeBrokerPublisherBuilder(Builder[FakeBrokerPublisher]): """For testing purposes.""" - def build(self) -> FakeBrokerPublisher: - return FakeBrokerPublisher() - FakeBrokerPublisher.set_builder(FakeBrokerPublisherBuilder) @@ -147,9 +144,6 @@ class FakeBrokerSubscriber(BuildableMixin): class FakeBrokerSubscriberBuilder(Builder[FakeBrokerSubscriber]): """For testing purposes.""" - def build(self) -> FakeBrokerSubscriber: - return FakeBrokerSubscriber() - FakeBrokerSubscriber.set_builder(FakeBrokerSubscriberBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/__init__.py b/packages/core/minos-microservice-networks/minos/networks/__init__.py index 427ae6ad5..9fc9f6c33 100644 --- a/packages/core/minos-microservice-networks/minos/networks/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/__init__.py @@ -39,6 +39,7 @@ PostgreSqlBrokerPublisherQueueQueryFactory, PostgreSqlBrokerQueue, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, PostgreSqlBrokerSubscriberQueue, PostgreSqlBrokerSubscriberQueueBuilder, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py index 6681a48fd..87d63f13c 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py @@ -50,6 +50,7 @@ InMemoryBrokerSubscriberQueue, InMemoryBrokerSubscriberQueueBuilder, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, PostgreSqlBrokerSubscriberQueue, PostgreSqlBrokerSubscriberQueueBuilder, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py index 63364dcd4..3b4bbdf56 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/__init__.py @@ -4,10 +4,10 @@ ) from .idempotent import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, IdempotentBrokerSubscriber, InMemoryBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, ) from .memory import ( diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index 029eb7e9f..577b5cd13 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -11,13 +11,13 @@ AsyncIterator, Iterable, ) -from contextlib import ( - suppress, -) from typing import ( TYPE_CHECKING, Any, + Generic, Optional, + TypeVar, + Union, ) from minos.common import ( @@ -34,7 +34,6 @@ if TYPE_CHECKING: from .idempotent import ( - BrokerSubscriberDuplicateDetectorBuilder, IdempotentBrokerSubscriber, ) from .queued import ( @@ -82,42 +81,116 @@ async def _receive(self) -> BrokerMessage: raise NotImplementedError +BrokerSubscriberCls = TypeVar("BrokerSubscriberCls", bound=BrokerSubscriber) + + @Injectable("broker_subscriber_builder") -class BrokerSubscriberBuilder(Builder[BrokerSubscriber]): +class BrokerSubscriberBuilder(Builder[BrokerSubscriberCls], Generic[BrokerSubscriberCls]): """Broker Subscriber Builder class.""" - impl_cls: type[BrokerSubscriber] - idempotent_cls: type[IdempotentBrokerSubscriber] - queued_cls: type[QueuedBrokerSubscriber] - def __init__( self, *args, - idempotent_builder: Optional[BrokerSubscriberDuplicateDetectorBuilder] = None, + idempotent_builder: Optional[Builder] = None, queue_builder: Optional[BrokerSubscriberQueueBuilder] = None, + idempotent_cls: Optional[type[IdempotentBrokerSubscriber]] = None, + queued_cls: Optional[type[QueuedBrokerSubscriber]] = None, **kwargs, ): super().__init__(*args, **kwargs) + + if idempotent_cls is None: + from .idempotent import ( + IdempotentBrokerSubscriber, + ) + + idempotent_cls = IdempotentBrokerSubscriber + + if queued_cls is None: + from .queued import ( + QueuedBrokerSubscriber, + ) + + queued_cls = QueuedBrokerSubscriber + self.duplicate_detector_builder = idempotent_builder self.queue_builder = queue_builder + self.idempotent_cls = idempotent_cls + self.queued_cls = queued_cls + + def with_idempotent_cls(self, idempotent_cls: type[IdempotentBrokerSubscriber]): + """TODO + + :param idempotent_cls: TODO + :return: TODO + """ + self.idempotent_cls = idempotent_cls + + return self + + def with_queued_cls(self, queued_cls: type[QueuedBrokerSubscriber]): + """TODO + + :param queued_cls: TODO + :return: TODO + """ + self.queued_cls = queued_cls + + return self + def with_config(self, config: Config): """Set config. :param config: The config to be set. :return: This method return the builder instance. """ - with suppress(MinosConfigException): - broker_config = config.get_interface_by_name("broker") - broker_subscriber_config = broker_config.get("subscriber", None) - if broker_subscriber_config is not None and broker_subscriber_config.get("idempotent", None) is not None: - self.duplicate_detector_builder = ( - broker_subscriber_config.get("idempotent").get_builder().new().with_config(config) - ) - if broker_subscriber_config is not None and broker_subscriber_config.get("queue", None) is not None: - self.queue_builder = broker_subscriber_config.get("queue").get_builder().new().with_config(config) + self._with_builders_from_config(config) + + if self.duplicate_detector_builder is not None: + self.duplicate_detector_builder.with_config(config) + if self.queue_builder is not None: + self.queue_builder.with_config(config) return super().with_config(config) + def _with_builders_from_config(self, config): + try: + broker_config = config.get_interface_by_name("broker") + except MinosConfigException: + return + + broker_subscriber_config = broker_config["subscriber"] + + if "idempotent" in broker_subscriber_config: + self.with_duplicate_detector(broker_subscriber_config["idempotent"]) + + if "queue" in broker_subscriber_config: + self.with_queue(broker_subscriber_config["queue"]) + + def with_duplicate_detector( + self, duplicate_detector: Union[IdempotentBrokerSubscriber, Builder[IdempotentBrokerSubscriber]] + ): + """TODO + + :param duplicate_detector: TODO + :return: TODO + """ + if not isinstance(duplicate_detector, Builder): + duplicate_detector = duplicate_detector.get_builder() + self.duplicate_detector_builder = duplicate_detector.new() + return self + + def with_queue(self, queue: Union[QueuedBrokerSubscriber, BrokerSubscriberQueueBuilder]): + """TODO + + :param queue: TODO + :return: TODO + """ + if not isinstance(queue, Builder): + queue = queue.get_builder() + self.queue_builder = queue.new() + return self + def with_kwargs(self, kwargs: dict[str, Any]): """Set kwargs. @@ -126,8 +199,10 @@ def with_kwargs(self, kwargs: dict[str, Any]): """ if self.duplicate_detector_builder is not None: self.duplicate_detector_builder.with_kwargs(kwargs) + if self.queue_builder is not None: self.queue_builder.with_kwargs(kwargs) + return super().with_kwargs(kwargs) def with_group_id(self, group_id: Optional[str]): @@ -137,6 +212,7 @@ def with_group_id(self, group_id: Optional[str]): :return: This method return the builder instance. """ self.kwargs["group_id"] = group_id + return self def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool): @@ -146,6 +222,7 @@ def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool): :return: This method return the builder instance. """ self.kwargs["remove_topics_on_destroy"] = remove_topics_on_destroy + return self def with_topics(self, topics: Iterable[str]): @@ -156,8 +233,10 @@ def with_topics(self, topics: Iterable[str]): """ topics = set(topics) self.kwargs["topics"] = set(topics) + if self.queue_builder is not None: self.queue_builder.with_topics(topics) + return self def build(self) -> BrokerSubscriber: @@ -165,23 +244,15 @@ def build(self) -> BrokerSubscriber: :return: A ``QueuedBrokerSubscriber`` instance. """ - impl = self.impl_cls(**self.kwargs) + impl = super().build() if self.duplicate_detector_builder is not None: - from .idempotent import ( - IdempotentBrokerSubscriber, - ) - duplicate_detector = self.duplicate_detector_builder.build() - impl = IdempotentBrokerSubscriber(impl=impl, duplicate_detector=duplicate_detector, **self.kwargs) + impl = self.idempotent_cls(impl=impl, duplicate_detector=duplicate_detector, **self.kwargs) if self.queue_builder is not None: - from .queued import ( - QueuedBrokerSubscriber, - ) - queue = self.queue_builder.build() - impl = QueuedBrokerSubscriber(impl=impl, queue=queue, **self.kwargs) + impl = self.queued_cls(impl=impl, queue=queue, **self.kwargs) return impl diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py index b8f819724..10ac806c8 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/__init__.py @@ -1,8 +1,8 @@ from .detectors import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, InMemoryBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, ) from .impl import ( diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py index 3c0371535..9a9e01451 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/__init__.py @@ -1,11 +1,11 @@ from .abc import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, ) from .memory import ( InMemoryBrokerSubscriberDuplicateDetector, ) from .pg import ( PostgreSqlBrokerSubscriberDuplicateDetector, + PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, ) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py index e80b1a858..154d5c3fa 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/abc.py @@ -12,7 +12,6 @@ from minos.common import ( BuildableMixin, - Builder, ) from ....messages import ( @@ -34,10 +33,3 @@ async def is_valid(self, message: BrokerMessage) -> bool: @abstractmethod async def _is_valid(self, topic: str, uuid: UUID) -> bool: raise NotImplementedError - - -class BrokerSubscriberDuplicateDetectorBuilder(Builder[BrokerSubscriberDuplicateDetector], ABC): - """TODO""" - - -BrokerSubscriberDuplicateDetector.set_builder(BrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py index c9f70ba71..62b605b35 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/memory.py @@ -7,7 +7,6 @@ from .abc import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, ) @@ -33,17 +32,3 @@ async def _is_valid(self, topic: str, uuid: UUID) -> bool: self._seen.add((topic, uuid)) return True return False - - -class InMemoryBrokerSubscriberDuplicateDetectorBuilder(BrokerSubscriberDuplicateDetectorBuilder): - """In Memory Broker Subscriber Queue Builder class.""" - - def build(self) -> InMemoryBrokerSubscriberDuplicateDetector: - """Build the instance. - - :return: An ``InMemoryBrokerSubscriberQueue`` instance. - """ - return InMemoryBrokerSubscriberDuplicateDetector(**self.kwargs) - - -InMemoryBrokerSubscriberDuplicateDetector.set_builder(InMemoryBrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py index 9d1c3015e..c8d9fc17e 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py @@ -17,13 +17,13 @@ ) from minos.common import ( + Builder, Config, PostgreSqlMinosDatabase, ) from .abc import ( BrokerSubscriberDuplicateDetector, - BrokerSubscriberDuplicateDetectorBuilder, ) @@ -38,10 +38,6 @@ def __init__( super().__init__(*args, **kwargs) self._query_factory = query_factory - @classmethod - def _from_config(cls, config: Config, **kwargs) -> PostgreSqlBrokerSubscriberDuplicateDetector: - return cls(**config.get_database_by_name("broker"), **kwargs) - async def _setup(self) -> None: await super()._setup() await self._create_table() @@ -72,7 +68,7 @@ async def _is_valid(self, topic: str, uuid: UUID) -> bool: return False -class PostgreSqlBrokerSubscriberDuplicateDetectorBuilder(BrokerSubscriberDuplicateDetectorBuilder): +class PostgreSqlBrokerSubscriberDuplicateDetectorBuilder(Builder[PostgreSqlBrokerSubscriberDuplicateDetector]): """TODO""" def with_config(self, config: Config): @@ -84,13 +80,6 @@ def with_config(self, config: Config): self.kwargs |= config.get_database_by_name("broker") return super().with_config(config) - def build(self) -> PostgreSqlBrokerSubscriberDuplicateDetector: - """Build the instance. - - :return: A ``BrokerSubscriberQueue`` instance. - """ - return PostgreSqlBrokerSubscriberDuplicateDetector(**self.kwargs) - PostgreSqlBrokerSubscriberDuplicateDetector.set_builder(PostgreSqlBrokerSubscriberDuplicateDetectorBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py index 19f8b327f..93d3511f9 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/impl.py @@ -1,3 +1,7 @@ +from minos.common import ( + Builder, +) + from ...messages import ( BrokerMessage, ) @@ -39,4 +43,4 @@ async def _receive(self) -> BrokerMessage: return message -IdempotentBrokerSubscriber.set_builder(None) +IdempotentBrokerSubscriber.set_builder(Builder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py index 86606c7ea..135da099d 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/memory.py @@ -41,11 +41,9 @@ async def _receive(self) -> BrokerMessage: return await self._queue.get() -class InMemoryBrokerSubscriberBuilder(BrokerSubscriberBuilder): +class InMemoryBrokerSubscriberBuilder(BrokerSubscriberBuilder[InMemoryBrokerSubscriber]): """In Memory Broker Subscriber Builder class.""" - impl_cls = InMemoryBrokerSubscriber - def with_messages(self, messages: Iterable[BrokerMessage]) -> InMemoryBrokerSubscriberBuilder: """Set messages. @@ -54,3 +52,6 @@ def with_messages(self, messages: Iterable[BrokerMessage]) -> InMemoryBrokerSubs """ self.kwargs["messages"] = messages return self + + +InMemoryBrokerSubscriber.set_builder(InMemoryBrokerSubscriberBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py index eb7438fcc..4a2d83d45 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py @@ -1,3 +1,8 @@ +from __future__ import ( + annotations, +) + +import warnings from asyncio import ( CancelledError, TimeoutError, @@ -81,17 +86,22 @@ def _receive(self) -> Awaitable[BrokerMessage]: return self.queue.dequeue() -class QueuedBrokerSubscriberBuilder(BrokerSubscriberBuilder): +class QueuedBrokerSubscriberBuilder(BrokerSubscriberBuilder[QueuedBrokerSubscriber]): """Queued Broker Subscriber Publisher class.""" def __init__( self, *args, impl_builder: BrokerSubscriberBuilder, queue_builder: BrokerSubscriberQueueBuilder, **kwargs ): + warnings.warn( + f"{type(self)!r} has been deprecated. Use {BrokerSubscriberBuilder} instead.", + DeprecationWarning + ) + super().__init__(*args, **kwargs) self.impl_builder = impl_builder self.queue_builder = queue_builder - def with_config(self, config: Config) -> BrokerSubscriberBuilder: + def with_config(self, config: Config) -> QueuedBrokerSubscriberBuilder: """Set config. :param config: The config to be set. @@ -101,7 +111,7 @@ def with_config(self, config: Config) -> BrokerSubscriberBuilder: self.queue_builder.with_config(config) return self - def with_kwargs(self, kwargs: dict[str, Any]) -> BrokerSubscriberBuilder: + def with_kwargs(self, kwargs: dict[str, Any]) -> QueuedBrokerSubscriberBuilder: """Set kwargs. :param kwargs: The kwargs to be set. @@ -111,7 +121,7 @@ def with_kwargs(self, kwargs: dict[str, Any]) -> BrokerSubscriberBuilder: self.queue_builder.with_kwargs(kwargs) return self - def with_topics(self, topics: Iterable[str]) -> BrokerSubscriberBuilder: + def with_topics(self, topics: Iterable[str]) -> QueuedBrokerSubscriberBuilder: """Set topics. :param topics: The topics to be set. @@ -122,7 +132,7 @@ def with_topics(self, topics: Iterable[str]) -> BrokerSubscriberBuilder: self.queue_builder.with_topics(topics) return self - def with_group_id(self, group_id: Optional[str]) -> BrokerSubscriberBuilder: + def with_group_id(self, group_id: Optional[str]) -> QueuedBrokerSubscriberBuilder: """Set group_id. :param group_id: The group_id to be set. @@ -131,7 +141,7 @@ def with_group_id(self, group_id: Optional[str]) -> BrokerSubscriberBuilder: self.impl_builder.with_group_id(group_id) return self - def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> BrokerSubscriberBuilder: + def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> QueuedBrokerSubscriberBuilder: """Set remove_topics_on_destroy. :param remove_topics_on_destroy: The remove_topics_on_destroy flag to be set. @@ -140,7 +150,7 @@ def with_remove_topics_on_destroy(self, remove_topics_on_destroy: bool) -> Broke self.impl_builder.with_remove_topics_on_destroy(remove_topics_on_destroy) return self - def build(self) -> BrokerSubscriber: + def build(self) -> QueuedBrokerSubscriber: """Build the instance. :return: A ``QueuedBrokerSubscriber`` instance. diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py index 82508d88c..2d88c61be 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py @@ -10,6 +10,7 @@ Iterable, ) from typing import ( + Generic, TypeVar, ) @@ -44,7 +45,10 @@ def topics(self) -> set[str]: return self._topics -class BrokerSubscriberQueueBuilder(Builder[BrokerSubscriberQueue], ABC): +BrokerSubscriberQueueCls = TypeVar("BrokerSubscriberQueueCls", bound=BrokerSubscriberQueue) + + +class BrokerSubscriberQueueBuilder(Builder[BrokerSubscriberQueueCls], Generic[BrokerSubscriberQueueCls]): """Broker Subscriber Queue Builder class.""" def with_topics(self: B, topics: Iterable[str]) -> B: diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py index 82db7a979..1d8e5c474 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/memory.py @@ -19,15 +19,8 @@ class InMemoryBrokerSubscriberQueue(InMemoryBrokerQueue, BrokerSubscriberQueue): """In Memory Broker Subscriber Queue class.""" -class InMemoryBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder): +class InMemoryBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder[InMemoryBrokerSubscriberQueue]): """In Memory Broker Subscriber Queue Builder class.""" - def build(self) -> BrokerSubscriberQueue: - """Build the instance. - - :return: An ``InMemoryBrokerSubscriberQueue`` instance. - """ - return InMemoryBrokerSubscriberQueue(**self.kwargs) - InMemoryBrokerSubscriberQueue.set_builder(InMemoryBrokerSubscriberQueueBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py index d0f952431..da5003067 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py @@ -127,7 +127,7 @@ def build_select_not_processed(self) -> SQL: ) -class PostgreSqlBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder): +class PostgreSqlBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder[PostgreSqlBrokerSubscriberQueue]): """PostgreSql Broker Subscriber Queue Builder class.""" def with_config(self, config: Config): @@ -140,12 +140,5 @@ def with_config(self, config: Config): self.kwargs |= config.get_interface_by_name("broker")["common"]["queue"] return super().with_config(config) - def build(self) -> PostgreSqlBrokerSubscriberQueue: - """Build the instance. - - :return: A ``BrokerSubscriberQueue`` instance. - """ - return PostgreSqlBrokerSubscriberQueue(**self.kwargs) - PostgreSqlBrokerSubscriberQueue.set_builder(PostgreSqlBrokerSubscriberQueueBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/utils.py b/packages/core/minos-microservice-networks/minos/networks/utils.py index f4a111ee4..d2984217d 100644 --- a/packages/core/minos-microservice-networks/minos/networks/utils.py +++ b/packages/core/minos-microservice-networks/minos/networks/utils.py @@ -5,9 +5,6 @@ import re import socket import warnings -from abc import ( - ABC, -) from asyncio import ( QueueEmpty, ) @@ -59,7 +56,7 @@ async def consume_queue(queue, max_count: int) -> None: break -class Builder(CommonBuilder, ABC): +class Builder(CommonBuilder): """Builder class.""" def __init__(self, *args, **kwargs): diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py index 7c0e7ccea..90541b7fd 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py @@ -19,14 +19,12 @@ class _BrokerSubscriber(BrokerSubscriber): + """For testing purposes.""" + async def _receive(self) -> BrokerMessage: """For testing purposes.""" -class _BrokerSubscriberBuilder(BrokerSubscriberBuilder): - impl_cls = _BrokerSubscriber - - class TestBrokerSubscriber(unittest.IsolatedAsyncioTestCase): def test_abstract(self): self.assertTrue(issubclass(BrokerSubscriber, (ABC, SetupMixin))) @@ -69,23 +67,23 @@ async def test_aiter(self): class TestBrokerSubscriberBuilder(unittest.TestCase): def test_with_group_id(self): - builder = _BrokerSubscriberBuilder().with_group_id("foobar") - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_group_id("foobar") + self.assertIsInstance(builder, BrokerSubscriberBuilder) self.assertEqual({"group_id": "foobar"}, builder.kwargs) def test_with_remove_topics_on_destroy(self): - builder = _BrokerSubscriberBuilder().with_remove_topics_on_destroy(False) - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_remove_topics_on_destroy(False) + self.assertIsInstance(builder, BrokerSubscriberBuilder) self.assertEqual({"remove_topics_on_destroy": False}, builder.kwargs) def test_with_topics(self): - builder = _BrokerSubscriberBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_topics({"one", "two"}) + self.assertIsInstance(builder, BrokerSubscriberBuilder) self.assertEqual({"topics": {"one", "two"}}, builder.kwargs) def test_build(self): - builder = _BrokerSubscriberBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberBuilder) + builder = BrokerSubscriberBuilder().with_cls(_BrokerSubscriber).with_topics({"one", "two"}) + self.assertIsInstance(builder, BrokerSubscriberBuilder) subscriber = builder.build() self.assertIsInstance(subscriber, _BrokerSubscriber) self.assertEqual({"one", "two"}, subscriber.topics) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py index 4ac0151fe..2ebcd2f80 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py @@ -3,9 +3,6 @@ ABC, ) -from minos.common import ( - SetupMixin, -) from minos.networks import ( BrokerMessage, BrokerQueue, @@ -24,14 +21,6 @@ async def _dequeue(self) -> BrokerMessage: """For testing purposes.""" -class _BrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder): - """For testing purposes.""" - - def build(self) -> BrokerSubscriberQueue: - """For testing purposes.""" - return _BrokerSubscriberQueue(**self.kwargs) - - class TestBrokerSubscriberQueue(unittest.IsolatedAsyncioTestCase): def setUp(self) -> None: self.topics = {"foo", "bar"} @@ -51,19 +40,14 @@ def test_topics_raises(self): class TestBrokerSubscriberBuilder(unittest.TestCase): - def test_abstract(self): - self.assertTrue(issubclass(BrokerSubscriberQueueBuilder, (ABC, SetupMixin))) - # noinspection PyUnresolvedReferences - self.assertEqual({"build"}, BrokerSubscriberQueueBuilder.__abstractmethods__) - def test_with_topics(self): - builder = _BrokerSubscriberQueueBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberQueueBuilder) + builder = BrokerSubscriberQueueBuilder().with_topics({"one", "two"}) + self.assertIsInstance(builder, BrokerSubscriberQueueBuilder) self.assertEqual({"topics": {"one", "two"}}, builder.kwargs) def test_build(self): - builder = _BrokerSubscriberQueueBuilder().with_topics({"one", "two"}) - self.assertIsInstance(builder, _BrokerSubscriberQueueBuilder) + builder = BrokerSubscriberQueueBuilder().with_topics({"one", "two"}).with_cls(_BrokerSubscriberQueue) + self.assertIsInstance(builder, BrokerSubscriberQueueBuilder) subscriber = builder.build() self.assertIsInstance(subscriber, _BrokerSubscriberQueue) self.assertEqual({"one", "two"}, subscriber.topics) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py b/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py index 41eb5c0b5..5e4cc21fd 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_utils.py @@ -37,13 +37,9 @@ def test_is_subclass(self): self.assertTrue(issubclass(Builder, CommonBuilder)) def test_warnings(self): - class _Builder(Builder): - def build(self) -> None: - """For testing purpose""" - with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - builder = _Builder() + builder = Builder() self.assertIsInstance(builder, CommonBuilder) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py index 764d33469..fca2844ed 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py @@ -150,11 +150,9 @@ def client(self) -> AIOKafkaConsumer: ) -class KafkaBrokerSubscriberBuilder(BrokerSubscriberBuilder): +class KafkaBrokerSubscriberBuilder(BrokerSubscriberBuilder[KafkaBrokerSubscriber]): """Kafka Broker Subscriber Builder class.""" - impl_cls = KafkaBrokerSubscriber - def with_config(self, config: Config) -> KafkaBrokerSubscriberBuilder: """Set config. From 31ad8efe47f202c47263adac2530d45dfd64b2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 10:42:39 +0100 Subject: [PATCH 05/17] ISSUE #150 * Fix bug related with builder construction. --- .../minos/networks/brokers/subscribers/abc.py | 8 ++++---- .../minos/networks/brokers/subscribers/queued/impl.py | 6 +----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index 577b5cd13..e2731ee26 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -176,8 +176,8 @@ def with_duplicate_detector( :return: TODO """ if not isinstance(duplicate_detector, Builder): - duplicate_detector = duplicate_detector.get_builder() - self.duplicate_detector_builder = duplicate_detector.new() + duplicate_detector = duplicate_detector.get_builder().new().with_cls(duplicate_detector) + self.duplicate_detector_builder = duplicate_detector.copy() return self def with_queue(self, queue: Union[QueuedBrokerSubscriber, BrokerSubscriberQueueBuilder]): @@ -187,8 +187,8 @@ def with_queue(self, queue: Union[QueuedBrokerSubscriber, BrokerSubscriberQueueB :return: TODO """ if not isinstance(queue, Builder): - queue = queue.get_builder() - self.queue_builder = queue.new() + queue = queue.get_builder().new().with_cls(queue) + self.queue_builder = queue.copy() return self def with_kwargs(self, kwargs: dict[str, Any]): diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py index af42b1f54..c6fbd6737 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/impl.py @@ -23,7 +23,6 @@ ) from minos.common import ( - Builder, Config, ) @@ -93,10 +92,7 @@ class QueuedBrokerSubscriberBuilder(BrokerSubscriberBuilder[QueuedBrokerSubscrib def __init__( self, *args, impl_builder: BrokerSubscriberBuilder, queue_builder: BrokerSubscriberQueueBuilder, **kwargs ): - warnings.warn( - f"{type(self)!r} has been deprecated. Use {BrokerSubscriberBuilder} instead.", - DeprecationWarning - ) + warnings.warn(f"{type(self)!r} has been deprecated. Use {BrokerSubscriberBuilder} instead.", DeprecationWarning) super().__init__(*args, **kwargs) self.impl_builder = impl_builder From 0164ca41326d9668de7bdc5ae8e173ffc03ca9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 10:57:15 +0100 Subject: [PATCH 06/17] ISSUE #150 * Improve `Builder` class. --- .../minos/common/builders.py | 29 +++++++++++++++---- .../minos/common/config/v2.py | 9 +++--- .../test_config/test_v2/test_base.py | 2 +- .../minos/networks/brokers/subscribers/abc.py | 8 ++--- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/packages/core/minos-microservice-common/minos/common/builders.py b/packages/core/minos-microservice-common/minos/common/builders.py index c2621f3cb..08e2f0b38 100644 --- a/packages/core/minos-microservice-common/minos/common/builders.py +++ b/packages/core/minos-microservice-common/minos/common/builders.py @@ -10,6 +10,7 @@ Generic, Optional, TypeVar, + Union, get_args, ) @@ -98,6 +99,12 @@ def build(self) -> Instance: """ return self.instance_cls(**self.kwargs) + def __eq__(self, other: Any) -> bool: + return isinstance(other, type(self)) and self.instance_cls == other.instance_cls and self.kwargs == other.kwargs + + def __repr__(self) -> str: + return f"{type(self)}({self.instance_cls!r}, {self.kwargs!r})" + Ins = TypeVar("Ins", bound="BuildableMixin") @@ -105,28 +112,38 @@ def build(self) -> Instance: class BuildableMixin(SetupMixin): """Buildable Mixin class.""" - _builder_cls: type[Builder[Ins]] = Builder + _builder: Union[Builder[Ins], type[Builder[Ins]]] = Builder @classmethod def _from_config(cls, config: Config, **kwargs): - return cls.get_builder().new().with_cls(cls).with_config(config).with_kwargs(kwargs).build() + return cls.get_builder().with_config(config).with_kwargs(kwargs).build() @classmethod - def set_builder(cls: type[Ins], builder: type[Builder[Ins]]) -> None: + def set_builder(cls: type[Ins], builder: Union[Builder[Ins], type[Builder[Ins]]]) -> None: """Set a builder class. :param builder: The builder class to be set. :return: This method does not return anything. """ - cls._builder_cls = builder + if not isinstance(builder, Builder) and not (isinstance(builder, type) and issubclass(builder, Builder)): + raise ValueError(f"Given builder value is invalid: {builder!r}") + + cls._builder = builder @classmethod - def get_builder(cls) -> type[Builder[Ins]]: + def get_builder(cls) -> Builder[Ins]: """Get the builder class. :return: A ``Builder`` subclass. """ - return cls._builder_cls + builder = cls._builder + + if isinstance(builder, Builder): + builder = builder.copy() + else: + builder = builder.new() + + return builder.with_cls(cls) B = TypeVar("B", bound=Builder) diff --git a/packages/core/minos-microservice-common/minos/common/config/v2.py b/packages/core/minos-microservice-common/minos/common/config/v2.py index 4325ebbcd..92a5f2839 100644 --- a/packages/core/minos-microservice-common/minos/common/config/v2.py +++ b/packages/core/minos-microservice-common/minos/common/config/v2.py @@ -14,6 +14,7 @@ from typing import ( TYPE_CHECKING, Any, + Union, ) from ..exceptions import ( @@ -42,7 +43,7 @@ def _version(self) -> int: def _get_name(self) -> str: return self.get_by_key("name") - def _get_injections(self) -> list[type[InjectableMixin]]: + def _get_injections(self) -> list[Union[InjectableMixin, type[InjectableMixin]]]: from ..builders import ( BuildableMixin, ) @@ -85,15 +86,15 @@ def _get_injections(self) -> list[type[InjectableMixin]]: if ( not issubclass(type_, InjectableMixin) and issubclass(type_, BuildableMixin) - and issubclass((builder_type := type_.get_builder()), InjectableMixin) + and isinstance((builder_type := type_.get_builder()), InjectableMixin) ): type_ = builder_type - - if not issubclass(type_, InjectableMixin): + elif not issubclass(type_, InjectableMixin): raise MinosConfigException(f"{type_!r} must be subclass of {InjectableMixin!r}.") ans.append(type_) + # noinspection PyTypeChecker return ans def _get_databases(self) -> dict[str, dict[str, Any]]: diff --git a/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py b/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py index d9207bcca..86228f72a 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py @@ -65,7 +65,7 @@ def test_injections(self): FakeBrokerClientPool, FakeHttpConnector, FakeBrokerPublisher, - FakeBrokerSubscriberBuilder, + FakeBrokerSubscriberBuilder(FakeBrokerSubscriber), FakeEventRepository, FakeSnapshotRepository, FakeTransactionRepository, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index e2731ee26..2567eea32 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -168,7 +168,7 @@ def _with_builders_from_config(self, config): self.with_queue(broker_subscriber_config["queue"]) def with_duplicate_detector( - self, duplicate_detector: Union[IdempotentBrokerSubscriber, Builder[IdempotentBrokerSubscriber]] + self, duplicate_detector: Union[type[IdempotentBrokerSubscriber], Builder[IdempotentBrokerSubscriber]] ): """TODO @@ -176,18 +176,18 @@ def with_duplicate_detector( :return: TODO """ if not isinstance(duplicate_detector, Builder): - duplicate_detector = duplicate_detector.get_builder().new().with_cls(duplicate_detector) + duplicate_detector = duplicate_detector.get_builder() self.duplicate_detector_builder = duplicate_detector.copy() return self - def with_queue(self, queue: Union[QueuedBrokerSubscriber, BrokerSubscriberQueueBuilder]): + def with_queue(self, queue: Union[type[QueuedBrokerSubscriber], BrokerSubscriberQueueBuilder]): """TODO :param queue: TODO :return: TODO """ if not isinstance(queue, Builder): - queue = queue.get_builder().new().with_cls(queue) + queue = queue.get_builder() self.queue_builder = queue.copy() return self From 34daa7101b1da18329a389fadf94735c67e08f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 12:11:30 +0100 Subject: [PATCH 07/17] ISSUE #150 * Add `BrokerPublisherBuilder`. --- .../minos/networks/__init__.py | 2 + .../minos/networks/brokers/__init__.py | 2 + .../networks/brokers/collections/__init__.py | 1 + .../brokers/collections/queues/__init__.py | 1 + .../brokers/collections/queues/abc.py | 4 +- .../networks/brokers/collections/queues/pg.py | 18 +++ .../networks/brokers/publishers/__init__.py | 1 + .../minos/networks/brokers/publishers/abc.py | 122 +++++++++++++++++- .../brokers/publishers/queued/impl.py | 7 + .../brokers/subscribers/queued/queues/abc.py | 3 +- .../brokers/subscribers/queued/queues/pg.py | 19 +-- .../minos/plugins/kafka/__init__.py | 2 +- .../minos/plugins/kafka/common.py | 44 +++++++ .../minos/plugins/kafka/mixins.py | 18 --- .../minos/plugins/kafka/publisher.py | 20 +-- .../minos/plugins/kafka/subscriber.py | 24 +--- 16 files changed, 217 insertions(+), 71 deletions(-) create mode 100644 packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py delete mode 100644 packages/plugins/minos-broker-kafka/minos/plugins/kafka/mixins.py diff --git a/packages/core/minos-microservice-networks/minos/networks/__init__.py b/packages/core/minos-microservice-networks/minos/networks/__init__.py index 9fc9f6c33..bd0e70be5 100644 --- a/packages/core/minos-microservice-networks/minos/networks/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/__init__.py @@ -16,6 +16,7 @@ BrokerMessageV1Status, BrokerMessageV1Strategy, BrokerPublisher, + BrokerPublisherBuilder, BrokerPublisherQueue, BrokerQueue, BrokerRequest, @@ -38,6 +39,7 @@ PostgreSqlBrokerPublisherQueue, PostgreSqlBrokerPublisherQueueQueryFactory, PostgreSqlBrokerQueue, + PostgreSqlBrokerQueueBuilder, PostgreSqlBrokerSubscriberDuplicateDetector, PostgreSqlBrokerSubscriberDuplicateDetectorBuilder, PostgreSqlBrokerSubscriberDuplicateDetectorQueryFactory, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py index 87d63f13c..382de106f 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/__init__.py @@ -5,6 +5,7 @@ BrokerQueue, InMemoryBrokerQueue, PostgreSqlBrokerQueue, + PostgreSqlBrokerQueueBuilder, ) from .dispatchers import ( BrokerDispatcher, @@ -30,6 +31,7 @@ ) from .publishers import ( BrokerPublisher, + BrokerPublisherBuilder, BrokerPublisherQueue, InMemoryBrokerPublisher, InMemoryBrokerPublisherQueue, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/__init__.py index bef343971..924871e4d 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/__init__.py @@ -2,5 +2,6 @@ BrokerQueue, InMemoryBrokerQueue, PostgreSqlBrokerQueue, + PostgreSqlBrokerQueueBuilder, PostgreSqlBrokerQueueQueryFactory, ) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/__init__.py index a889b2ff7..6bba1e258 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/__init__.py @@ -6,5 +6,6 @@ ) from .pg import ( PostgreSqlBrokerQueue, + PostgreSqlBrokerQueueBuilder, PostgreSqlBrokerQueueQueryFactory, ) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/abc.py index a63082438..6d81a058b 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/abc.py @@ -12,7 +12,7 @@ ) from minos.common import ( - SetupMixin, + BuildableMixin, ) from ...messages import ( @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) -class BrokerQueue(ABC, SetupMixin): +class BrokerQueue(ABC, BuildableMixin): """Broker Queue class.""" async def enqueue(self, message: BrokerMessage) -> None: diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py index dc393a397..d963bb7d8 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py @@ -40,6 +40,7 @@ ) from ....utils import ( + Builder, consume_queue, ) from ...messages import ( @@ -199,6 +200,23 @@ async def _dequeue_rows(self, cursor: Cursor) -> list[Any]: return await cursor.fetchall() +class PostgreSqlBrokerQueueBuilder(Builder): + """TODO""" + + def with_config(self, config: Config): + """Set config. + + :param config: The config to be set. + :return: This method return the builder instance. + """ + self.kwargs |= config.get_database_by_name("broker") + self.kwargs |= config.get_interface_by_name("broker")["common"]["queue"] + return super().with_config(config) + + +PostgreSqlBrokerQueue.set_builder(PostgreSqlBrokerQueueBuilder) + + class PostgreSqlBrokerQueueQueryFactory(ABC): """PostgreSql Broker Queue Query Factory class.""" diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/__init__.py b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/__init__.py index 50253e07f..7d3037037 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/__init__.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/__init__.py @@ -1,5 +1,6 @@ from .abc import ( BrokerPublisher, + BrokerPublisherBuilder, ) from .memory import ( InMemoryBrokerPublisher, diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py index 44c033b7e..b49b88df1 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py @@ -1,23 +1,43 @@ +from __future__ import ( + annotations, +) + import logging from abc import ( ABC, abstractmethod, ) +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Optional, + TypeVar, + Union, +) from minos.common import ( + BuildableMixin, + Builder, + Config, Injectable, - SetupMixin, + MinosConfigException, ) from ..messages import ( BrokerMessage, ) +if TYPE_CHECKING: + from .queued import ( + QueuedBrokerPublisher, + ) + logger = logging.getLogger(__name__) @Injectable("broker_publisher") -class BrokerPublisher(ABC, SetupMixin): +class BrokerPublisher(ABC, BuildableMixin): """Broker Publisher class.""" async def send(self, message: BrokerMessage) -> None: @@ -32,3 +52,101 @@ async def send(self, message: BrokerMessage) -> None: @abstractmethod async def _send(self, message: BrokerMessage) -> None: raise NotImplementedError + + +BrokerPublisherCls = TypeVar("BrokerPublisherCls", bound=BrokerPublisher) + + +class BrokerPublisherBuilder(Builder[BrokerPublisher], Generic[BrokerPublisherCls]): + """Broker Publisher Builder class.""" + + def __init__( + self, + *args, + queue_builder: Optional[Builder] = None, + queued_cls: Optional[type[QueuedBrokerPublisher]] = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + + if queued_cls is None: + from .queued import ( + QueuedBrokerPublisher, + ) + + queued_cls = QueuedBrokerPublisher + + self.queue_builder = queue_builder + + self.queued_cls = queued_cls + + def with_queued_cls(self, queued_cls: type[QueuedBrokerPublisher]): + """TODO + + :param queued_cls: TODO + :return: TODO + """ + self.queued_cls = queued_cls + + return self + + def with_config(self, config: Config): + """Set config. + + :param config: The config to be set. + :return: This method return the builder instance. + """ + self._with_builders_from_config(config) + + if self.queue_builder is not None: + self.queue_builder.with_config(config) + return super().with_config(config) + + def _with_builders_from_config(self, config): + try: + broker_config = config.get_interface_by_name("broker") + except MinosConfigException: + return + + broker_publisher_config = broker_config["publisher"] + + if "queue" in broker_publisher_config: + self.with_queue(broker_publisher_config["queue"]) + + def with_queue(self, queue: Union[type[QueuedBrokerPublisher], Builder[QueuedBrokerPublisher]]): + """TODO + + :param queue: TODO + :return: TODO + """ + if not isinstance(queue, Builder): + queue = queue.get_builder() + self.queue_builder = queue.copy() + return self + + def with_kwargs(self, kwargs: dict[str, Any]): + """Set kwargs. + + :param kwargs: The kwargs to be set. + :return: This method return the builder instance. + """ + if self.queue_builder is not None: + self.queue_builder.with_kwargs(kwargs) + + return super().with_kwargs(kwargs) + + def build(self) -> BrokerPublisher: + """Build the instance. + + :return: A ``QueuedBrokerSubscriber`` instance. + """ + impl = super().build() + + if self.queue_builder is not None: + queue = self.queue_builder.build() + impl = self.queued_cls(impl=impl, queue=queue, **self.kwargs) + + return impl + + +BrokerPublisher.set_builder(BrokerPublisherBuilder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/queued/impl.py b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/queued/impl.py index f4c1a44b3..2d709c2e1 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/queued/impl.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/queued/impl.py @@ -11,6 +11,10 @@ NoReturn, ) +from minos.common import ( + Builder, +) + from ...messages import ( BrokerMessage, ) @@ -65,3 +69,6 @@ async def _run(self) -> NoReturn: async def _send(self, message: BrokerMessage) -> None: await self.queue.enqueue(message) + + +QueuedBrokerPublisher.set_builder(Builder) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py index 2d88c61be..2e933ee97 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/abc.py @@ -15,7 +15,6 @@ ) from minos.common import ( - BuildableMixin, Builder, ) @@ -26,7 +25,7 @@ logger = logging.getLogger(__name__) -class BrokerSubscriberQueue(BrokerQueue, BuildableMixin, ABC): +class BrokerSubscriberQueue(BrokerQueue, ABC): """Broker Subscriber Queue class.""" def __init__(self, topics: Iterable[str], **kwargs): diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py index da5003067..1cd1cdeec 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py @@ -16,12 +16,9 @@ Identifier, ) -from minos.common import ( - Config, -) - from ....collections import ( PostgreSqlBrokerQueue, + PostgreSqlBrokerQueueBuilder, PostgreSqlBrokerQueueQueryFactory, ) from ....messages import ( @@ -127,18 +124,10 @@ def build_select_not_processed(self) -> SQL: ) -class PostgreSqlBrokerSubscriberQueueBuilder(BrokerSubscriberQueueBuilder[PostgreSqlBrokerSubscriberQueue]): +class PostgreSqlBrokerSubscriberQueueBuilder( + BrokerSubscriberQueueBuilder[PostgreSqlBrokerSubscriberQueue], PostgreSqlBrokerQueueBuilder +): """PostgreSql Broker Subscriber Queue Builder class.""" - def with_config(self, config: Config): - """Set config. - - :param config: The config to be set. - :return: This method return the builder instance. - """ - self.kwargs |= config.get_database_by_name("broker") - self.kwargs |= config.get_interface_by_name("broker")["common"]["queue"] - return super().with_config(config) - PostgreSqlBrokerSubscriberQueue.set_builder(PostgreSqlBrokerSubscriberQueueBuilder) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py index d4dff278b..033d04d35 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py @@ -2,7 +2,7 @@ __email__ = "hey@minos.run" __version__ = "0.5.1" -from .mixins import ( +from .common import ( KafkaCircuitBreakerMixin, ) from .publisher import ( diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py new file mode 100644 index 000000000..19473c3f9 --- /dev/null +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py @@ -0,0 +1,44 @@ +from __future__ import ( + annotations, +) + +from collections.abc import ( + Iterable, +) + +from kafka.errors import ( + KafkaError, +) + +from minos.common import ( + Builder, + CircuitBreakerMixin, + Config, +) + + +class KafkaCircuitBreakerMixin(CircuitBreakerMixin): + """Kafka Circuit Breaker Mixin class.""" + + def __init__(self, *args, circuit_breaker_exceptions: Iterable[type] = tuple(), **kwargs): + super().__init__(*args, circuit_breaker_exceptions=(KafkaError, *circuit_breaker_exceptions), **kwargs) + + +class KafkaBrokerBuilderMixin(Builder): + """TODO""" + + def with_config(self, config: Config): + """Set config. + + :param config: The config to be set. + :return: This method return the builder instance. + """ + broker_config = config.get_interface_by_name("broker") + common_config = broker_config["common"] + + self.kwargs |= { + "group_id": config.get_name(), + "broker_host": common_config["host"], + "broker_port": common_config["port"], + } + return super().with_config(config) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/mixins.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/mixins.py deleted file mode 100644 index 1879f5ecb..000000000 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/mixins.py +++ /dev/null @@ -1,18 +0,0 @@ -from collections.abc import ( - Iterable, -) - -from kafka.errors import ( - KafkaError, -) - -from minos.common import ( - CircuitBreakerMixin, -) - - -class KafkaCircuitBreakerMixin(CircuitBreakerMixin): - """Kafka Circuit Breaker Mixin class.""" - - def __init__(self, *args, circuit_breaker_exceptions: Iterable[type] = tuple(), **kwargs): - super().__init__(*args, circuit_breaker_exceptions=(KafkaError, *circuit_breaker_exceptions), **kwargs) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py index 50a28f841..969ab9977 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py @@ -24,12 +24,14 @@ from minos.networks import ( BrokerMessage, BrokerPublisher, + BrokerPublisherBuilder, InMemoryBrokerPublisherQueue, PostgreSqlBrokerPublisherQueue, QueuedBrokerPublisher, ) -from .mixins import ( +from .common import ( + KafkaBrokerBuilderMixin, KafkaCircuitBreakerMixin, ) @@ -67,15 +69,6 @@ def __init__(self, *args, broker_host: str, broker_port: int, **kwargs): self._client = None - @classmethod - def _from_config(cls, config: Config, **kwargs) -> KafkaBrokerPublisher: - broker_config = config.get_interface_by_name("broker") - common_config = broker_config["common"] - - kwargs["broker_host"] = common_config["host"] - kwargs["broker_port"] = common_config["port"] - return cls(**kwargs) - async def _setup(self) -> None: await super()._setup() await self._start_client() @@ -116,3 +109,10 @@ def _build_client(self) -> AIOKafkaProducer: @property def _bootstrap_servers(self): return f"{self.broker_host}:{self.broker_port}" + + +class KafkaBrokerPublisherBuilder(BrokerPublisherBuilder[KafkaBrokerPublisher], KafkaBrokerBuilderMixin): + """TODO""" + + +KafkaBrokerPublisher.set_builder(KafkaBrokerPublisherBuilder) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py index fca2844ed..da2b1bc8d 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py @@ -36,9 +36,6 @@ TopicAlreadyExistsError, ) -from minos.common import ( - Config, -) from minos.networks import ( BrokerMessage, BrokerSubscriber, @@ -48,7 +45,8 @@ QueuedBrokerSubscriberBuilder, ) -from .mixins import ( +from .common import ( + KafkaBrokerBuilderMixin, KafkaCircuitBreakerMixin, ) @@ -150,25 +148,9 @@ def client(self) -> AIOKafkaConsumer: ) -class KafkaBrokerSubscriberBuilder(BrokerSubscriberBuilder[KafkaBrokerSubscriber]): +class KafkaBrokerSubscriberBuilder(BrokerSubscriberBuilder[KafkaBrokerSubscriber], KafkaBrokerBuilderMixin): """Kafka Broker Subscriber Builder class.""" - def with_config(self, config: Config) -> KafkaBrokerSubscriberBuilder: - """Set config. - - :param config: The config to be set. - :return: This method return the builder instance. - """ - broker_config = config.get_interface_by_name("broker") - common_config = broker_config["common"] - - self.kwargs |= { - "group_id": config.get_name(), - "broker_host": common_config["host"], - "broker_port": common_config["port"], - } - return super().with_config(config) - KafkaBrokerSubscriber.set_builder(KafkaBrokerSubscriberBuilder) From 9938e788012fd36dce838e6a8c60c59258b4b7ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 14:58:02 +0100 Subject: [PATCH 08/17] ISSUE #150 * Increase coverage. * Minor improvements. --- .../minos/common/builders.py | 7 +- .../tests/test_common/test_builders.py | 108 ++++++++++++++++++ 2 files changed, 110 insertions(+), 5 deletions(-) diff --git a/packages/core/minos-microservice-common/minos/common/builders.py b/packages/core/minos-microservice-common/minos/common/builders.py index 08e2f0b38..aa7682ec5 100644 --- a/packages/core/minos-microservice-common/minos/common/builders.py +++ b/packages/core/minos-microservice-common/minos/common/builders.py @@ -40,10 +40,7 @@ def _get_cls(self) -> Optional[type]: # noinspection PyUnresolvedReferences bases = self.__orig_bases__ - try: - instance_cls = get_args(next((base for base in bases if len(get_args(base))), None))[0] - except Exception: - return None + instance_cls = get_args(next((base for base in bases if len(get_args(base))), None))[0] if not isinstance(instance_cls, type): return None @@ -103,7 +100,7 @@ def __eq__(self, other: Any) -> bool: return isinstance(other, type(self)) and self.instance_cls == other.instance_cls and self.kwargs == other.kwargs def __repr__(self) -> str: - return f"{type(self)}({self.instance_cls!r}, {self.kwargs!r})" + return f"{type(self).__name__}({self.instance_cls.__name__}, {self.kwargs!r})" Ins = TypeVar("Ins", bound="BuildableMixin") diff --git a/packages/core/minos-microservice-common/tests/test_common/test_builders.py b/packages/core/minos-microservice-common/tests/test_common/test_builders.py index 1cf34af71..13db1f03f 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_builders.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_builders.py @@ -1,6 +1,11 @@ import unittest +from typing import ( + Generic, + TypeVar, +) from minos.common import ( + BuildableMixin, Builder, Config, ) @@ -39,6 +44,109 @@ def test_build(self): self.assertIsInstance(builder, Builder) self.assertEqual({"one": "two"}, builder.build()) + def test_str(self): + builder = Builder().with_cls(dict).with_kwargs({"one": "two"}) + + self.assertEqual("Builder(dict, {'one': 'two'})", repr(builder)) + + def test_cmp(self): + base = Builder().with_cls(dict).with_kwargs({"one": "two"}) + + one = Builder().with_cls(dict).with_kwargs({"one": "two"}) + self.assertEqual(base, one) + + two = Builder().with_cls(int).with_kwargs({"one": "two"}) + self.assertNotEqual(base, two) + + three = Builder().with_cls(dict).with_kwargs({"three": "four"}) + self.assertNotEqual(base, three) + + def test_instance_cls_from_generic(self): + class _Builder(Builder): + """For Testing purposes.""" + + class _Builder2(Builder[int]): + """For Testing purposes.""" + + class _Builder3(_Builder2): + """For Testing purposes.""" + + T = TypeVar("T") + + class _Builder4(_Builder2, Generic[T]): + """For Testing purposes.""" + + class _Builder5(_Builder4[float]): + """For Testing purposes.""" + + self.assertEqual(None, _Builder().instance_cls) + self.assertEqual(int, _Builder2().instance_cls) + self.assertEqual(int, _Builder3().instance_cls) + self.assertEqual(None, _Builder4().instance_cls) + self.assertEqual(float, _Builder5().instance_cls) + + +class TestBuildableMixin(unittest.TestCase): + def test_get_builder_default(self): + class _Foo(BuildableMixin): + """For Testing purposes.""" + + self.assertEqual(Builder().with_cls(_Foo), _Foo.get_builder()) + + def test_get_builder_custom_type(self): + class _Foo(BuildableMixin): + """For Testing purposes.""" + + class _Builder(Builder): + """For Testing purposes.""" + + _Foo.set_builder(_Builder) + + self.assertEqual(_Builder().with_cls(_Foo), _Foo.get_builder()) + + def test_get_builder_custom_instance(self): + class _Foo(BuildableMixin): + """For Testing purposes.""" + + class _Builder(Builder): + """For Testing purposes.""" + + _Foo.set_builder(_Builder().with_kwargs({"foo": "bar"})) + + self.assertEqual(_Builder().with_cls(_Foo).with_kwargs({"foo": "bar"}), _Foo.get_builder()) + + def test_set_builder_raises(self): + class _Foo(BuildableMixin): + """For Testing purposes.""" + + with self.assertRaises(ValueError): + # noinspection PyTypeChecker + _Foo.set_builder(int) + + def test_from_config(self): + class _Foo(BuildableMixin): + """For Testing purposes.""" + + # noinspection PyShadowingNames + def __init__(self, config): + super().__init__() + self.config = config + + class _Builder(Builder): + """For Testing purposes.""" + + def with_config(self, config: Config): + """For Testing purposes.""" + self.kwargs["config"] = config + return super().with_config(config) + + _Foo.set_builder(_Builder) + + config = Config(CONFIG_FILE_PATH) + foo = _Foo.from_config(config) + + self.assertEqual(config, foo.config) + if __name__ == "__main__": unittest.main() From 3ae14ecf3912a6cb2d59242831090152d684e5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 17:12:04 +0100 Subject: [PATCH 09/17] ISSUE #150 * Add tests. --- .../minos/plugins/kafka/__init__.py | 2 + .../minos/plugins/kafka/publisher.py | 9 ++++ .../tests/test_kafka/test_common.py | 43 +++++++++++++++++++ .../tests/test_kafka/test_mixins.py | 19 -------- .../tests/test_kafka/test_publisher.py | 35 ++++++++++++++- .../tests/test_kafka/test_subscriber.py | 13 +++++- 6 files changed, 98 insertions(+), 23 deletions(-) create mode 100644 packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py delete mode 100644 packages/plugins/minos-broker-kafka/tests/test_kafka/test_mixins.py diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py index 033d04d35..b6f11bf23 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py @@ -3,11 +3,13 @@ __version__ = "0.5.1" from .common import ( + KafkaBrokerBuilderMixin, KafkaCircuitBreakerMixin, ) from .publisher import ( InMemoryQueuedKafkaBrokerPublisher, KafkaBrokerPublisher, + KafkaBrokerPublisherBuilder, PostgreSqlQueuedKafkaBrokerPublisher, ) from .subscriber import ( diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py index 969ab9977..dc5f735dc 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py @@ -3,6 +3,7 @@ ) import logging +import warnings from asyncio import ( TimeoutError, wait_for, @@ -41,6 +42,10 @@ class PostgreSqlQueuedKafkaBrokerPublisher(QueuedBrokerPublisher): """PostgreSql Queued Kafka Broker Publisher class.""" + def __init__(self, *args, **kwargs): + warnings.warn(f"{PostgreSqlQueuedKafkaBrokerPublisher!r} has been deprecated.", DeprecationWarning) + super().__init__(*args, **kwargs) + @classmethod def _from_config(cls, config: Config, **kwargs) -> PostgreSqlQueuedKafkaBrokerPublisher: impl = KafkaBrokerPublisher.from_config(config, **kwargs) @@ -51,6 +56,10 @@ def _from_config(cls, config: Config, **kwargs) -> PostgreSqlQueuedKafkaBrokerPu class InMemoryQueuedKafkaBrokerPublisher(QueuedBrokerPublisher): """In Memory Queued Kafka Broker Publisher class.""" + def __init__(self, *args, **kwargs): + warnings.warn(f"{InMemoryQueuedKafkaBrokerPublisher!r} has been deprecated.", DeprecationWarning) + super().__init__(*args, **kwargs) + @classmethod def _from_config(cls, config: Config, **kwargs) -> InMemoryQueuedKafkaBrokerPublisher: impl = KafkaBrokerPublisher.from_config(config, **kwargs) diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py new file mode 100644 index 000000000..3a0055801 --- /dev/null +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py @@ -0,0 +1,43 @@ +import unittest + +from kafka.errors import ( + KafkaError, +) + +from minos.common import ( + Config, +) +from minos.plugins.kafka import ( + KafkaBrokerBuilderMixin, + KafkaCircuitBreakerMixin, +) +from tests.utils import ( + CONFIG_FILE_PATH, +) + + +class TestKafkaCircuitBreakerMixin(unittest.IsolatedAsyncioTestCase): + def test_constructor(self): + mixin = KafkaCircuitBreakerMixin() + self.assertEqual((KafkaError,), mixin.circuit_breaker_exceptions) + + +class TestKafkaBrokerBuilderMixin(unittest.IsolatedAsyncioTestCase): + def test_constructor(self): + mixin = KafkaBrokerBuilderMixin() + + config = Config(CONFIG_FILE_PATH) + mixin.with_config(config) + + common_config = config.get_interface_by_name("broker")["common"] + + expected = { + "group_id": config.get_name(), + "broker_host": common_config["host"], + "broker_port": common_config["port"], + } + self.assertEqual(expected, mixin.kwargs) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_mixins.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_mixins.py deleted file mode 100644 index 485ae199a..000000000 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_mixins.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from kafka.errors import ( - KafkaError, -) - -from minos.plugins.kafka import ( - KafkaCircuitBreakerMixin, -) - - -class TestKafkaCircuitBreakerMixin(unittest.IsolatedAsyncioTestCase): - def test_constructor(self): - mixin = KafkaCircuitBreakerMixin() - self.assertEqual((KafkaError,), mixin.circuit_breaker_exceptions) - - -if __name__ == "__main__": - unittest.main() diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py index 63aba848f..1e30f9289 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py @@ -1,4 +1,5 @@ import unittest +import warnings from unittest.mock import ( AsyncMock, ) @@ -24,6 +25,7 @@ from minos.plugins.kafka import ( InMemoryQueuedKafkaBrokerPublisher, KafkaBrokerPublisher, + KafkaBrokerPublisherBuilder, PostgreSqlQueuedKafkaBrokerPublisher, ) from tests.utils import ( @@ -119,9 +121,36 @@ async def test_setup_destroy(self): self.assertEqual(1, stop_mock.call_count) +class TestKafkaBrokerPublisherBuilder(unittest.TestCase): + def setUp(self) -> None: + self.config = Config(CONFIG_FILE_PATH) + + def test_with_config(self): + builder = KafkaBrokerPublisherBuilder().with_config(self.config) + common_config = self.config.get_interface_by_name("broker")["common"] + + expected = { + "group_id": self.config.get_name(), + "broker_host": common_config["host"], + "broker_port": common_config["port"], + } + self.assertEqual(expected, builder.kwargs) + + def test_build(self): + common_config = self.config.get_interface_by_name("broker")["common"] + builder = KafkaBrokerPublisherBuilder().with_config(self.config) + publisher = builder.build() + + self.assertIsInstance(publisher, KafkaBrokerPublisher) + self.assertEqual(common_config["host"], publisher.broker_host) + self.assertEqual(common_config["port"], publisher.broker_port) + + class TestPostgreSqlQueuedKafkaBrokerPublisher(unittest.IsolatedAsyncioTestCase): def test_from_config(self): - publisher = PostgreSqlQueuedKafkaBrokerPublisher.from_config(CONFIG_FILE_PATH) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + publisher = PostgreSqlQueuedKafkaBrokerPublisher.from_config(CONFIG_FILE_PATH) self.assertIsInstance(publisher, PostgreSqlQueuedKafkaBrokerPublisher) self.assertIsInstance(publisher.impl, KafkaBrokerPublisher) self.assertIsInstance(publisher.queue, PostgreSqlBrokerPublisherQueue) @@ -129,7 +158,9 @@ def test_from_config(self): class TestInMemoryQueuedKafkaBrokerPublisher(unittest.IsolatedAsyncioTestCase): def test_from_config(self): - publisher = InMemoryQueuedKafkaBrokerPublisher.from_config(CONFIG_FILE_PATH) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + publisher = InMemoryQueuedKafkaBrokerPublisher.from_config(CONFIG_FILE_PATH) self.assertIsInstance(publisher, InMemoryQueuedKafkaBrokerPublisher) self.assertIsInstance(publisher.impl, KafkaBrokerPublisher) self.assertIsInstance(publisher.queue, InMemoryBrokerPublisherQueue) diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py index 1f3929ade..bed9fecea 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py @@ -1,4 +1,5 @@ import unittest +import warnings from collections import ( namedtuple, ) @@ -230,7 +231,12 @@ def setUp(self) -> None: self.config = Config(CONFIG_FILE_PATH) def test_build(self): - builder = PostgreSqlQueuedKafkaBrokerSubscriberBuilder().with_config(self.config).with_topics({"one", "two"}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = ( + PostgreSqlQueuedKafkaBrokerSubscriberBuilder().with_config(self.config).with_topics({"one", "two"}) + ) + subscriber = builder.build() self.assertIsInstance(subscriber, QueuedBrokerSubscriber) @@ -243,7 +249,10 @@ def setUp(self) -> None: self.config = Config(CONFIG_FILE_PATH) def test_build(self): - builder = InMemoryQueuedKafkaBrokerSubscriberBuilder().with_config(self.config).with_topics({"one", "two"}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = InMemoryQueuedKafkaBrokerSubscriberBuilder().with_config(self.config).with_topics({"one", "two"}) + subscriber = builder.build() self.assertIsInstance(subscriber, QueuedBrokerSubscriber) From e37fe064774c86dacafd6cbd448d1bf15b3554cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 17:14:32 +0100 Subject: [PATCH 10/17] ISSUE #150 * Add tests (2). --- .../test_subscribers/test_queued/test_impl.py | 26 ++++++++++++++----- .../test_queued/test_queues/test_pg.py | 5 +++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py index 83beeffb8..2c0f2bd92 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_impl.py @@ -1,4 +1,5 @@ import unittest +import warnings from asyncio import ( sleep, ) @@ -115,7 +116,9 @@ def test_with_kwargs(self): self.impl_builder.with_kwargs = impl_mock self.queue_builder.with_kwargs = queue_mock - builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_kwargs({"foo": "bar"}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_kwargs({"foo": "bar"}) self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) self.assertEqual([call({"foo": "bar"})], impl_mock.call_args_list) @@ -127,7 +130,9 @@ def test_with_config(self): self.impl_builder.with_config = impl_mock self.queue_builder.with_config = queue_mock - builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_config(self.config) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_config(self.config) self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) self.assertEqual([call(self.config)], impl_mock.call_args_list) @@ -137,7 +142,10 @@ def test_with_group_id(self): impl_mock = MagicMock(side_effect=self.impl_builder.with_group_id) self.impl_builder.with_group_id = impl_mock - builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_group_id("foobar") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_group_id("foobar") + self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) self.assertEqual([call("foobar")], impl_mock.call_args_list) @@ -146,7 +154,9 @@ def test_with_remove_topics_on_destroy(self): impl_mock = MagicMock(side_effect=self.impl_builder.with_remove_topics_on_destroy) self.impl_builder.with_remove_topics_on_destroy = impl_mock - builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_remove_topics_on_destroy(False) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_remove_topics_on_destroy(False) self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) self.assertEqual([call(False)], impl_mock.call_args_list) @@ -157,7 +167,9 @@ def test_with_topics(self): self.impl_builder.with_topics = impl_mock self.queue_builder.with_topics = queue_mock - builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_topics({"one", "two"}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_topics({"one", "two"}) self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) self.assertEqual([call({"one", "two"})], impl_mock.call_args_list) @@ -169,7 +181,9 @@ def test_build(self): self.impl_builder.build = impl_mock self.queue_builder.build = queue_mock - builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_topics({"one", "two"}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = QueuedBrokerSubscriberBuilder(**self._kwargs).with_topics({"one", "two"}) self.assertIsInstance(builder, QueuedBrokerSubscriberBuilder) subscriber = builder.build() self.assertIsInstance(subscriber, QueuedBrokerSubscriber) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_pg.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_pg.py index bbf82b846..45e7c9786 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_pg.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_pg.py @@ -1,4 +1,5 @@ import unittest +import warnings from asyncio import ( sleep, ) @@ -90,7 +91,9 @@ def setUp(self) -> None: self.config = Config(CONFIG_FILE_PATH) def test_build(self): - builder = PostgreSqlBrokerSubscriberQueueBuilder().with_config(self.config).with_topics({"one", "two"}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + builder = PostgreSqlBrokerSubscriberQueueBuilder().with_config(self.config).with_topics({"one", "two"}) subscriber = builder.build() self.assertIsInstance(subscriber, PostgreSqlBrokerSubscriberQueue) From d0584ca96e838521a8d0b1704f8522acb8ed65b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 17:33:26 +0100 Subject: [PATCH 11/17] ISSUE #150 * Add tests for `BrokerPublisherBuilder`. --- .../minos/networks/brokers/publishers/abc.py | 4 +- .../test_brokers/test_publishers/test_abc.py | 100 +++++++++++++++++- 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py index b49b88df1..bec8573d4 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py @@ -30,7 +30,7 @@ if TYPE_CHECKING: from .queued import ( - QueuedBrokerPublisher, + QueuedBrokerPublisher,BrokerPublisherQueue ) logger = logging.getLogger(__name__) @@ -113,7 +113,7 @@ def _with_builders_from_config(self, config): if "queue" in broker_publisher_config: self.with_queue(broker_publisher_config["queue"]) - def with_queue(self, queue: Union[type[QueuedBrokerPublisher], Builder[QueuedBrokerPublisher]]): + def with_queue(self, queue: Union[type[BrokerPublisherQueue], Builder[BrokerPublisherQueue]]): """TODO :param queue: TODO diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py index d6032c2c1..f26a6728f 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py @@ -4,18 +4,25 @@ ) from unittest.mock import ( AsyncMock, - call, + call, MagicMock, ) from minos.common import ( + Builder, SetupMixin, + Config, MinosConfigException, ) from minos.networks import ( BrokerMessage, BrokerMessageV1, BrokerMessageV1Payload, BrokerPublisher, + BrokerPublisherBuilder, + InMemoryBrokerPublisherQueue, + QueuedBrokerPublisher, + InMemoryBrokerPublisher, ) +from tests.utils import CONFIG_FILE_PATH class _BrokerPublisher(BrokerPublisher): @@ -42,5 +49,96 @@ async def test_send(self): self.assertEqual([call(message)], mock.call_args_list) +class TestBrokerPublisherBuilder(unittest.TestCase): + def test_constructor(self): + builder = BrokerPublisherBuilder() + self.assertEqual(None, builder.queue_builder) + self.assertEqual(QueuedBrokerPublisher, builder.queued_cls) + + def test_with_queued_cls(self): + # noinspection PyTypeChecker + builder = BrokerPublisherBuilder().with_queued_cls(int) + self.assertEqual(int, builder.queued_cls) + + def test_constructor_with_queue_builder(self): + queue_builder = Builder().with_cls(InMemoryBrokerPublisherQueue) + builder = BrokerPublisherBuilder(queue_builder=queue_builder) + self.assertEqual(queue_builder, builder.queue_builder) + self.assertEqual(QueuedBrokerPublisher, builder.queued_cls) + + def test_with_config_none(self): + config = Config(CONFIG_FILE_PATH) + + mock = MagicMock(side_effect=MinosConfigException("")) + config.get_interface_by_name = mock + + builder = BrokerPublisherBuilder().with_config(config) + self.assertEqual(None, builder.queue_builder) + self.assertEqual({}, builder.kwargs) + + def test_with_config_empty(self): + config = Config(CONFIG_FILE_PATH) + + mock = MagicMock(return_value={"publisher": {}}) + config.get_interface_by_name = mock + + builder = BrokerPublisherBuilder().with_config(config) + self.assertEqual(None, builder.queue_builder) + self.assertEqual({}, builder.kwargs) + + def test_with_config(self): + config = Config(CONFIG_FILE_PATH) + + mock = MagicMock(return_value={"publisher": {"queue": InMemoryBrokerPublisherQueue}}) + config.get_interface_by_name = mock + + builder = BrokerPublisherBuilder().with_config(config) + self.assertEqual(Builder().with_cls(InMemoryBrokerPublisherQueue), builder.queue_builder) + self.assertEqual({}, builder.kwargs) + + def test_with_queue_with_config(self): + config = Config(CONFIG_FILE_PATH) + + builder = BrokerPublisherBuilder().with_config(config).with_queue(InMemoryBrokerPublisherQueue) + self.assertEqual({}, builder.kwargs) + self.assertEqual(Builder().with_cls(InMemoryBrokerPublisherQueue), builder.queue_builder) + + def test_with_kwargs(self): + builder = BrokerPublisherBuilder().with_kwargs({"foo": "bar"}) + self.assertEqual(None, builder.queue_builder) + self.assertEqual({"foo": "bar"}, builder.kwargs) + + def test_with_queue_with_kwargs(self): + builder = BrokerPublisherBuilder().with_queue(InMemoryBrokerPublisherQueue).with_kwargs({"foo": "bar"}) + self.assertEqual( + Builder().with_cls(InMemoryBrokerPublisherQueue).with_kwargs({"foo": "bar"}), builder.queue_builder + ) + self.assertEqual({"foo": "bar"}, builder.kwargs) + + def test_with_queue_cls(self): + queue_builder = Builder().with_cls(InMemoryBrokerPublisherQueue) + builder = BrokerPublisherBuilder().with_queue(InMemoryBrokerPublisherQueue) + self.assertEqual(queue_builder, builder.queue_builder) + + def test_with_queue_builder(self): + queue_builder = Builder().with_cls(InMemoryBrokerPublisherQueue) + builder = BrokerPublisherBuilder().with_queue(queue_builder) + self.assertEqual(queue_builder, builder.queue_builder) + + def test_build(self): + publisher = BrokerPublisherBuilder().with_cls(InMemoryBrokerPublisher).build() + + self.assertIsInstance(publisher, InMemoryBrokerPublisher) + + def test_build_with_queue(self): + publisher = ( + BrokerPublisherBuilder().with_cls(InMemoryBrokerPublisher).with_queue(InMemoryBrokerPublisherQueue).build() + ) + + self.assertIsInstance(publisher, QueuedBrokerPublisher) + self.assertIsInstance(publisher.impl, InMemoryBrokerPublisher) + self.assertIsInstance(publisher.queue, InMemoryBrokerPublisherQueue) + + if __name__ == "__main__": unittest.main() From fd3cbe0910c4e6d1ff7e20064046754f65af3562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 17:34:43 +0100 Subject: [PATCH 12/17] ISSUE #150 * Minor change. --- .../test_subscribers/test_queued/test_queues/test_abc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py index 2ebcd2f80..5de2334a3 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_queued/test_queues/test_abc.py @@ -39,7 +39,7 @@ def test_topics_raises(self): _BrokerSubscriberQueue([]) -class TestBrokerSubscriberBuilder(unittest.TestCase): +class TestBrokerSubscriberQueueBuilder(unittest.TestCase): def test_with_topics(self): builder = BrokerSubscriberQueueBuilder().with_topics({"one", "two"}) self.assertIsInstance(builder, BrokerSubscriberQueueBuilder) From dc0e3322235e062dc5875f2d5f07c23db0ed8b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 17:51:23 +0100 Subject: [PATCH 13/17] ISSUE #150 * Increase coverage. --- .../minos/networks/brokers/subscribers/abc.py | 8 +- .../test_brokers/test_publishers/test_abc.py | 2 +- .../test_brokers/test_subscribers/test_abc.py | 199 +++++++++++++++++- 3 files changed, 201 insertions(+), 8 deletions(-) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index 2567eea32..9d0994deb 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -34,13 +34,16 @@ if TYPE_CHECKING: from .idempotent import ( + BrokerSubscriberDuplicateDetector, IdempotentBrokerSubscriber, ) from .queued import ( + BrokerSubscriberQueue, BrokerSubscriberQueueBuilder, QueuedBrokerSubscriber, ) + logger = logging.getLogger(__name__) @@ -168,7 +171,8 @@ def _with_builders_from_config(self, config): self.with_queue(broker_subscriber_config["queue"]) def with_duplicate_detector( - self, duplicate_detector: Union[type[IdempotentBrokerSubscriber], Builder[IdempotentBrokerSubscriber]] + self, + duplicate_detector: Union[type[BrokerSubscriberDuplicateDetector], Builder[BrokerSubscriberDuplicateDetector]], ): """TODO @@ -180,7 +184,7 @@ def with_duplicate_detector( self.duplicate_detector_builder = duplicate_detector.copy() return self - def with_queue(self, queue: Union[type[QueuedBrokerSubscriber], BrokerSubscriberQueueBuilder]): + def with_queue(self, queue: Union[type[BrokerSubscriberQueue], BrokerSubscriberQueueBuilder]): """TODO :param queue: TODO diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py index f26a6728f..8b9e1f78e 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py @@ -99,7 +99,7 @@ def test_with_config(self): def test_with_queue_with_config(self): config = Config(CONFIG_FILE_PATH) - builder = BrokerPublisherBuilder().with_config(config).with_queue(InMemoryBrokerPublisherQueue) + builder = BrokerPublisherBuilder().with_queue(InMemoryBrokerPublisherQueue).with_config(config) self.assertEqual({}, builder.kwargs) self.assertEqual(Builder().with_cls(InMemoryBrokerPublisherQueue), builder.queue_builder) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py index 90541b7fd..aa1631d61 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_subscribers/test_abc.py @@ -4,9 +4,13 @@ ) from unittest.mock import ( AsyncMock, + MagicMock, ) from minos.common import ( + Builder, + Config, + MinosConfigException, SetupMixin, ) from minos.networks import ( @@ -15,6 +19,15 @@ BrokerMessageV1Payload, BrokerSubscriber, BrokerSubscriberBuilder, + IdempotentBrokerSubscriber, + InMemoryBrokerSubscriber, + InMemoryBrokerSubscriberDuplicateDetector, + InMemoryBrokerSubscriberQueue, + InMemoryBrokerSubscriberQueueBuilder, + QueuedBrokerSubscriber, +) +from tests.utils import ( + CONFIG_FILE_PATH, ) @@ -66,6 +79,183 @@ async def test_aiter(self): class TestBrokerSubscriberBuilder(unittest.TestCase): + def test_constructor(self): + builder = BrokerSubscriberBuilder() + self.assertEqual(None, builder.queue_builder) + self.assertEqual(None, builder.duplicate_detector_builder) + self.assertEqual(QueuedBrokerSubscriber, builder.queued_cls) + + def test_with_queued_cls(self): + # noinspection PyTypeChecker + builder = BrokerSubscriberBuilder().with_queued_cls(int) + self.assertEqual(int, builder.queued_cls) + + def test_with_idempotent_cls(self): + # noinspection PyTypeChecker + builder = BrokerSubscriberBuilder().with_idempotent_cls(int) + self.assertEqual(int, builder.idempotent_cls) + + def test_constructor_with_queue_builder(self): + queue_builder = InMemoryBrokerSubscriberQueueBuilder() + builder = BrokerSubscriberBuilder(queue_builder=queue_builder) + self.assertEqual(queue_builder, builder.queue_builder) + self.assertEqual(QueuedBrokerSubscriber, builder.queued_cls) + + def test_constructor_with_duplicate_detector(self): + idempotent_builder = Builder().with_cls(InMemoryBrokerSubscriberDuplicateDetector) + builder = BrokerSubscriberBuilder(idempotent_builder=idempotent_builder) + self.assertEqual(idempotent_builder, builder.duplicate_detector_builder) + self.assertEqual(QueuedBrokerSubscriber, builder.queued_cls) + + def test_with_config_none(self): + config = Config(CONFIG_FILE_PATH) + + mock = MagicMock(side_effect=MinosConfigException("")) + config.get_interface_by_name = mock + + builder = BrokerSubscriberBuilder().with_config(config) + self.assertEqual(None, builder.queue_builder) + self.assertEqual({}, builder.kwargs) + + def test_with_config_empty(self): + config = Config(CONFIG_FILE_PATH) + + mock = MagicMock(return_value={"subscriber": {}}) + config.get_interface_by_name = mock + + builder = BrokerSubscriberBuilder().with_config(config) + self.assertEqual(None, builder.queue_builder) + self.assertEqual({}, builder.kwargs) + + def test_with_config(self): + config = Config(CONFIG_FILE_PATH) + + mock = MagicMock( + return_value={ + "subscriber": { + "queue": InMemoryBrokerSubscriberQueue, + "idempotent": InMemoryBrokerSubscriberDuplicateDetector, + } + } + ) + config.get_interface_by_name = mock + + builder = BrokerSubscriberBuilder().with_config(config) + self.assertEqual(InMemoryBrokerSubscriberQueueBuilder(), builder.queue_builder) + self.assertEqual( + Builder().with_cls(InMemoryBrokerSubscriberDuplicateDetector), builder.duplicate_detector_builder + ) + self.assertEqual({}, builder.kwargs) + + def test_with_queue_with_config(self): + config = Config(CONFIG_FILE_PATH) + + builder = BrokerSubscriberBuilder().with_queue(InMemoryBrokerSubscriberQueue).with_config(config) + self.assertEqual({}, builder.kwargs) + self.assertEqual(InMemoryBrokerSubscriberQueueBuilder(), builder.queue_builder) + + def test_with_duplicate_with_config(self): + config = Config(CONFIG_FILE_PATH) + + builder = ( + BrokerSubscriberBuilder() + .with_duplicate_detector(InMemoryBrokerSubscriberDuplicateDetector) + .with_config(config) + ) + self.assertEqual({}, builder.kwargs) + self.assertEqual( + Builder().with_cls(InMemoryBrokerSubscriberDuplicateDetector), builder.duplicate_detector_builder + ) + + def test_with_kwargs(self): + builder = BrokerSubscriberBuilder().with_kwargs({"foo": "bar"}) + self.assertEqual(None, builder.queue_builder) + self.assertEqual({"foo": "bar"}, builder.kwargs) + + def test_with_queue_with_kwargs(self): + builder = BrokerSubscriberBuilder().with_queue(InMemoryBrokerSubscriberQueue).with_kwargs({"foo": "bar"}) + self.assertEqual(InMemoryBrokerSubscriberQueueBuilder().with_kwargs({"foo": "bar"}), builder.queue_builder) + self.assertEqual({"foo": "bar"}, builder.kwargs) + + def test_with_duplicate_detector_with_kwargs(self): + builder = ( + BrokerSubscriberBuilder() + .with_duplicate_detector(InMemoryBrokerSubscriberDuplicateDetector) + .with_kwargs({"foo": "bar"}) + ) + self.assertEqual( + Builder().with_cls(InMemoryBrokerSubscriberDuplicateDetector).with_kwargs({"foo": "bar"}), + builder.duplicate_detector_builder, + ) + self.assertEqual({"foo": "bar"}, builder.kwargs) + + def test_with_queue_cls(self): + queue_builder = InMemoryBrokerSubscriberQueueBuilder() + builder = BrokerSubscriberBuilder().with_queue(InMemoryBrokerSubscriberQueue) + self.assertEqual(queue_builder, builder.queue_builder) + + def test_with_queue_builder(self): + queue_builder = InMemoryBrokerSubscriberQueueBuilder() + builder = BrokerSubscriberBuilder().with_queue(queue_builder) + self.assertEqual(queue_builder, builder.queue_builder) + + def test_with_duplicate_detector_cls(self): + duplicate_detector_builder = Builder().with_cls(InMemoryBrokerSubscriberDuplicateDetector) + builder = BrokerSubscriberBuilder().with_duplicate_detector(InMemoryBrokerSubscriberDuplicateDetector) + self.assertEqual(duplicate_detector_builder, builder.duplicate_detector_builder) + + def test_with_duplicate_detector_builder(self): + duplicate_detector_builder = Builder().with_cls(InMemoryBrokerSubscriberDuplicateDetector) + builder = BrokerSubscriberBuilder().with_duplicate_detector(duplicate_detector_builder) + self.assertEqual(duplicate_detector_builder, builder.duplicate_detector_builder) + + def test_build(self): + subscriber = BrokerSubscriberBuilder().with_topics({"one", "two"}).with_cls(InMemoryBrokerSubscriber).build() + + self.assertIsInstance(subscriber, InMemoryBrokerSubscriber) + + def test_build_with_queue(self): + subscriber = ( + BrokerSubscriberBuilder() + .with_cls(InMemoryBrokerSubscriber) + .with_queue(InMemoryBrokerSubscriberQueue) + .with_topics({"one", "two"}) + .build() + ) + + self.assertIsInstance(subscriber, QueuedBrokerSubscriber) + self.assertIsInstance(subscriber.impl, InMemoryBrokerSubscriber) + self.assertIsInstance(subscriber.queue, InMemoryBrokerSubscriberQueue) + + def test_build_with_duplicate_detector(self): + subscriber = ( + BrokerSubscriberBuilder() + .with_cls(InMemoryBrokerSubscriber) + .with_duplicate_detector(InMemoryBrokerSubscriberDuplicateDetector) + .with_topics({"one", "two"}) + .build() + ) + + self.assertIsInstance(subscriber, IdempotentBrokerSubscriber) + self.assertIsInstance(subscriber.impl, InMemoryBrokerSubscriber) + self.assertIsInstance(subscriber.duplicate_detector, InMemoryBrokerSubscriberDuplicateDetector) + + def test_build_with_duplicate_detector_with_queue(self): + subscriber = ( + BrokerSubscriberBuilder() + .with_cls(InMemoryBrokerSubscriber) + .with_duplicate_detector(InMemoryBrokerSubscriberDuplicateDetector) + .with_queue(InMemoryBrokerSubscriberQueue) + .with_topics({"one", "two"}) + .build() + ) + self.assertIsInstance(subscriber, QueuedBrokerSubscriber) + self.assertIsInstance(subscriber.queue, InMemoryBrokerSubscriberQueue) + + self.assertIsInstance(subscriber.impl, IdempotentBrokerSubscriber) + self.assertIsInstance(subscriber.impl.impl, InMemoryBrokerSubscriber) + self.assertIsInstance(subscriber.impl.duplicate_detector, InMemoryBrokerSubscriberDuplicateDetector) + def test_with_group_id(self): builder = BrokerSubscriberBuilder().with_group_id("foobar") self.assertIsInstance(builder, BrokerSubscriberBuilder) @@ -81,12 +271,11 @@ def test_with_topics(self): self.assertIsInstance(builder, BrokerSubscriberBuilder) self.assertEqual({"topics": {"one", "two"}}, builder.kwargs) - def test_build(self): - builder = BrokerSubscriberBuilder().with_cls(_BrokerSubscriber).with_topics({"one", "two"}) + def test_with_topics_with_queue(self): + builder = BrokerSubscriberBuilder().with_queue(InMemoryBrokerSubscriberQueue).with_topics({"one", "two"}) self.assertIsInstance(builder, BrokerSubscriberBuilder) - subscriber = builder.build() - self.assertIsInstance(subscriber, _BrokerSubscriber) - self.assertEqual({"one", "two"}, subscriber.topics) + self.assertEqual(InMemoryBrokerSubscriberQueueBuilder().with_topics({"one", "two"}), builder.queue_builder) + self.assertEqual({"topics": {"one", "two"}}, builder.kwargs) if __name__ == "__main__": From f693ba58580761294ca530e8b24da70029b63c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 23 Mar 2022 17:55:18 +0100 Subject: [PATCH 14/17] ISSUE #150 * Add docstring. --- .../minos/common/builders.py | 8 +++---- .../networks/brokers/collections/queues/pg.py | 2 +- .../minos/networks/brokers/publishers/abc.py | 15 ++++++------ .../minos/networks/brokers/subscribers/abc.py | 24 +++++++++---------- .../subscribers/idempotent/detectors/pg.py | 2 +- .../test_brokers/test_publishers/test_abc.py | 12 ++++++---- .../minos/plugins/kafka/common.py | 2 +- .../minos/plugins/kafka/publisher.py | 2 +- 8 files changed, 36 insertions(+), 31 deletions(-) diff --git a/packages/core/minos-microservice-common/minos/common/builders.py b/packages/core/minos-microservice-common/minos/common/builders.py index aa7682ec5..acd1c1385 100644 --- a/packages/core/minos-microservice-common/minos/common/builders.py +++ b/packages/core/minos-microservice-common/minos/common/builders.py @@ -63,10 +63,10 @@ def new(cls: type[B]) -> B: return cls() def with_cls(self: B, cls: type) -> B: - """TODO + """Set class to be built. - :param cls: TODO - :return: TODO + :param cls: The class to be set. + :return: This method return the builder instance. """ self.instance_cls = cls return self @@ -92,7 +92,7 @@ def with_config(self: B, config: Config) -> B: def build(self) -> Instance: """Build the instance. - :return: A ``BrokerSubscriber`` instance. + :return: A ``Instance`` instance. """ return self.instance_cls(**self.kwargs) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py index d963bb7d8..ce2b48d7d 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py @@ -201,7 +201,7 @@ async def _dequeue_rows(self, cursor: Cursor) -> list[Any]: class PostgreSqlBrokerQueueBuilder(Builder): - """TODO""" + """PostgreSql Broker Queue Builder class.""" def with_config(self, config: Config): """Set config. diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py index bec8573d4..89cc8e7d7 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/publishers/abc.py @@ -30,7 +30,8 @@ if TYPE_CHECKING: from .queued import ( - QueuedBrokerPublisher,BrokerPublisherQueue + BrokerPublisherQueue, + QueuedBrokerPublisher, ) logger = logging.getLogger(__name__) @@ -81,10 +82,10 @@ def __init__( self.queued_cls = queued_cls def with_queued_cls(self, queued_cls: type[QueuedBrokerPublisher]): - """TODO + """Set the queued class. - :param queued_cls: TODO - :return: TODO + :param queued_cls: A subclass of ``QueuedBrokerPublisher``. + :return: This method return the builder instance. """ self.queued_cls = queued_cls @@ -114,10 +115,10 @@ def _with_builders_from_config(self, config): self.with_queue(broker_publisher_config["queue"]) def with_queue(self, queue: Union[type[BrokerPublisherQueue], Builder[BrokerPublisherQueue]]): - """TODO + """Set the queue builder. - :param queue: TODO - :return: TODO + :param queue: The queue builder to be set. + :return: This method return the builder instance. """ if not isinstance(queue, Builder): queue = queue.get_builder() diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py index 9d0994deb..a6e51b16e 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/abc.py @@ -123,20 +123,20 @@ def __init__( self.queued_cls = queued_cls def with_idempotent_cls(self, idempotent_cls: type[IdempotentBrokerSubscriber]): - """TODO + """Set the idempotent class. - :param idempotent_cls: TODO - :return: TODO + :param idempotent_cls: A subclass of ``IdempotentBrokerSubscriber``. + :return: This method return the builder instance. """ self.idempotent_cls = idempotent_cls return self def with_queued_cls(self, queued_cls: type[QueuedBrokerSubscriber]): - """TODO + """Set the queued class. - :param queued_cls: TODO - :return: TODO + :param queued_cls: A subclass of ``QueuedBrokerSubscriber``. + :return: This method return the builder instance. """ self.queued_cls = queued_cls @@ -174,10 +174,10 @@ def with_duplicate_detector( self, duplicate_detector: Union[type[BrokerSubscriberDuplicateDetector], Builder[BrokerSubscriberDuplicateDetector]], ): - """TODO + """Set the duplicate detector. - :param duplicate_detector: TODO - :return: TODO + :param duplicate_detector: The duplicate detector to be set. + :return: This method return the builder instance. """ if not isinstance(duplicate_detector, Builder): duplicate_detector = duplicate_detector.get_builder() @@ -185,10 +185,10 @@ def with_duplicate_detector( return self def with_queue(self, queue: Union[type[BrokerSubscriberQueue], BrokerSubscriberQueueBuilder]): - """TODO + """Set the queue builder. - :param queue: TODO - :return: TODO + :param queue: The queue to be set. + :return: This method return the builder instance. """ if not isinstance(queue, Builder): queue = queue.get_builder() diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py index c8d9fc17e..079219814 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/idempotent/detectors/pg.py @@ -69,7 +69,7 @@ async def _is_valid(self, topic: str, uuid: UUID) -> bool: class PostgreSqlBrokerSubscriberDuplicateDetectorBuilder(Builder[PostgreSqlBrokerSubscriberDuplicateDetector]): - """TODO""" + """PostgreSql Broker Subscriber Duplicate Detector Builder class.""" def with_config(self, config: Config): """Set config. diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py index 8b9e1f78e..ff6944caf 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_publishers/test_abc.py @@ -4,13 +4,15 @@ ) from unittest.mock import ( AsyncMock, - call, MagicMock, + MagicMock, + call, ) from minos.common import ( Builder, + Config, + MinosConfigException, SetupMixin, - Config, MinosConfigException, ) from minos.networks import ( BrokerMessage, @@ -18,11 +20,13 @@ BrokerMessageV1Payload, BrokerPublisher, BrokerPublisherBuilder, + InMemoryBrokerPublisher, InMemoryBrokerPublisherQueue, QueuedBrokerPublisher, - InMemoryBrokerPublisher, ) -from tests.utils import CONFIG_FILE_PATH +from tests.utils import ( + CONFIG_FILE_PATH, +) class _BrokerPublisher(BrokerPublisher): diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py index 19473c3f9..4860c3860 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/common.py @@ -25,7 +25,7 @@ def __init__(self, *args, circuit_breaker_exceptions: Iterable[type] = tuple(), class KafkaBrokerBuilderMixin(Builder): - """TODO""" + """Kafka Broker Builder Mixin class.""" def with_config(self, config: Config): """Set config. diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py index dc5f735dc..ea7759a8b 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py @@ -121,7 +121,7 @@ def _bootstrap_servers(self): class KafkaBrokerPublisherBuilder(BrokerPublisherBuilder[KafkaBrokerPublisher], KafkaBrokerBuilderMixin): - """TODO""" + """Kafka Broker Publisher Builder class.""" KafkaBrokerPublisher.set_builder(KafkaBrokerPublisherBuilder) From 577b2bbc3376cfbd91443f278eef305b0d2e462a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 24 Mar 2022 08:33:20 +0100 Subject: [PATCH 15/17] ISSUE #150 * Minor change. --- .../networks/brokers/collections/queues/pg.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py index ce2b48d7d..0e737325c 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py @@ -200,23 +200,6 @@ async def _dequeue_rows(self, cursor: Cursor) -> list[Any]: return await cursor.fetchall() -class PostgreSqlBrokerQueueBuilder(Builder): - """PostgreSql Broker Queue Builder class.""" - - def with_config(self, config: Config): - """Set config. - - :param config: The config to be set. - :return: This method return the builder instance. - """ - self.kwargs |= config.get_database_by_name("broker") - self.kwargs |= config.get_interface_by_name("broker")["common"]["queue"] - return super().with_config(config) - - -PostgreSqlBrokerQueue.set_builder(PostgreSqlBrokerQueueBuilder) - - class PostgreSqlBrokerQueueQueryFactory(ABC): """PostgreSql Broker Queue Query Factory class.""" @@ -341,3 +324,20 @@ def __lt__(self, other: Any) -> bool: return isinstance(other, type(self)) and self.data < other.data except Exception: return False + + +class PostgreSqlBrokerQueueBuilder(Builder): + """PostgreSql Broker Queue Builder class.""" + + def with_config(self, config: Config): + """Set config. + + :param config: The config to be set. + :return: This method return the builder instance. + """ + self.kwargs |= config.get_database_by_name("broker") + self.kwargs |= config.get_interface_by_name("broker")["common"]["queue"] + return super().with_config(config) + + +PostgreSqlBrokerQueue.set_builder(PostgreSqlBrokerQueueBuilder) From fcfc6cf9b8b5ba6f66c6c4a6a2b5cea578356601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 24 Mar 2022 08:40:50 +0100 Subject: [PATCH 16/17] ISSUE #150 * Minor change (2). --- .../minos-microservice-common/minos/common/builders.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/minos-microservice-common/minos/common/builders.py b/packages/core/minos-microservice-common/minos/common/builders.py index acd1c1385..73df3b18b 100644 --- a/packages/core/minos-microservice-common/minos/common/builders.py +++ b/packages/core/minos-microservice-common/minos/common/builders.py @@ -21,7 +21,7 @@ SetupMixin, ) -Instance = TypeVar("Instance", bound=type) +Instance = TypeVar("Instance") class Builder(SetupMixin, ABC, Generic[Instance]): @@ -50,7 +50,7 @@ def _get_cls(self) -> Optional[type]: def copy(self: type[B]) -> B: """Get a copy of the instance. - :return: A ``BrokerSubscriberBuilder`` instance. + :return: A ``Builder`` instance. """ return self.new().with_cls(self.instance_cls).with_kwargs(self.kwargs) @@ -58,7 +58,7 @@ def copy(self: type[B]) -> B: def new(cls: type[B]) -> B: """Get a new instance. - :return: A ``BrokerSubscriberBuilder`` instance. + :return: A ``Builder`` instance. """ return cls() @@ -131,7 +131,7 @@ def set_builder(cls: type[Ins], builder: Union[Builder[Ins], type[Builder[Ins]]] def get_builder(cls) -> Builder[Ins]: """Get the builder class. - :return: A ``Builder`` subclass. + :return: A ``Builder`` instance. """ builder = cls._builder From 0ee393c71c49f0ed489b60245ec94b66e43a7d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 24 Mar 2022 16:04:31 +0100 Subject: [PATCH 17/17] ISSUE #150 * Fix renaming bug. --- .../minos-broker-kafka/tests/test_kafka/test_common.py | 4 ++-- .../minos-broker-kafka/tests/test_kafka/test_publisher.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py index 3a0055801..a77050094 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_common.py @@ -33,8 +33,8 @@ def test_constructor(self): expected = { "group_id": config.get_name(), - "broker_host": common_config["host"], - "broker_port": common_config["port"], + "host": common_config["host"], + "port": common_config["port"], } self.assertEqual(expected, mixin.kwargs) diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py index 77fdfb038..c734086da 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py @@ -136,8 +136,8 @@ def test_with_config(self): expected = { "group_id": self.config.get_name(), - "broker_host": common_config["host"], - "broker_port": common_config["port"], + "host": common_config["host"], + "port": common_config["port"], } self.assertEqual(expected, builder.kwargs) @@ -147,8 +147,8 @@ def test_build(self): publisher = builder.build() self.assertIsInstance(publisher, KafkaBrokerPublisher) - self.assertEqual(common_config["host"], publisher.broker_host) - self.assertEqual(common_config["port"], publisher.broker_port) + self.assertEqual(common_config["host"], publisher.host) + self.assertEqual(common_config["port"], publisher.port) class TestPostgreSqlQueuedKafkaBrokerPublisher(unittest.IsolatedAsyncioTestCase):