Skip to content

[pulsar-io] pass client builder if no service url provided to debezium connector #12145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
codelipenghui merged 7 commits into from
Jan 18, 2022
Merged

[pulsar-io] pass client builder if no service url provided to debezium connector #12145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
83dd8e7
stash
freeznet Sep 23, 2021
2e8838b
only pass client builder if no service url provided
freeznet Sep 23, 2021
8acf919
add test cases in integration tests
freeznet Nov 17, 2021
4a27ce7
fix
freeznet Nov 17, 2021
33407ce
revert and keep one case
freeznet Nov 18, 2021
47a87f6
fix CI
freeznet Nov 19, 2021
365e5b6
Merge branch 'master' into freeznet/fix-debezium-client-auth
codelipenghui Jan 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/DebeziumSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
package org.apache.pulsar.io.debezium;

import io.debezium.relational.HistorizedRelationalDatabaseConnectorConfig;
import io.debezium.relational.history.DatabaseHistory;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.common.naming.TopicName;
Expand Down Expand Up @@ -49,10 +48,7 @@ public static void throwExceptionIfConfigNotMatch(Map<String, Object> config,
}

public static void setConfigIfNull(Map<String, Object> config, String key, String value) {
Object orig = config.get(key);
if (orig == null) {
config.put(key, value);
}
config.putIfAbsent(key, value);
}

// namespace for output topics, default value is "tenant/namespace"
Expand Down Expand Up @@ -80,9 +76,6 @@ public void open(Map<String, Object> config, SourceContext sourceContext) throws

// database.history.pulsar.service.url
String pulsarUrl = (String) config.get(PulsarDatabaseHistory.SERVICE_URL.name());
if (StringUtils.isEmpty(pulsarUrl)) {
throw new IllegalArgumentException("Pulsar service URL for History Database not provided.");
}

String topicNamespace = topicNamespace(sourceContext);
// topic.namespace
Expand All @@ -96,8 +89,11 @@ public void open(Map<String, Object> config, SourceContext sourceContext) throws
setConfigIfNull(config, PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG,
topicNamespace + "/" + sourceName + "-" + DEFAULT_OFFSET_TOPIC);

config.put(DatabaseHistory.CONFIGURATION_FIELD_PREFIX_STRING + "pulsar.client.builder",
SerDeUtils.serialize(sourceContext.getPulsarClientBuilder()));
// pass pulsar.client.builder if database.history.pulsar.service.url is not provided
if (StringUtils.isEmpty(pulsarUrl)) {
String pulsarClientBuilder = SerDeUtils.serialize(sourceContext.getPulsarClientBuilder());
config.put(PulsarDatabaseHistory.CLIENT_BUILDER.name(), pulsarClientBuilder);
}

super.open(config, sourceContext);
}
Expand Down
6 changes: 3 additions & 3 deletions ...r-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,12 @@ public void configure(
}
this.topicName = config.getString(TOPIC);

if (config.getString(CLIENT_BUILDER) == null && config.getString(SERVICE_URL) == null) {
String clientBuilderBase64Encoded = config.getString(CLIENT_BUILDER);
if (isBlank(clientBuilderBase64Encoded) && isBlank(config.getString(SERVICE_URL))) {
throw new IllegalArgumentException("Neither Pulsar Service URL nor ClientBuilder provided.");
}
String clientBuilderBase64Encoded = config.getString(CLIENT_BUILDER);
this.clientBuilder = PulsarClient.builder();
if (null != clientBuilderBase64Encoded) {
if (!isBlank(clientBuilderBase64Encoded)) {
// deserialize the client builder to the same classloader
this.clientBuilder = (ClientBuilder) SerDeUtils.deserialize(clientBuilderBase64Encoded,
this.clientBuilder.getClass().getClassLoader());
Expand Down
7 changes: 5 additions & 2 deletions ...va/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ public class DebeziumMySqlSourceTester extends SourceTester<DebeziumMySQLContain

private final PulsarCluster pulsarCluster;

public DebeziumMySqlSourceTester(PulsarCluster cluster, String converterClassName) {
public DebeziumMySqlSourceTester(PulsarCluster cluster, String converterClassName,
boolean testWithClientBuilder) {
super(NAME);
this.pulsarCluster = cluster;
pulsarServiceUrl = "pulsar://pulsar-proxy:" + PulsarContainer.BROKER_PORT;
Expand All @@ -61,7 +62,9 @@ public DebeziumMySqlSourceTester(PulsarCluster cluster, String converterClassNam
sourceConfig.put("database.server.id", "184054");
sourceConfig.put("database.server.name", "dbserver1");
sourceConfig.put("database.whitelist", "inventory");
sourceConfig.put("database.history.pulsar.service.url", pulsarServiceUrl);
if (!testWithClientBuilder) {
sourceConfig.put("database.history.pulsar.service.url", pulsarServiceUrl);
}
sourceConfig.put("key.converter", converterClassName);
sourceConfig.put("value.converter", converterClassName);
sourceConfig.put("topic.namespace", "debezium/mysql-" +
Expand Down
15 changes: 11 additions & 4 deletions ...va/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,20 +45,26 @@ public class PulsarDebeziumSourcesTest extends PulsarIOTestBase {

@Test(groups = "source")
public void testDebeziumMySqlSourceJson() throws Exception {
testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true);
testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true, false);
}

@Test(groups = "source")
public void testDebeziumMySqlSourceJsonWithClientBuilder() throws Exception {
testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true, true);
}

@Test(groups = "source")
public void testDebeziumMySqlSourceAvro() throws Exception {
testDebeziumMySqlConnect(
"org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false);
"org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false, false);
}

@Test(groups = "source")
public void testDebeziumPostgreSqlSource() throws Exception {
testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true);
}


@Test(groups = "source")
public void testDebeziumMongoDbSource() throws Exception{
testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true);
Expand All @@ -69,7 +75,8 @@ public void testDebeziumMsSqlSource() throws Exception{
testDebeziumMsSqlConnect("org.apache.kafka.connect.json.JsonConverter", true);
}

private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception {
private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope,
boolean testWithClientBuilder) throws Exception {

final String tenant = TopicName.PUBLIC_TENANT;
final String namespace = TopicName.DEFAULT_NAMESPACE;
Expand Down Expand Up @@ -104,7 +111,7 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit
admin.topics().createNonPartitionedTopic(outputTopicName);

@Cleanup
DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName);
DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName, testWithClientBuilder);
sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope);

// setup debezium mysql server
Expand Down