Skip to content

Commit 22a51c9

Browse files
itsankit-googleitsankit-google
authored
Merge pull request #249 from data-integrations/cherry-pick/PLUGIN-1209
[PLUGIN-1209] Failed to deploy the plugin using CloudSQL MySQL & Postgres source and sink with connection
2 parents a88dfc3 + abfd258 commit 22a51c9

13 files changed

Lines changed: 114 additions & 159 deletions

File tree

‎cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.cdap.cdap.etl.api.action.Action;
2626
import io.cdap.plugin.db.batch.action.AbstractDBAction;
2727
import io.cdap.plugin.db.batch.action.QueryConfig;
28+
import io.cdap.plugin.util.CloudSQLUtil;
2829

2930
import java.util.Map;
3031
import javax.annotation.Nullable;
@@ -48,7 +49,7 @@ public CloudSQLMySQLAction(CloudSQLMySQLActionConfig cloudsqlMysqlActionConfig)
4849
public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
4950
FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector();
5051

51-
CloudSQLMySQLUtil.checkConnectionName(
52+
CloudSQLUtil.checkConnectionName(
5253
failureCollector,
5354
cloudsqlMysqlActionConfig.instanceType,
5455
cloudsqlMysqlActionConfig.connectionName);
@@ -60,10 +61,10 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
6061
public static class CloudSQLMySQLActionConfig extends QueryConfig {
6162

6263
public CloudSQLMySQLActionConfig() {
63-
this.instanceType = CloudSQLMySQLConstants.PUBLIC_INSTANCE;
64+
this.instanceType = CloudSQLUtil.PUBLIC_INSTANCE;
6465
}
6566

66-
@Name(CloudSQLMySQLConstants.CONNECTION_NAME)
67+
@Name(CloudSQLUtil.CONNECTION_NAME)
6768
@Description(
6869
"The CloudSQL instance to connect to. For a public instance, the connection string should be in the format "
6970
+ "<PROJECT_ID>:<REGION>:<INSTANCE_NAME> which can be found in the instance overview page. For a private "
@@ -82,14 +83,14 @@ public CloudSQLMySQLActionConfig() {
8283
@Nullable
8384
public Integer connectionTimeout;
8485

85-
@Name(CloudSQLMySQLConstants.INSTANCE_TYPE)
86+
@Name(CloudSQLUtil.INSTANCE_TYPE)
8687
@Description("Whether the CloudSQL instance to connect to is private or public.")
8788
@Nullable
8889
public String instanceType;
8990

9091
@Override
9192
public String getConnectionString() {
92-
if (CloudSQLMySQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
93+
if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
9394
return String.format(
9495
CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT,
9596
connectionName,

‎cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import io.cdap.cdap.api.annotation.Name;
2121
import io.cdap.plugin.db.ConnectionConfig;
2222
import io.cdap.plugin.db.connector.AbstractDBConnectorConfig;
23+
import io.cdap.plugin.util.CloudSQLUtil;
2324

2425
import java.util.Properties;
2526
import javax.annotation.Nullable;
@@ -32,7 +33,7 @@ public class CloudSQLMySQLConnectorConfig extends AbstractDBConnectorConfig {
3233
private static final String JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS = "connectTimeout";
3334
private static final String JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS = "socketTimeout";
3435

35-
@Name(CloudSQLMySQLConstants.CONNECTION_NAME)
36+
@Name(CloudSQLUtil.CONNECTION_NAME)
3637
@Description(
3738
"The CloudSQL instance to connect to. For a public instance, the connection string should be in the format "
3839
+ "<PROJECT_ID>:<REGION>:<INSTANCE_NAME> which can be found in the instance overview page. For a private "
@@ -43,9 +44,8 @@ public class CloudSQLMySQLConnectorConfig extends AbstractDBConnectorConfig {
4344
@Description("Database name to connect to")
4445
private String database;
4546

46-
@Name(CloudSQLMySQLConstants.INSTANCE_TYPE)
47+
@Name(CloudSQLUtil.INSTANCE_TYPE)
4748
@Description("Whether the CloudSQL instance to connect to is private or public.")
48-
@Nullable
4949
private String instanceType;
5050

5151
public CloudSQLMySQLConnectorConfig(String user, String password, String jdbcPluginName, String connectionArguments,
@@ -73,7 +73,7 @@ public String getConnectionName() {
7373

7474
@Override
7575
public String getConnectionString() {
76-
if (CloudSQLMySQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
76+
if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
7777
return String.format(
7878
CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT,
7979
connectionName,

‎cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,8 @@ private CloudSQLMySQLConstants() {
2323
}
2424

2525
public static final String PLUGIN_NAME = "CloudSQLMySQL";
26-
public static final String CONNECTION_NAME = "connectionName";
27-
public static final String CONNECTION_NAME_PATTERN = "[a-z0-9-]+:[a-z0-9-]+:[a-z0-9-]+";
2826
public static final String CONNECTION_TIMEOUT = "connectionTimeout";
29-
public static final String PUBLIC_INSTANCE = "public";
3027
public static final String PUBLIC_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT =
3128
"jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory";
32-
public static final String INSTANCE_TYPE = "instanceType";
33-
public static final String PRIVATE_INSTANCE = "private";
3429
public static final String PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s/%s";
3530
}

‎cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import io.cdap.plugin.db.SchemaReader;
3333
import io.cdap.plugin.db.batch.config.AbstractDBSpecificSinkConfig;
3434
import io.cdap.plugin.db.batch.sink.AbstractDBSink;
35+
import io.cdap.plugin.util.CloudSQLUtil;
3536

3637
import java.util.Map;
3738
import javax.annotation.Nullable;
@@ -54,11 +55,14 @@ public CloudSQLMySQLSink(CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig) {
5455
@Override
5556
public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
5657
FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector();
57-
58-
CloudSQLMySQLUtil.checkConnectionName(
58+
59+
if (!cloudsqlMysqlSinkConfig.containsMacro(CloudSQLUtil.INSTANCE_TYPE) &&
60+
!cloudsqlMysqlSinkConfig.containsMacro(CloudSQLUtil.CONNECTION_NAME)) {
61+
CloudSQLUtil.checkConnectionName(
5962
failureCollector,
6063
cloudsqlMysqlSinkConfig.connection.getInstanceType(),
6164
cloudsqlMysqlSinkConfig.connection.getConnectionName());
65+
}
6266

6367
super.configurePipeline(pipelineConfigurer);
6468
}

‎cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import io.cdap.plugin.db.SchemaReader;
3232
import io.cdap.plugin.db.batch.config.AbstractDBSpecificSourceConfig;
3333
import io.cdap.plugin.db.batch.source.AbstractDBSource;
34+
import io.cdap.plugin.util.CloudSQLUtil;
3435

3536
import java.util.Collections;
3637
import java.util.HashMap;
@@ -57,10 +58,13 @@ public CloudSQLMySQLSource(CloudSQLMySQLSourceConfig cloudsqlMysqlSourceConfig)
5758
public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
5859
FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector();
5960

60-
CloudSQLMySQLUtil.checkConnectionName(
61+
if (!cloudsqlMysqlSourceConfig.containsMacro(CloudSQLUtil.INSTANCE_TYPE) &&
62+
!cloudsqlMysqlSourceConfig.containsMacro(CloudSQLUtil.CONNECTION_NAME)) {
63+
CloudSQLUtil.checkConnectionName(
6164
failureCollector,
6265
cloudsqlMysqlSourceConfig.connection.getInstanceType(),
6366
cloudsqlMysqlSourceConfig.connection.getConnectionName());
67+
}
6468

6569
super.configurePipeline(pipelineConfigurer);
6670
}
@@ -72,7 +76,7 @@ protected SchemaReader getSchemaReader() {
7276

7377
@Override
7478
protected String createConnectionString() {
75-
if (CloudSQLMySQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(
79+
if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(
7680
cloudsqlMysqlSourceConfig.connection.getInstanceType())) {
7781
return String.format(
7882
CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT,

‎cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLUtil.java‎

Lines changed: 0 additions & 60 deletions
This file was deleted.

‎cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.cdap.cdap.etl.api.action.Action;
2626
import io.cdap.plugin.db.batch.action.AbstractDBAction;
2727
import io.cdap.plugin.db.batch.action.QueryConfig;
28+
import io.cdap.plugin.util.CloudSQLUtil;
2829

2930
import java.util.Map;
3031
import javax.annotation.Nullable;
@@ -48,7 +49,7 @@ public CloudSQLPostgreSQLAction(CloudSQLPostgreSQLActionConfig cloudsqlPostgresq
4849
public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
4950
FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector();
5051

51-
CloudSQLPostgreSQLUtil.checkConnectionName(
52+
CloudSQLUtil.checkConnectionName(
5253
failureCollector,
5354
cloudsqlPostgresqlActionConfig.instanceType,
5455
cloudsqlPostgresqlActionConfig.connectionName);
@@ -60,10 +61,10 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
6061
public static class CloudSQLPostgreSQLActionConfig extends QueryConfig {
6162

6263
public CloudSQLPostgreSQLActionConfig() {
63-
this.instanceType = CloudSQLPostgreSQLConstants.PUBLIC_INSTANCE;
64+
this.instanceType = CloudSQLUtil.PUBLIC_INSTANCE;
6465
}
6566

66-
@Name(CloudSQLPostgreSQLConstants.CONNECTION_NAME)
67+
@Name(CloudSQLUtil.CONNECTION_NAME)
6768
@Description(
6869
"The CloudSQL instance to connect to. For a public instance, the connection string should be in the format "
6970
+ "<PROJECT_ID>:<REGION>:<INSTANCE_NAME> which can be found in the instance overview page. For a private "
@@ -82,14 +83,14 @@ public CloudSQLPostgreSQLActionConfig() {
8283
@Nullable
8384
public Integer connectionTimeout;
8485

85-
@Name(CloudSQLPostgreSQLConstants.INSTANCE_TYPE)
86+
@Name(CloudSQLUtil.INSTANCE_TYPE)
8687
@Description("Whether the CloudSQL instance to connect to is private or public.")
8788
@Nullable
8889
public String instanceType;
8990

9091
@Override
9192
public String getConnectionString() {
92-
if (CloudSQLPostgreSQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
93+
if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
9394
return String.format(
9495
CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT,
9596
connectionName,

‎cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import io.cdap.cdap.api.annotation.Name;
2121
import io.cdap.plugin.db.ConnectionConfig;
2222
import io.cdap.plugin.db.connector.AbstractDBConnectorConfig;
23+
import io.cdap.plugin.util.CloudSQLUtil;
2324

2425
import javax.annotation.Nullable;
2526

@@ -28,7 +29,7 @@
2829
*/
2930
public class CloudSQLPostgreSQLConnectorConfig extends AbstractDBConnectorConfig {
3031

31-
@Name(CloudSQLPostgreSQLConstants.CONNECTION_NAME)
32+
@Name(CloudSQLUtil.CONNECTION_NAME)
3233
@Description(
3334
"The CloudSQL instance to connect to. For a public instance, the connection string should be in the format "
3435
+ "<PROJECT_ID>:<REGION>:<INSTANCE_NAME> which can be found in the instance overview page. For a private "
@@ -39,9 +40,8 @@ public class CloudSQLPostgreSQLConnectorConfig extends AbstractDBConnectorConfig
3940
@Description("Database name to connect to")
4041
private String database;
4142

42-
@Name(CloudSQLPostgreSQLConstants.INSTANCE_TYPE)
43+
@Name(CloudSQLUtil.INSTANCE_TYPE)
4344
@Description("Whether the CloudSQL instance to connect to is private or public.")
44-
@Nullable
4545
private String instanceType;
4646

4747
public CloudSQLPostgreSQLConnectorConfig(String username, String password, String jdbcPluginName,
@@ -70,7 +70,7 @@ public String getConnectionName() {
7070

7171
@Override
7272
public String getConnectionString() {
73-
if (CloudSQLPostgreSQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
73+
if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) {
7474
return String.format(
7575
CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT,
7676
connectionName,

‎cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,8 @@ private CloudSQLPostgreSQLConstants() {
2323
}
2424

2525
public static final String PLUGIN_NAME = "CloudSQLPostgreSQL";
26-
public static final String CONNECTION_NAME = "connectionName";
27-
public static final String CONNECTION_NAME_PATTERN = "[a-z0-9-]+:[a-z0-9-]+:[a-z0-9-]+";
2826
public static final String CONNECTION_TIMEOUT = "connectionTimeout";
29-
public static final String PUBLIC_INSTANCE = "public";
3027
public static final String PUBLIC_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT =
3128
"jdbc:postgresql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.postgres.SocketFactory";
32-
public static final String INSTANCE_TYPE = "instanceType";
33-
public static final String PRIVATE_INSTANCE = "private";
3429
public static final String PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT = "jdbc:postgresql://%s/%s";
3530
}

‎cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import io.cdap.plugin.postgres.PostgresDBRecord;
3939
import io.cdap.plugin.postgres.PostgresFieldsValidator;
4040
import io.cdap.plugin.postgres.PostgresSchemaReader;
41+
import io.cdap.plugin.util.CloudSQLUtil;
4142

4243
import java.util.ArrayList;
4344
import java.util.Collections;
@@ -66,11 +67,14 @@ public CloudSQLPostgreSQLSink(CloudSQLPostgreSQLSinkConfig cloudsqlPostgresqlSin
6667
@Override
6768
public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
6869
FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector();
69-
70-
CloudSQLPostgreSQLUtil.checkConnectionName(
70+
71+
if (!cloudsqlPostgresqlSinkConfig.containsMacro(CloudSQLUtil.INSTANCE_TYPE) &&
72+
!cloudsqlPostgresqlSinkConfig.containsMacro(CloudSQLUtil.CONNECTION_NAME)) {
73+
CloudSQLUtil.checkConnectionName(
7174
failureCollector,
7275
cloudsqlPostgresqlSinkConfig.connection.getInstanceType(),
7376
cloudsqlPostgresqlSinkConfig.connection.getConnectionName());
77+
}
7478

7579
super.configurePipeline(pipelineConfigurer);
7680
}

0 commit comments

Comments
 (0)