Skip to content

Commit 734f817

Browse files
jockechjockech
authored
add e2e test for mysql plugin (#273)
* add e2e test for mysql plugin * fix comments and `mvn` conflicting issue.(will merge commits after the approval)
1 parent 4341572 commit 734f817

12 files changed

Lines changed: 493 additions & 1 deletion

File tree

‎mysql-plugin/pom.xml‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
<name>Mysql plugin</name>
2727
<artifactId>mysql-plugin</artifactId>
2828
<modelVersion>4.0.0</modelVersion>
29-
29+
3030
<dependencies>
3131
<dependency>
3232
<groupId>io.cdap.cdap</groupId>
@@ -89,11 +89,101 @@
8989
</dependency>
9090
</dependencies>
9191
<build>
92+
<testSourceDirectory>${testSourceLocation}</testSourceDirectory>
9293
<plugins>
9394
<plugin>
9495
<groupId>io.cdap</groupId>
9596
<artifactId>cdap-maven-plugin</artifactId>
9697
</plugin>
9798
</plugins>
9899
</build>
100+
<profiles>
101+
<profile>
102+
<id>e2e-tests</id>
103+
<properties>
104+
<testSourceLocation>src/e2e-test/java</testSourceLocation>
105+
</properties>
106+
<build>
107+
<testResources>
108+
<testResource>
109+
<directory>src/e2e-test/resources</directory>
110+
</testResource>
111+
</testResources>
112+
<plugins>
113+
<plugin>
114+
<groupId>org.apache.maven.plugins</groupId>
115+
<artifactId>maven-surefire-plugin</artifactId>
116+
<configuration>
117+
<skipTests>true</skipTests>
118+
</configuration>
119+
</plugin>
120+
121+
<plugin>
122+
<groupId>org.apache.maven.plugins</groupId>
123+
<artifactId>maven-failsafe-plugin</artifactId>
124+
<configuration>
125+
<includes>
126+
<include>TestRunner.java</include>
127+
</includes>
128+
</configuration>
129+
<executions>
130+
<execution>
131+
<goals>
132+
<goal>integration-test</goal>
133+
</goals>
134+
</execution>
135+
</executions>
136+
</plugin>
137+
138+
<plugin>
139+
<groupId>net.masterthought</groupId>
140+
<artifactId>maven-cucumber-reporting</artifactId>
141+
<version>5.5.0</version>
142+
143+
<executions>
144+
<execution>
145+
<id>execution</id>
146+
<phase>verify</phase>
147+
<goals>
148+
<goal>generate</goal>
149+
</goals>
150+
<configuration>
151+
<projectName>Cucumber Reports</projectName> <!-- Replace with project name -->
152+
<outputDirectory>target/cucumber-reports/advanced-reports</outputDirectory>
153+
<buildNumber>1</buildNumber>
154+
<skip>false</skip>
155+
<inputDirectory>${project.build.directory}/cucumber-reports</inputDirectory>
156+
<jsonFiles> <!-- supports wildcard or name pattern -->
157+
<param>**/*.json</param>
158+
</jsonFiles> <!-- optional, defaults to outputDirectory if not specified -->
159+
<classificationDirectory>${project.build.directory}/cucumber-reports</classificationDirectory>
160+
<checkBuildResult>true</checkBuildResult>
161+
</configuration>
162+
</execution>
163+
</executions>
164+
</plugin>
165+
</plugins>
166+
</build>
167+
168+
<dependencies>
169+
<dependency>
170+
<groupId>io.cdap.tests.e2e</groupId>
171+
<artifactId>cdap-e2e-framework</artifactId>
172+
<version>0.0.1-SNAPSHOT</version>
173+
<scope>test</scope>
174+
</dependency>
175+
<dependency>
176+
<groupId>ch.qos.logback</groupId>
177+
<artifactId>logback-core</artifactId>
178+
<version>1.2.8</version>
179+
<scope>runtime</scope>
180+
</dependency>
181+
<dependency>
182+
<groupId>com.google.guava</groupId>
183+
<artifactId>guava</artifactId>
184+
<version>27.0.1-jre</version>
185+
</dependency>
186+
</dependencies>
187+
</profile>
188+
</profiles>
99189
</project>

‎mysql-plugin/src/e2e-test/features/mysql/MySql.feature‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#
2+
# Copyright © 2022 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Mysql
18+
Feature: Mysql - Verify Mysql source data transfer
19+
@MYSQL_SOURCE_TEST @MYSQL_SINK_TEST
20+
Scenario: To verify data is getting transferred from Mysql to Mysql successfully
21+
Given Open Datafusion Project to configure pipeline
22+
When Expand Plugin group in the LHS plugins list: "Source"
23+
When Select plugin: "MySQL" from the plugins list as: "Source"
24+
When Expand Plugin group in the LHS plugins list: "Sink"
25+
When Select plugin: "MySQL" from the plugins list as: "Sink"
26+
Then Connect plugins: "MySQL" and "MySQL2" to establish connection
27+
Then Navigate to the properties page of plugin: "MySQL"
28+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driver"
29+
Then Replace input plugin property: "host" with value: "host"
30+
Then Replace input plugin property: "port" with value: "port"
31+
Then Replace input plugin property: "user" with value: "username"
32+
Then Replace input plugin property: "password" with value: "password"
33+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
34+
Then Replace input plugin property: "database" with value: "database"
35+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
36+
Then Click on the Get Schema button
37+
Then Verify the Output Schema matches the Expected Schema: "outputSchema"
38+
Then Validate "MySQL" plugin properties
39+
Then Close the Plugin Properties page
40+
Then Navigate to the properties page of plugin: "MySQL2"
41+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driver"
42+
Then Replace input plugin property: "host" with value: "host"
43+
Then Replace input plugin property: "port" with value: "port"
44+
Then Replace input plugin property: "database" with value: "database"
45+
Then Replace input plugin property: "tableName" with value: "targetTable"
46+
Then Replace input plugin property: "user" with value: "username"
47+
Then Replace input plugin property: "password" with value: "password"
48+
Then Enter input plugin property: "referenceName" with value: "targetRef"
49+
Then Validate "MySQL2" plugin properties
50+
Then Close the Plugin Properties page
51+
Then Save the pipeline
52+
Then Preview and run the pipeline
53+
Then Verify the preview of pipeline is "success"
54+
Then Click on preview data for MySQL sink
55+
Then Verify preview output schema matches the outputSchema captured in properties
56+
Then Close the preview data
57+
Then Deploy the pipeline
58+
Then Run the Pipeline in Runtime
59+
Then Wait till pipeline is in running state
60+
Then Open and capture logs
61+
Then Verify the pipeline status is "Succeeded"
62+
Then Get count of no of records transferred to target MySQL Table
63+
Then Validate records transferred to target table is equal to number of records from source table

‎mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Copyright © 2021 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
21+
import java.sql.Connection;
22+
import java.sql.DriverManager;
23+
import java.sql.ResultSet;
24+
import java.sql.SQLException;
25+
import java.sql.Statement;
26+
27+
/**
28+
* MySQL client.
29+
*/
30+
public class MysqlClient {
31+
private static final String host = PluginPropertyUtils.pluginProp("host");
32+
private static final int port = Integer.parseInt(PluginPropertyUtils.pluginProp("port"));
33+
private static final String database = PluginPropertyUtils.pluginProp("database");
34+
35+
private static Connection getMysqlConnection() throws SQLException, ClassNotFoundException {
36+
Class.forName("com.mysql.cj.jdbc.Driver");
37+
String username = PluginPropertyUtils.pluginProp("username");
38+
String password = PluginPropertyUtils.pluginProp("password");
39+
return DriverManager.getConnection("jdbc:mysql://" + host + ":" + port + "/" + database, username, password);
40+
}
41+
42+
public static int countRecord(String table) throws SQLException, ClassNotFoundException {
43+
String countQuery = "SELECT COUNT(*) as total FROM " + table;
44+
try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement();
45+
ResultSet rs = statement.executeQuery(countQuery)) {
46+
int num = 0;
47+
while (rs.next()) {
48+
num = (rs.getInt(1));
49+
}
50+
return num;
51+
}
52+
}
53+
54+
public static void createSourceTable(String sourceTable) throws SQLException, ClassNotFoundException {
55+
try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) {
56+
String createSourceTableQuery = "CREATE TABLE IF NOT EXISTS " + sourceTable +
57+
"(id int, lastName varchar(255), PRIMARY KEY (id))";
58+
statement.executeUpdate(createSourceTableQuery);
59+
60+
// Truncate table to clean the data of last failure run.
61+
String truncateSourceTableQuery = "TRUNCATE TABLE " + sourceTable;
62+
statement.executeUpdate(truncateSourceTableQuery);
63+
64+
// Insert dummy data.
65+
statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" +
66+
"VALUES (1, 'Simpson')");
67+
statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" +
68+
"VALUES (2, 'McBeal')");
69+
statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" +
70+
"VALUES (3, 'Flinstone')");
71+
}
72+
}
73+
74+
public static void createTargetTable(String targetTable) throws SQLException, ClassNotFoundException {
75+
try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) {
76+
String createTargetTableQuery = "CREATE TABLE IF NOT EXISTS " + targetTable +
77+
"(id int, lastName varchar(255), PRIMARY KEY (id))";
78+
statement.executeUpdate(createTargetTableQuery);
79+
// Truncate table to clean the data of last failure run.
80+
String truncateTargetTableQuery = "TRUNCATE TABLE " + targetTable;
81+
statement.executeUpdate(truncateTargetTableQuery);
82+
}
83+
}
84+
85+
public static void dropTables(String[] tables) throws SQLException, ClassNotFoundException {
86+
try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) {
87+
for (String table : tables) {
88+
String dropTableQuery = "Drop Table " + table;
89+
statement.executeUpdate(dropTableQuery);
90+
}
91+
}
92+
}
93+
}

‎mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright © 2021 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.common.stepsdesign;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cdap.plugin.MysqlClient;
21+
import io.cucumber.java.After;
22+
import io.cucumber.java.Before;
23+
24+
import java.sql.SQLException;
25+
26+
/**
27+
* MYSQL test hooks.
28+
*/
29+
public class TestSetupHooks {
30+
31+
@Before(order = 1)
32+
public static void overrideUserAndPasswordIfProvided() {
33+
String username = System.getenv("username");
34+
if (username != null && !username.isEmpty()) {
35+
PluginPropertyUtils.addPluginProp("username", username);
36+
}
37+
String password = System.getenv("password");
38+
if (password != null && !password.isEmpty()) {
39+
PluginPropertyUtils.addPluginProp("password", password);
40+
}
41+
}
42+
43+
@Before(order = 1, value = "@MYSQL_SOURCE_TEST")
44+
public static void setSelectQuery() {
45+
String sourceTable = PluginPropertyUtils.pluginProp("sourceTable");
46+
PluginPropertyUtils.addPluginProp("selectQuery",
47+
PluginPropertyUtils.pluginProp("selectQuery").
48+
replace("${table}", sourceTable));
49+
}
50+
51+
@Before(order = 2, value = "@MYSQL_SOURCE_TEST")
52+
public static void createTables() throws SQLException, ClassNotFoundException {
53+
MysqlClient.createSourceTable(PluginPropertyUtils.pluginProp("sourceTable"));
54+
MysqlClient.createTargetTable(PluginPropertyUtils.pluginProp("targetTable"));
55+
}
56+
57+
@After(order = 2, value = "@MYSQL_SINK_TEST")
58+
public static void dropTables() throws SQLException, ClassNotFoundException {
59+
MysqlClient.dropTables(new String[]{PluginPropertyUtils.pluginProp("sourceTable"),
60+
PluginPropertyUtils.pluginProp("targetTable")});
61+
}
62+
63+
}

‎mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains the locators for the Joiner plugin.
19+
*/
20+
package io.cdap.plugin.common.stepsdesign;

‎mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunner.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.mysql.runners;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute Mysql plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.mysql.stepsdesign"},
29+
tags = {"@Mysql"},
30+
plugin = {"pretty", "html:target/cucumber-html-report/mysql",
31+
"json:target/cucumber-reports/cucumber-mysql.json",
32+
"junit:target/cucumber-reports/cucumber-mysql.xml"}
33+
)
34+
public class TestRunner {
35+
}

0 commit comments

Comments
 (0)