bq validation

Ishakaushik-cloud · Ishakaushik-cloud · commit 150782294870 · 2023-06-01T12:53:53.000+05:30
diff --git a/cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySql/BQValidation.java b/cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySql/BQValidation.java
@@ -10,6 +10,7 @@
 import io.cdap.plugin.common.stepsdesign.TestSetupHooks;
 import org.junit.Assert;
 import org.junit.Test;
+
 import java.io.IOException;
 import java.sql.*;
 import java.sql.Date;
@@ -20,27 +21,25 @@
 import java.util.*;
 
 /**
- *  BQValidation
+ * BQValidation
  */
 
 public class BQValidation {
     public static void main(String[] args) throws SQLException, ParseException, IOException, ClassNotFoundException, InterruptedException {
-        //TestSetupHooks.createTables();
-//        TestSetupHooks.createTempSourceBQTable();
- //       validateBQAndDBRecordValues("SourceTable_qGMpakGOTZ", "mySqlToBQTable");
-       // validateDBAndBQRecordValues("SourceTable_qGMpakGOTZ", "mySqlToBQTable");
-        //      validateDBAndBQRecordValues("SourceTable_qGMpakGOTZ", "mySqlToBQTable");
-        validateDBAndBQRecordValues("E2E_SOURCE_4b406","TargetTable_TRFSpppjsZ");
+
+        //       validateDBToBQRecordValues("SourceTable_qGMpakGOTZ", "mySqlToBQTable");
+        validateBQToDBRecordValues("E2E_SOURCE_f9dc4", "TargetTable_TRFSpppjsZ");
     }
 
     /**
      * Extracts entire data from source and target tables.
+     *
      * @param sourceTable table at the source side
      * @param targetTable table at the sink side
      * @return true if the values in source and target side are equal
      */
 
-    public static boolean validateBQAndDBRecordValues(String sourceTable, String targetTable)
+    public static boolean validateDBToBQRecordValues(String sourceTable, String targetTable)
             throws SQLException, ClassNotFoundException, ParseException, IOException, InterruptedException {
         List<JsonObject> jsonResponse = new ArrayList<>();
         List<Object> bigQueryRows = new ArrayList<>();
@@ -49,7 +48,7 @@ public static boolean validateBQAndDBRecordValues(String sourceTable, String tar
             JsonObject json = new Gson().fromJson(String.valueOf(rows), JsonObject.class);
             jsonResponse.add(json);
         }
-        String getSourceQuery = "SELECT * FROM "+ sourceTable;
+        String getSourceQuery = "SELECT * FROM " + sourceTable;
         try (Connection connect = CloudMySqlClient.getCloudMysqlConnection()) {
             connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT);
             Statement statement1 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE,
@@ -59,7 +58,7 @@ public static boolean validateBQAndDBRecordValues(String sourceTable, String tar
         }
     }
 
-    public static boolean validateDBAndBQRecordValues(String sourceTable, String targetTable)
+    public static boolean validateBQToDBRecordValues(String sourceTable, String targetTable)
             throws SQLException, ClassNotFoundException, ParseException, IOException, InterruptedException {
         List<JsonObject> jsonResponse = new ArrayList<>();
         List<Object> bigQueryRows = new ArrayList<>();
@@ -83,16 +82,11 @@ public static boolean validateDBAndBQRecordValues(String sourceTable, String tar
     }
 
 
-
-
-
-
     /**
      * Retrieves the data from a specified BigQuery table and populates it into the provided list of objects.
      *
-     * @param table The name of the BigQuery table to fetch data from.
-     * @param bigQueryRows   The list to store the fetched BigQuery data.
-     *
+     * @param table        The name of the BigQuery table to fetch data from.
+     * @param bigQueryRows The list to store the fetched BigQuery data.
      */
 
     private static void getBigQueryTableData(String table, List<Object> bigQueryRows)
@@ -102,15 +96,14 @@ private static void getBigQueryTableData(String table, List<Object> bigQueryRows
         String dataset = PluginPropertyUtils.pluginProp("dataset");
         String selectQuery = "SELECT TO_JSON(t) FROM `" + projectId + "." + dataset + "." + table + "` AS t";
         TableResult result = BigQueryClient.getQueryResult(selectQuery);
-         result.iterateAll().forEach(value -> bigQueryRows.add(value.get(0).getValue()));
+        result.iterateAll().forEach(value -> bigQueryRows.add(value.get(0).getValue()));
     }
 
     /**
      * Compares the data in the result set obtained from the Oracle database with the provided BigQuery JSON objects.
      *
-     * @param rsSource The result set obtained from the Oracle database.
-     * @param bigQueryData  The list of BigQuery JSON objects to compare with the result set data.
-     *
+     * @param rsSource     The result set obtained from the Oracle database.
+     * @param bigQueryData The list of BigQuery JSON objects to compare with the result set data.
      * @return True if the result set data matches the BigQuery data, false otherwise.
      * @throws SQLException   If an SQL error occurs during the result set operations.
      * @throws ParseException If an error occurs while parsing the data.
@@ -139,7 +132,7 @@ public static boolean compareResultSetData(ResultSet rsSource, List<JsonObject>
         //Variable 'jsonObjectIdx' to track the index of the current JsonObject in the bigQueryData list,
         int jsonObjectIdx = 0;
         while (rsSource.next()) {
-            int currentColumnCount = 2;
+            int currentColumnCount = 3;
             while (currentColumnCount <= columnCountSource) {
                 String columnTypeName = mdSource.getColumnTypeName(currentColumnCount);
                 int columnType = mdSource.getColumnType(currentColumnCount);
@@ -162,7 +155,7 @@ public static boolean compareResultSetData(ResultSet rsSource, List<JsonObject>
                         Assert.assertTrue("Different values found for column : %s",
                                 String.valueOf(sourceTinyInt).equals(String.valueOf(targetTinyInt)));
                         break;
-
+                    case Types.BIGINT:
                     case Types.REAL:
                         Float sourceFloat = rsSource.getFloat(currentColumnCount);
                         Float targetFloat = Float.parseFloat(bigQueryData.get(jsonObjectIdx).get(columnName).getAsString());
@@ -171,7 +164,7 @@ public static boolean compareResultSetData(ResultSet rsSource, List<JsonObject>
                         break;
 
                     case Types.DOUBLE:
-                        Double sourceDouble= rsSource.getDouble(currentColumnCount);
+                        Double sourceDouble = rsSource.getDouble(currentColumnCount);
                         Double targetDouble = Double.parseDouble(bigQueryData.get(jsonObjectIdx).get(columnName).getAsString());
                         Assert.assertTrue("Different values found for column : %s",
                                 String.valueOf(sourceDouble).equals(String.valueOf(targetDouble)));
@@ -185,7 +178,7 @@ public static boolean compareResultSetData(ResultSet rsSource, List<JsonObject>
                         break;
 
                     case Types.TIME:
-                        Time sourceTime= rsSource.getTime(currentColumnCount);
+                        Time sourceTime = rsSource.getTime(currentColumnCount);
                         Time targetTime = Time.valueOf(bigQueryData.get(jsonObjectIdx).get(columnName).getAsString());
                         Assert.assertTrue("Different values found for column : %s",
                                 String.valueOf(sourceTime).equals(String.valueOf(targetTime)));
@@ -215,12 +208,12 @@ public static boolean compareResultSetData(ResultSet rsSource, List<JsonObject>
                         break;
 
                     case Types.TIMESTAMP:
-                        String sourceTS= String.valueOf(rsSource.getTimestamp(currentColumnCount));
-                        String targetTS=bigQueryData.get(jsonObjectIdx).get(columnName).getAsString();
-                        LocalDateTime timestamp = LocalDateTime.parse(targetTS, DateTimeFormatter.ISO_DATE_TIME);
-                        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S");
-                        String formattedTimestamp = timestamp.format(formatter);
-                        Assert.assertEquals(sourceTS, formattedTimestamp);
+//                        String sourceTS= String.valueOf(rsSource.getTimestamp(currentColumnCount));
+//                        String targetTS=bigQueryData.get(jsonObjectIdx).get(columnName).getAsString();
+//                        LocalDateTime timestamp = LocalDateTime.parse(targetTS, DateTimeFormatter.ISO_DATE_TIME);
+//                        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S");
+//                        String formattedTimestamp = timestamp.format(formatter);
+//                        Assert.assertEquals(sourceTS, formattedTimestamp);
                         break;
 
                     default:
@@ -261,6 +254,7 @@ private static void getTimestampValidation(ResultSet rsSource, String sourceTime
         Timestamp targetTs = new Timestamp(parsedDate.getTime());
         Assert.assertEquals(sourceTS, targetTs);
     }
+
     private static String getTimeZoneIdFromSource(ResultSet rsTimezone) throws SQLException {
         if (rsTimezone.next()) {
             return rsTimezone.getString("timezone");
diff --git a/cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySqlClient.java b/cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySqlClient.java
@@ -9,30 +9,20 @@
 import java.util.TimeZone;
 
 public class CloudMySqlClient {
-
-    private static final String database = PluginPropertyUtils.pluginProp("DatabaseName");
-    private static final String connectionName = PluginPropertyUtils.pluginProp("ConnectionName");
-
     public static void main(String[] args) throws SQLException, ClassNotFoundException {
-      //  getCloudMysqlConnection();
-
-        //createSourceTable("myTable");
-//        createSourceTable("newTable");
-//        String[] tablesToDrop = {"newTable"};
-//        dropTables(tablesToDrop);
-        //System.out.println("done");
-
+        getCloudMysqlConnection();
     }
 
+
     public static Connection getCloudMysqlConnection() throws SQLException, ClassNotFoundException {
         Class.forName("com.google.cloud.sql.mysql.SocketFactory");
-        String instanceConnectionName = "cdf-athena:us-central1:sql-automation-test-instance";
-        String databaseName = "TestDatabase";
-        String Username = "v";
-        String Password = "v@123";
-        String jdbcUrl = String.format("jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory&user=%s&password=%s", databaseName, instanceConnectionName, Username, Password);
+        String instanceConnectionName = System.getenv("CLOUDSQLMYSQL_CONNECTIONNAME");
+        String database = System.getenv("DatabaseName");
+        System.out.println(database);
+        String username = System.getenv("CLOUDSQLMYSQL_USERNAME");
+        String password = System.getenv("CLOUDSQLMYSQL_PASSWORD");
+        String jdbcUrl = String.format("jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory&user=%s&password=%s", database, instanceConnectionName, username, password);
         Connection conn = DriverManager.getConnection(jdbcUrl);
-        System.out.println("connected to database");
         return conn;
     }
 
diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/pluginParameters.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/pluginParameters.properties
@@ -1,7 +1,7 @@
 driverName=cloudsql-mysql
 username=v
 password=v@123
-DatabaseName=CreateBQTableQueryFile
+DatabaseName=TestDatabase
 connectionArgumentsList=[{"key":"numSplits","value":"1"}]
 invalidconnectionArgumentsList=[{"key":"numSplits","value":"%$^&#"}]
 invalidImportQuery=select
@@ -75,9 +75,9 @@ datatypesSchema=[{"key":"ID","value":"string"},{"key":"COL1","value":"boolean"},
  {"key":"COL24","value":"bytes"},{"key":"COL25","value":"string"},{"key":"COL26","value":"string"},\
  {"key":"COL27","value":"bytes"},{"key":"COL28","value":"string"},{"key":"COL29","value":"string"}]
 
-bqOutputMultipleDatatypesSchema= [{"key":"col1","value":"bytes"},{"key":"col2","value":"string"},\
-  {"key":"col3","value":"date"},{"key":"col4","value":"double"},{"key":"col6","value":"timestamp"},\
-  {"key":"col8","value":"boolean"},{"key":"col9","value":"long"},{"key":"col10","value":"time"}]
+bqOutputMultipleDatatypesSchema= [{"key":"COL1","value":"bytes"},{"key":"COL2","value":"string"},\
+  {"key":"COL3","value":"date"},{"key":"COL4","value":"double"},{"key":"COL6","value":"timestamp"},\
+  {"key":"COL8","value":"boolean"},{"key":"COL9","value":"long"},{"key":"COL10","value":"time"}]
 SqlServerDatatypesColumns=(COL1 VARBINARY(100) , COL2 VARCHAR(100), COL3 DATE, COL4 DOUBLE,\
   COL6 TIMESTAMP, COL8 BIT, COL9 BIGINT, COL10 TIME)
 #bq queries file path
