RANGER-5543:Ranger KMS GCP always try to create master key irrespective of its existence (#910)

Author: bhaveshamre

distro/src/main/assembly/kms.xml

@@ -166,7 +166,10 @@
                     <include>com.google.android:annotations</include>
                     <include>io.grpc:grpc-alts</include>
                     <include>io.grpc:grpc-grpclb</include>
+                    <include>com.google.protobuf:protobuf-java:jar:${gcp.protobuf-java.version}</include>
                     <include>com.google.protobuf:protobuf-java-util:jar:${gcp.protobuf-java.version}</include>
+                    <include>com.google.guava:guava:jar:${google.guava.version}</include>
+                    <include>com.google.guava:failureaccess:jar:${google.failureaccess.version}</include>
                     <include>org.conscrypt:conscrypt-openjdk-uber</include>
                     <include>org.threeten:threetenbp</include>
                     <include>io.grpc:grpc-auth</include>

kms/pom.xml

@@ -103,6 +103,16 @@
             <artifactId>jsr305</artifactId>
             <version>${jsr305.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>failureaccess</artifactId>
+            <version>${google.failureaccess.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${google.guava.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java</artifactId>

kms/src/main/java/org/apache/hadoop/crypto/key/RangerGoogleCloudHSMProvider.java

@@ -16,7 +16,7 @@
  */
 package org.apache.hadoop.crypto.key;
 
-import com.google.api.gax.rpc.AlreadyExistsException;
+import com.google.api.gax.rpc.NotFoundException;
 import com.google.cloud.kms.v1.CryptoKey;
 import com.google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose;
 import com.google.cloud.kms.v1.CryptoKeyName;
@@ -68,37 +68,34 @@ public RangerGoogleCloudHSMProvider(Configuration conf) throws Exception {
 
     @Override
     public boolean generateMasterKey(String unusedPassword) throws Throwable {
-        //The ENCRYPT_DECRYPT key purpose enables symmetric encryption.
-        //All keys with key purpose ENCRYPT_DECRYPT use the GOOGLE_SYMMETRIC_ENCRYPTION algorithm.
-        //No parameters are used with this algorithm.
-        CryptoKey key = CryptoKey.newBuilder()
-                .setPurpose(CryptoKeyPurpose.ENCRYPT_DECRYPT)
-                .setVersionTemplate(CryptoKeyVersionTemplate.newBuilder()
-                        .setProtectionLevel(ProtectionLevel.HSM)
-                        .setAlgorithm(CryptoKeyVersionAlgorithm.GOOGLE_SYMMETRIC_ENCRYPTION))
-                .build();
-
-        // Create the key.
-        CryptoKey createdKey = null;
-        try {
-            createdKey = client.createCryptoKey(this.keyRingName, this.gcpMasterKeyName, key);
-        } catch (Exception e) {
-            if (e instanceof AlreadyExistsException) {
-                logger.info("MasterKey with the name '{}' already exist.", this.gcpMasterKeyName);
-                return true;
-            } else {
+        boolean isMKGenerated = false;
+        if (!this.masterKeyExists()) {
+            //The ENCRYPT_DECRYPT key purpose enables symmetric encryption.
+            //All keys with key purpose ENCRYPT_DECRYPT use the GOOGLE_SYMMETRIC_ENCRYPTION algorithm.
+            //No parameters are used with this algorithm.
+            CryptoKey key = CryptoKey.newBuilder()
+                    .setPurpose(CryptoKeyPurpose.ENCRYPT_DECRYPT)
+                    .setVersionTemplate(CryptoKeyVersionTemplate.newBuilder()
+                            .setProtectionLevel(ProtectionLevel.HSM)
+                            .setAlgorithm(CryptoKeyVersionAlgorithm.GOOGLE_SYMMETRIC_ENCRYPTION))
+                    .build();
+
+            // Create the key.
+            CryptoKey createdKey = null;
+            try {
+                createdKey = client.createCryptoKey(this.keyRingName, this.gcpMasterKeyName, key);
+            } catch (Exception e) {
                 throw new RuntimeCryptoException("Failed to create master key with name '" + this.gcpMasterKeyName + "', Error - " + e.getMessage());
             }
-        }
 
-        if (createdKey == null) {
-            logger.info("Failed to create master key : {}", this.gcpMasterKeyName);
-            return false;
+            if (createdKey != null) {
+                logger.info("Master Key Created Successfully On Google Cloud HSM : {}", this.gcpMasterKeyName);
+                isMKGenerated = true;
+            } else {
+                logger.info("Failed to create master key : {}", this.gcpMasterKeyName);
+            }
         }
-
-        logger.info("Master Key Created Successfully On Google Cloud HSM : {}", this.gcpMasterKeyName);
-
-        return true;
+        return isMKGenerated;
     }
 
     @Override
@@ -221,4 +218,26 @@ private static void updateEnv(String name, String val) throws ReflectiveOperatio
 
         writeAbleEnvMap.put(name, val);
     }
+
+    private boolean masterKeyExists() throws Throwable {
+        boolean exists = false;
+
+        if (this.client == null) {
+            throw new RuntimeCryptoException("Google Cloud KMS client is not initialized; call onInitialization() first.");
+        }
+
+        CryptoKeyName keyName = CryptoKeyName.of(this.gcpProjectId, this.gcpLocationId, this.gcpKeyRingId, this.gcpMasterKeyName);
+
+        try {
+            CryptoKey cryptoKey = this.client.getCryptoKey(keyName);
+            logger.info("Ranger masterKey present with name: {}", cryptoKey.getName());
+            exists = true;
+        } catch (NotFoundException e) {
+            logger.info("Ranger masterKey not found with name: {}", keyName);
+        } catch (Exception e) {
+            logger.error("Error checking for masterkey: " + e.getMessage());
+            throw e;
+        }
+        return exists;
+    }
 }

pom.xml

@@ -99,6 +99,7 @@
 
         <!-- GCP HSM -->
         <google.cloud.kms>2.3.0</google.cloud.kms>
+        <google.failureaccess.version>1.0.3</google.failureaccess.version>
         <google.guava.version>33.4.8-jre</google.guava.version>
         <google.re2j.version>1.2</google.re2j.version>
         <googlecode.log4jdbc.version>1.2</googlecode.log4jdbc.version>