feat!: customer-managed encryption keys for Spanner (#666)

* feat: add support for encrypted databases

* fix: fix deps and clirr failures

* tests: add additional tests for keys

* tests: remove IT and add unit

* fix: set null instead of default instance

* fix: does not set encryption info if null

Does not set encryption info in the request if it is null

* fix: fixes dependencies

* feature: adds support for encrypted backup

Adds the possibility to set encryption config info in the creation of a
backup.

* feature: adds support for restoring encrypted dbs

* Revert "tests: remove IT and add unit"

This reverts commit cc19cf2.

* fix: makes the setEncryptionConfigInfo public

This is so a backup can be encrypted

* feature: adds tests for cmek

Adds tests for creating encrypted database, creating encrypted backups
and restoring encrypted databases.

* fix: removes keys after test finishes

Destroy keys used in CMEK tests

* fix: fixes clirr errors

* fix: ignores failing cmek tests

Ignores the failing CMEK tests until the backend support is enabled in
production.

* fix: uses wrapper encryption info for backups

* fix: fixes clirr issues

* fix: re-orders clirr issues

* fix: addresses PR comments

* test: fixes database admin client tests

* chore: re-formats the code

* chore: fixes clirr checks

* tests: adds unit tests for domain classes

Adds unit tests for EncryptionConfigInfo, EncryptionConfig, Backup and
Restore.

* chore: renames EncryptionConfigInfo

Renames EncryptionConfigInfo to EncryptionConfig in order to mirror what
is the protobuf definition.

* tests: do not create a key on CMEK test

Instead use an existing key and fails if the key is not present.

* feat: allows multiple encryption configs

Allows customer managed encryption for create databases (google default
encryption is just nullifying the value here).
Allows customer managed encryption, google default encryption and
database encryption for create backups.
Allows customer managed encryption, google default encryption and backup
encryption for restore databases.

* docs: adds java doc to Restore class

* chore: refactors pom.xml

Uses variables to define project id and instance id for running
integration tests.

* test: fixes cmek integration test

* chore: fixes linting

* Revert "chore: refactors pom.xml"

This reverts commit d182b83.

* test: unifies cmek backup and restore tests

* chore: adds toString to encryption classes

* docs: updates DatabaseInfo javadoc

Co-authored-by: Knut Olav Løite <[email protected]>

* docs: updates Restore javadocs

Co-authored-by: Knut Olav Løite <[email protected]>

* docs: updates DatabaseInfo javadocs

Co-authored-by: Knut Olav Løite <[email protected]>

* fix: addresses PR comments

* tests: reformats

Co-authored-by: Olav Loite <[email protected]>

Author: thiagotnunes

google-cloud-spanner/clirr-ignored-differences.xml

@@ -319,7 +319,7 @@
     <className>com/google/cloud/spanner/Value</className>
     <method>java.util.List getNumericArray()</method>
   </difference>
-  
+
   <!-- Async Connection API -->
   <difference>
     <differenceType>7012</differenceType>
@@ -406,7 +406,7 @@
     <className>com/google/cloud/spanner/AbstractLazyInitializer</className>
     <method>java.lang.Object initialize()</method>
   </difference>
-  
+
   <!-- TransactionOptions and UpdateOptions -->
   <difference>
     <differenceType>7004</differenceType>
@@ -504,4 +504,71 @@
     <className>com/google/cloud/spanner/DatabaseAdminClient</className>
     <method>com.google.api.gax.longrunning.OperationFuture createBackup(com.google.cloud.spanner.Backup)</method>
   </difference>
+
+  <!-- Support creating encrypted databases -->
+  <difference>
+    <differenceType>7004</differenceType>
+    <className>com/google/cloud/spanner/spi/v1/SpannerRpc</className>
+    <method>com.google.api.gax.longrunning.OperationFuture createDatabase(java.lang.String, java.lang.String, java.lang.Iterable)</method>
+  </difference>
+  <difference>
+    <differenceType>7004</differenceType>
+    <className>com/google/cloud/spanner/spi/v1/SpannerRpc</className>
+    <method>com.google.api.gax.longrunning.OperationFuture createBackup(java.lang.String, java.lang.String, com.google.spanner.admin.database.v1.Backup)</method>
+  </difference>
+  <difference>
+    <differenceType>7004</differenceType>
+    <className>com/google/cloud/spanner/spi/v1/SpannerRpc</className>
+    <method>com.google.api.gax.longrunning.OperationFuture restoreDatabase(java.lang.String, java.lang.String, java.lang.String)</method>
+  </difference>
+  <difference>
+    <differenceType>7004</differenceType>
+    <className>com/google/cloud/spanner/spi/v1/GapicSpannerRpc</className>
+    <method>com.google.api.gax.longrunning.OperationFuture createDatabase(java.lang.String, java.lang.String, java.lang.Iterable)</method>
+  </difference>
+  <difference>
+    <differenceType>7004</differenceType>
+    <className>com/google/cloud/spanner/spi/v1/GapicSpannerRpc</className>
+    <method>com.google.api.gax.longrunning.OperationFuture createBackup(java.lang.String, java.lang.String, com.google.spanner.admin.database.v1.Backup)</method>
+  </difference>
+  <difference>
+    <differenceType>7004</differenceType>
+    <className>com/google/cloud/spanner/spi/v1/GapicSpannerRpc</className>
+    <method>com.google.api.gax.longrunning.OperationFuture restoreDatabase(java.lang.String, java.lang.String, java.lang.String)</method>
+  </difference>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/DatabaseAdminClient</className>
+    <method>com.google.api.gax.longrunning.OperationFuture createDatabase(com.google.cloud.spanner.Database, java.lang.Iterable)</method>
+  </difference>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/DatabaseAdminClient</className>
+    <method>com.google.api.gax.longrunning.OperationFuture createBackup(com.google.cloud.spanner.Backup)</method>
+  </difference>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/DatabaseAdminClient</className>
+    <method>com.google.api.gax.longrunning.OperationFuture restoreDatabase(com.google.cloud.spanner.Restore)</method>
+  </difference>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/DatabaseAdminClient</className>
+    <method>com.google.cloud.spanner.Database$Builder newDatabaseBuilder(com.google.cloud.spanner.DatabaseId)</method>
+  </difference>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/DatabaseAdminClient</className>
+    <method>com.google.cloud.spanner.Restore$Builder newRestoreBuilder(com.google.cloud.spanner.BackupId, com.google.cloud.spanner.DatabaseId)</method>
+  </difference>
+  <difference>
+    <differenceType>7013</differenceType>
+    <className>com/google/cloud/spanner/DatabaseInfo$Builder</className>
+    <method>com.google.cloud.spanner.DatabaseInfo$Builder setEncryptionConfig(com.google.cloud.spanner.encryption.CustomerManagedEncryption)</method>
+  </difference>
+  <difference>
+    <differenceType>7013</differenceType>
+    <className>com/google/cloud/spanner/BackupInfo$Builder</className>
+    <method>com.google.cloud.spanner.BackupInfo$Builder setEncryptionConfig(com.google.cloud.spanner.encryption.BackupEncryptionConfig)</method>
+  </difference>
 </differences>

google-cloud-spanner/pom.xml

@@ -73,6 +73,7 @@
             <spanner.testenv.config.class>com.google.cloud.spanner.GceTestEnvConfig</spanner.testenv.config.class>
             <spanner.testenv.instance>projects/gcloud-devel/instances/spanner-testing</spanner.testenv.instance>
             <spanner.gce.config.project_id>gcloud-devel</spanner.gce.config.project_id>
+            <spanner.testenv.kms_key.name>projects/gcloud-devel/locations/us-central1/keyRings/spanner-test-keyring/cryptoKeys/spanner-test-key</spanner.testenv.kms_key.name>
           </systemPropertyVariables>
           <forkedProcessTimeoutInSeconds>3000</forkedProcessTimeoutInSeconds>
         </configuration>
@@ -383,7 +384,7 @@
       </build>
     </profile>
     <profile>
-      <!-- Profile for generating new sql test scripts. See ConnectionImplGeneratedSqlScriptTest 
+      <!-- Profile for generating new sql test scripts. See ConnectionImplGeneratedSqlScriptTest
         for more information. -->
       <id>generate-test-sql-scripts</id>
       <build>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/Backup.java

@@ -23,6 +23,7 @@
 import com.google.api.gax.paging.Page;
 import com.google.cloud.Policy;
 import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.encryption.EncryptionInfo;
 import com.google.longrunning.Operation;
 import com.google.spanner.admin.database.v1.CreateBackupMetadata;
 import com.google.spanner.admin.database.v1.RestoreDatabaseMetadata;
@@ -61,10 +62,6 @@ public Backup build() {
 
   /** Creates a backup on the server based on the source of this {@link Backup} instance. */
   public OperationFuture<Backup, CreateBackupMetadata> create() {
-    Preconditions.checkState(
-        getExpireTime() != null, "Cannot create a backup without an expire time");
-    Preconditions.checkState(
-        getDatabase() != null, "Cannot create a backup without a source database");
     return dbClient.createBackup(this);
   }
 
@@ -184,6 +181,7 @@ static Backup fromProto(
         .setExpireTime(Timestamp.fromProto(proto.getExpireTime()))
         .setVersionTime(Timestamp.fromProto(proto.getVersionTime()))
         .setDatabase(DatabaseId.of(proto.getDatabase()))
+        .setEncryptionInfo(EncryptionInfo.fromProtoOrNull(proto.getEncryptionInfo()))
         .setProto(proto)
         .build();
   }

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BackupInfo.java

@@ -18,6 +18,8 @@
 
 import com.google.api.client.util.Preconditions;
 import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.encryption.BackupEncryptionConfig;
+import com.google.cloud.spanner.encryption.EncryptionInfo;
 import com.google.spanner.admin.database.v1.Database;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -29,8 +31,29 @@ public abstract static class Builder {
 
     abstract Builder setSize(long size);
 
+    /**
+     * Returned when retrieving a backup.
+     *
+     * <p>The encryption information for the backup. If the encryption key protecting this resource
+     * is customer managed, then kms_key_version will be filled.
+     */
+    abstract Builder setEncryptionInfo(EncryptionInfo encryptionInfo);
+
     abstract Builder setProto(com.google.spanner.admin.database.v1.Backup proto);
 
+    /**
+     * Optional for creating a new backup.
+     *
+     * <p>The encryption configuration to be used for the backup. The possible configurations are
+     * {@link com.google.cloud.spanner.encryption.CustomerManagedEncryption}, {@link
+     * com.google.cloud.spanner.encryption.GoogleDefaultEncryption} and {@link
+     * com.google.cloud.spanner.encryption.UseDatabaseEncryption}.
+     *
+     * <p>If no encryption config is given the backup will be created with the same encryption as
+     * set by the database ({@link com.google.cloud.spanner.encryption.UseDatabaseEncryption}).
+     */
+    public abstract Builder setEncryptionConfig(BackupEncryptionConfig encryptionConfig);
+
     /**
      * Required for creating a new backup.
      *
@@ -70,6 +93,8 @@ abstract static class BuilderImpl extends Builder {
     private Timestamp versionTime;
     private DatabaseId database;
     private long size;
+    private BackupEncryptionConfig encryptionConfig;
+    private EncryptionInfo encryptionInfo;
     private com.google.spanner.admin.database.v1.Backup proto;
 
     BuilderImpl(BackupId id) {
@@ -83,6 +108,8 @@ abstract static class BuilderImpl extends Builder {
       this.versionTime = other.versionTime;
       this.database = other.database;
       this.size = other.size;
+      this.encryptionConfig = other.encryptionConfig;
+      this.encryptionInfo = other.encryptionInfo;
       this.proto = other.proto;
     }
 
@@ -113,12 +140,24 @@ public Builder setDatabase(DatabaseId database) {
       return this;
     }
 
+    @Override
+    public Builder setEncryptionConfig(BackupEncryptionConfig encryptionConfig) {
+      this.encryptionConfig = encryptionConfig;
+      return this;
+    }
+
     @Override
     Builder setSize(long size) {
       this.size = size;
       return this;
     }
 
+    @Override
+    Builder setEncryptionInfo(EncryptionInfo encryptionInfo) {
+      this.encryptionInfo = encryptionInfo;
+      return this;
+    }
+
     @Override
     Builder setProto(@Nullable com.google.spanner.admin.database.v1.Backup proto) {
       this.proto = proto;
@@ -142,12 +181,16 @@ public enum State {
   private final Timestamp versionTime;
   private final DatabaseId database;
   private final long size;
+  private final BackupEncryptionConfig encryptionConfig;
+  private final EncryptionInfo encryptionInfo;
   private final com.google.spanner.admin.database.v1.Backup proto;
 
   BackupInfo(BuilderImpl builder) {
     this.id = builder.id;
     this.state = builder.state;
     this.size = builder.size;
+    this.encryptionConfig = builder.encryptionConfig;
+    this.encryptionInfo = builder.encryptionInfo;
     this.expireTime = builder.expireTime;
     this.versionTime = builder.versionTime;
     this.database = builder.database;
@@ -174,6 +217,22 @@ public long getSize() {
     return size;
   }
 
+  /**
+   * Returns the {@link BackupEncryptionConfig} to encrypt the backup during its creation. Returns
+   * <code>null</code> if no customer-managed encryption key should be used.
+   */
+  public BackupEncryptionConfig getEncryptionConfig() {
+    return encryptionConfig;
+  }
+
+  /**
+   * Returns the {@link EncryptionInfo} of the backup if the backup is encrypted, or <code>null
+   * </code> if this backup is not encrypted.
+   */
+  public EncryptionInfo getEncryptionInfo() {
+    return encryptionInfo;
+  }
+
   /** Returns the expire time of the backup. */
   public Timestamp getExpireTime() {
     return expireTime;
@@ -206,20 +265,30 @@ public boolean equals(Object o) {
     return id.equals(that.id)
         && state == that.state
         && size == that.size
+        && Objects.equals(encryptionConfig, that.encryptionConfig)
+        && Objects.equals(encryptionInfo, that.encryptionInfo)
         && Objects.equals(expireTime, that.expireTime)
         && Objects.equals(versionTime, that.versionTime)
         && Objects.equals(database, that.database);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(id, state, size, expireTime, versionTime, database);
+    return Objects.hash(
+        id, state, size, encryptionConfig, encryptionInfo, expireTime, versionTime, database);
   }
 
   @Override
   public String toString() {
     return String.format(
-        "Backup[%s, %s, %d, %s, %s, %s]",
-        id.getName(), state, size, expireTime, versionTime, database);
+        "Backup[%s, %s, %d, %s, %s, %s, %s, %s]",
+        id.getName(),
+        state,
+        size,
+        encryptionConfig,
+        encryptionInfo,
+        expireTime,
+        versionTime,
+        database);
   }
 }

google-cloud-spanner/src/main/java/com/google/cloud/spanner/Database.java

@@ -22,6 +22,7 @@
 import com.google.api.gax.paging.Page;
 import com.google.cloud.Policy;
 import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.encryption.CustomerManagedEncryption;
 import com.google.common.base.Preconditions;
 import com.google.longrunning.Operation;
 import com.google.spanner.admin.database.v1.CreateBackupMetadata;
@@ -185,6 +186,7 @@ static Database fromProto(
         .setRestoreInfo(RestoreInfo.fromProtoOrNullIfDefaultInstance(proto.getRestoreInfo()))
         .setVersionRetentionPeriod(proto.getVersionRetentionPeriod())
         .setEarliestVersionTime(Timestamp.fromProto(proto.getEarliestVersionTime()))
+        .setEncryptionConfig(CustomerManagedEncryption.fromProtoOrNull(proto.getEncryptionConfig()))
         .setProto(proto)
         .build();
   }