From 4b229e6c68cf2219989eca412e930b1ea150c561 Mon Sep 17 00:00:00 2001 From: Stella Chung Date: Mon, 6 Jun 2022 11:47:32 -0700 Subject: [PATCH 01/12] Add vault secret persistence --- .../main/java/io/airbyte/config/Configs.java | 18 ++++- .../java/io/airbyte/config/EnvConfigs.java | 13 ++++ airbyte-config/persistence/build.gradle | 1 + .../split_secrets/SecretPersistence.java | 6 ++ .../split_secrets/VaultSecretPersistence.java | 66 +++++++++++++++++++ docs/operator-guides/configuring-airbyte.md | 4 +- 6 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java index dd19fc10bd43..d38ab0d700b4 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java @@ -103,11 +103,22 @@ public interface Configs { /** * Defines the Secret Persistence type. None by default. Set to GOOGLE_SECRET_MANAGER to use Google - * Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Alpha support. - * Undefined behavior will result if this is turned on and then off. + * Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use + * Hashicorp Vault. Alpha support. Undefined behavior will result if this is turned on and then off. */ SecretPersistenceType getSecretPersistenceType(); + /** + * Define the vault address to read/write Airbyte Configuration to Hashicorp Vault. Alpha Support. + */ + String getVaultAddress(); + + /** + * Define the vault path prefix to read/write Airbyte Configuration to Hashicorp Vault. Empty by + * default. Alpha Support. + */ + String getVaultPrefix(); + // Database /** * Define the Jobs Database user. @@ -536,7 +547,8 @@ enum DeploymentMode { enum SecretPersistenceType { NONE, TESTING_CONFIG_DB_TABLE, - GOOGLE_SECRET_MANAGER + GOOGLE_SECRET_MANAGER, + VAULT } } diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java index cfc7ebe0bd40..aff05cbdf709 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -161,6 +161,9 @@ public class EnvConfigs implements Configs { private static final long DEFAULT_MAXIMUM_WORKSPACE_SIZE_MB = 5000; private static final int DEFAULT_DATABASE_INITIALIZATION_TIMEOUT_MS = 60 * 1000; + private static final String VAULT_ADDRESS = "VAULT_ADDRESS"; + private static final String VAULT_PREFIX = "VAULT_PREFIX"; + public static final long DEFAULT_MAX_SPEC_WORKERS = 5; public static final long DEFAULT_MAX_CHECK_WORKERS = 5; public static final long DEFAULT_MAX_DISCOVER_WORKERS = 5; @@ -332,6 +335,16 @@ public SecretPersistenceType getSecretPersistenceType() { return SecretPersistenceType.valueOf(secretPersistenceStr); } + @Override + public String getVaultAddress() { + return getEnv(VAULT_ADDRESS); + } + + @Override + public String getVaultPrefix() { + return getEnvOrDefault(VAULT_PREFIX, ""); + } + // Database @Override public String getDatabaseUser() { diff --git a/airbyte-config/persistence/build.gradle b/airbyte-config/persistence/build.gradle index 6c415f1e6af9..004caf7b6611 100644 --- a/airbyte-config/persistence/build.gradle +++ b/airbyte-config/persistence/build.gradle @@ -14,6 +14,7 @@ dependencies { implementation 'commons-io:commons-io:2.7' implementation 'com.google.cloud:google-cloud-secretmanager:2.0.5' + implementation 'com.bettercloud:vault-java-driver:5.1.0' testImplementation 'org.hamcrest:hamcrest-all:1.3' testImplementation libs.testcontainers.postgresql diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java index 3b1abc4e9053..d9f6546981ba 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java @@ -32,6 +32,9 @@ static Optional getLongLived(final DSLContext dslContext, fin case GOOGLE_SECRET_MANAGER -> { return Optional.of(GoogleSecretManagerPersistence.getLongLived(configs.getSecretStoreGcpProjectId(), configs.getSecretStoreGcpCredentials())); } + case VAULT -> { + return Optional.of(new VaultSecretPersistence(configs.getVaultAddress(), configs.getVaultPrefix())); + } default -> { return Optional.empty(); } @@ -57,6 +60,9 @@ static Optional getEphemeral(final DSLContext dslContext, fin case GOOGLE_SECRET_MANAGER -> { return Optional.of(GoogleSecretManagerPersistence.getEphemeral(configs.getSecretStoreGcpProjectId(), configs.getSecretStoreGcpCredentials())); } + case VAULT -> { + return Optional.of(new VaultSecretPersistence(configs.getVaultAddress(), configs.getVaultPrefix())); + } default -> { return Optional.empty(); } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java new file mode 100644 index 000000000000..912ecd140a37 --- /dev/null +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence.split_secrets; + +import com.bettercloud.vault.Vault; +import com.bettercloud.vault.VaultConfig; +import com.bettercloud.vault.VaultException; +import io.airbyte.commons.lang.Exceptions; +import java.util.HashMap; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +final public class VaultSecretPersistence implements SecretPersistence { + + private final String secretKey = "value"; + private final Vault vault; + private final String pathPrefix; + + public VaultSecretPersistence(final String address, final String prefix) { + this.vault = Exceptions.toRuntime(() -> getVaultClient(address)); + this.pathPrefix = prefix; + } + + @Override + public Optional read(final SecretCoordinate coordinate) { + try { + final var response = vault.logical().read(pathPrefix + coordinate.getFullCoordinate()); + log.info(pathPrefix); + final var restResponse = response.getRestResponse(); + final var responseCode = restResponse.getStatus(); + if (responseCode != 200) { + log.error("failed on read", response); + return Optional.empty(); + } + final var data = response.getData(); + return Optional.of(data.get(secretKey)); + } catch (final VaultException e) { + return Optional.empty(); + } + } + + @Override + public void write(final SecretCoordinate coordinate, final String payload) { + try { + final var newSecret = new HashMap(); + newSecret.put(secretKey, payload); + vault.logical().write(pathPrefix + coordinate.getFullCoordinate(), newSecret); + } catch (final VaultException e) { + log.error("failed on write", e); + } + } + + /** + * This creates a vault client using a vault agent which uses AWS IAM for auth. + */ + public static Vault getVaultClient(String address) throws VaultException { + final var config = new VaultConfig() + .address(address) + .engineVersion(2) + .build(); + return new Vault(config); + } +} diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index 894b9cbe6123..d8a96163f661 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -46,7 +46,9 @@ The following variables are relevant to both Docker and Kubernetes. #### Secrets 1. `SECRET_STORE_GCP_PROJECT_ID` - Defines the GCP Project to store secrets in. Alpha support. 2. `SECRET_STORE_GCP_CREDENTIALS` - Define the JSON credentials used to read/write Airbyte Configuration to Google Secret Manager. These credentials must have Secret Manager Read/Write access. Alpha support. -3. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Alpha support. Undefined behavior will result if this is turned on and then off. +3. `VAULT_ADDRESS` - Define the vault address to read/write Airbyte Configuration to Hashicorp Vault. Alpha Support. +4. `VAULT_PREFIX` - Define the vault path prefix. Empty by default. Alpha Support. +5. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use Hashicorp Vault. Alpha support. Undefined behavior will result if this is turned on and then off. #### Database 1. `DATABASE_USER` - Define the Jobs Database user. From 4c496f308b347a4f801e90e68db311f0df96d84c Mon Sep 17 00:00:00 2001 From: Stella Chung Date: Tue, 7 Jun 2022 14:49:35 -0700 Subject: [PATCH 02/12] Add test --- airbyte-config/persistence/build.gradle | 1 + .../split_secrets/VaultSecretPersistence.java | 24 +++++++- .../VaultSecretPersistenceTest.java | 58 +++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java diff --git a/airbyte-config/persistence/build.gradle b/airbyte-config/persistence/build.gradle index 004caf7b6611..9da3a1406041 100644 --- a/airbyte-config/persistence/build.gradle +++ b/airbyte-config/persistence/build.gradle @@ -20,5 +20,6 @@ dependencies { testImplementation libs.testcontainers.postgresql testImplementation libs.flyway.core testImplementation project(':airbyte-test-utils') + testImplementation "org.testcontainers:vault:1.17.2" integrationTestJavaImplementation project(':airbyte-config:persistence') } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index 912ecd140a37..46eef4e46d8e 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -24,15 +24,22 @@ public VaultSecretPersistence(final String address, final String prefix) { this.pathPrefix = prefix; } + /** + * Constructor for testing + */ + public VaultSecretPersistence(final String address, final String prefix, final String token) { + this.vault = Exceptions.toRuntime(() -> getVaultClient(address, token)); + this.pathPrefix = prefix; + } + @Override public Optional read(final SecretCoordinate coordinate) { try { final var response = vault.logical().read(pathPrefix + coordinate.getFullCoordinate()); - log.info(pathPrefix); final var restResponse = response.getRestResponse(); final var responseCode = restResponse.getStatus(); if (responseCode != 200) { - log.error("failed on read", response); + log.error("failed on read. Response code: " + responseCode); return Optional.empty(); } final var data = response.getData(); @@ -63,4 +70,17 @@ public static Vault getVaultClient(String address) throws VaultException { .build(); return new Vault(config); } + + + /** + * Vault client for testing + */ + public static Vault getVaultClient(String address, String token) throws VaultException { + final var config = new VaultConfig() + .address(address) + .token(token) + .engineVersion(2) + .build(); + return new Vault(config); + } } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java new file mode 100644 index 000000000000..a654c69dd260 --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java @@ -0,0 +1,58 @@ +package io.airbyte.config.persistence.split_secrets; + +import org.apache.commons.lang3.RandomUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.vault.VaultContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class VaultSecretPersistenceTest { + private VaultSecretPersistence persistence; + private String baseCoordinate; + + private VaultContainer vaultContainer; + + @BeforeEach + void setUp() { + vaultContainer = new VaultContainer("vault").withVaultToken("vault-dev-token-id"); + vaultContainer.start(); + + final var vaultAddress = "http://" + vaultContainer.getHost() + ":" + vaultContainer.getFirstMappedPort(); + + persistence = new VaultSecretPersistence(vaultAddress, "secret/testing", "vault-dev-token-id"); + baseCoordinate = "VaultSecretPersistenceIntegrationTest_coordinate_" + RandomUtils.nextInt() % 20000; + } + + @AfterEach + void tearDown() { + vaultContainer.stop(); + } + + @Test + void testReadWriteUpdate() { + final var coordinate1 = new SecretCoordinate(baseCoordinate, 1); + + // try reading non-existent value + final var firstRead = persistence.read(coordinate1); + assertTrue(firstRead.isEmpty()); + + // write + final var firstPayload = "abc"; + persistence.write(coordinate1, firstPayload); + final var secondRead = persistence.read(coordinate1); + assertTrue(secondRead.isPresent()); + assertEquals(firstPayload, secondRead.get()); + + // update + final var secondPayload = "def"; + final var coordinate2 = new SecretCoordinate(baseCoordinate, 2); + persistence.write(coordinate2, secondPayload); + final var thirdRead = persistence.read(coordinate2); + assertTrue(thirdRead.isPresent()); + assertEquals(secondPayload, thirdRead.get()); + } +} + From 613231eca195ad2d0dd50a2fbf91c63db51cb7bc Mon Sep 17 00:00:00 2001 From: Stella Chung Date: Tue, 7 Jun 2022 15:03:08 -0700 Subject: [PATCH 03/12] Add final --- .../persistence/split_secrets/VaultSecretPersistence.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index 46eef4e46d8e..24761360b6c6 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -63,7 +63,7 @@ public void write(final SecretCoordinate coordinate, final String payload) { /** * This creates a vault client using a vault agent which uses AWS IAM for auth. */ - public static Vault getVaultClient(String address) throws VaultException { + public static Vault getVaultClient(final String address) throws VaultException { final var config = new VaultConfig() .address(address) .engineVersion(2) @@ -71,11 +71,10 @@ public static Vault getVaultClient(String address) throws VaultException { return new Vault(config); } - /** * Vault client for testing */ - public static Vault getVaultClient(String address, String token) throws VaultException { + public static Vault getVaultClient(final String address, final String token) throws VaultException { final var config = new VaultConfig() .address(address) .token(token) From 9d600b6a3d8c44c2a0b845b004fb4eb09591f0eb Mon Sep 17 00:00:00 2001 From: Stella Chung Date: Wed, 8 Jun 2022 08:57:37 -0700 Subject: [PATCH 04/12] Adjust permissions --- .../persistence/split_secrets/VaultSecretPersistence.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index 24761360b6c6..a9d8e2559be7 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -27,7 +27,7 @@ public VaultSecretPersistence(final String address, final String prefix) { /** * Constructor for testing */ - public VaultSecretPersistence(final String address, final String prefix, final String token) { + protected VaultSecretPersistence(final String address, final String prefix, final String token) { this.vault = Exceptions.toRuntime(() -> getVaultClient(address, token)); this.pathPrefix = prefix; } @@ -61,9 +61,9 @@ public void write(final SecretCoordinate coordinate, final String payload) { } /** - * This creates a vault client using a vault agent which uses AWS IAM for auth. + * This creates a vault client using a vault agent which uses AWS IAM for auth using engine version 2. */ - public static Vault getVaultClient(final String address) throws VaultException { + private static Vault getVaultClient(final String address) throws VaultException { final var config = new VaultConfig() .address(address) .engineVersion(2) @@ -74,7 +74,7 @@ public static Vault getVaultClient(final String address) throws VaultException { /** * Vault client for testing */ - public static Vault getVaultClient(final String address, final String token) throws VaultException { + private static Vault getVaultClient(final String address, final String token) throws VaultException { final var config = new VaultConfig() .address(address) .token(token) From 23925c54ab2f5e8118ce212dd8b54b73819d0a3a Mon Sep 17 00:00:00 2001 From: Amanda Murphy Date: Thu, 16 Jun 2022 08:18:08 -0700 Subject: [PATCH 05/12] Address PR comments --- .../split_secrets/VaultSecretPersistence.java | 25 ++++++++++--------- .../VaultSecretPersistenceTest.java | 25 ++++++++++--------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index a9d8e2559be7..1adc7315fc6c 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -11,11 +11,12 @@ import java.util.HashMap; import java.util.Optional; import lombok.extern.slf4j.Slf4j; +import lombok.val; @Slf4j final public class VaultSecretPersistence implements SecretPersistence { - private final String secretKey = "value"; + private final String SECRET_KEY = "value"; private final Vault vault; private final String pathPrefix; @@ -35,15 +36,15 @@ protected VaultSecretPersistence(final String address, final String prefix, fina @Override public Optional read(final SecretCoordinate coordinate) { try { - final var response = vault.logical().read(pathPrefix + coordinate.getFullCoordinate()); - final var restResponse = response.getRestResponse(); - final var responseCode = restResponse.getStatus(); + val response = vault.logical().read(pathPrefix + coordinate.getFullCoordinate()); + val restResponse = response.getRestResponse(); + val responseCode = restResponse.getStatus(); if (responseCode != 200) { - log.error("failed on read. Response code: " + responseCode); + log.error("Vault failed on read. Response code: " + responseCode); return Optional.empty(); } - final var data = response.getData(); - return Optional.of(data.get(secretKey)); + val data = response.getData(); + return Optional.of(data.get(SECRET_KEY)); } catch (final VaultException e) { return Optional.empty(); } @@ -52,11 +53,11 @@ public Optional read(final SecretCoordinate coordinate) { @Override public void write(final SecretCoordinate coordinate, final String payload) { try { - final var newSecret = new HashMap(); - newSecret.put(secretKey, payload); + val newSecret = new HashMap(); + newSecret.put(SECRET_KEY, payload); vault.logical().write(pathPrefix + coordinate.getFullCoordinate(), newSecret); } catch (final VaultException e) { - log.error("failed on write", e); + log.error("Vault failed on write", e); } } @@ -64,7 +65,7 @@ public void write(final SecretCoordinate coordinate, final String payload) { * This creates a vault client using a vault agent which uses AWS IAM for auth using engine version 2. */ private static Vault getVaultClient(final String address) throws VaultException { - final var config = new VaultConfig() + val config = new VaultConfig() .address(address) .engineVersion(2) .build(); @@ -75,7 +76,7 @@ private static Vault getVaultClient(final String address) throws VaultException * Vault client for testing */ private static Vault getVaultClient(final String address, final String token) throws VaultException { - final var config = new VaultConfig() + val config = new VaultConfig() .address(address) .token(token) .engineVersion(2) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java index a654c69dd260..f1ad31ed4d0c 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java @@ -1,5 +1,6 @@ package io.airbyte.config.persistence.split_secrets; +import lombok.val; import org.apache.commons.lang3.RandomUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -7,7 +8,7 @@ import org.testcontainers.vault.VaultContainer; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; public class VaultSecretPersistenceTest { private VaultSecretPersistence persistence; @@ -20,7 +21,7 @@ void setUp() { vaultContainer = new VaultContainer("vault").withVaultToken("vault-dev-token-id"); vaultContainer.start(); - final var vaultAddress = "http://" + vaultContainer.getHost() + ":" + vaultContainer.getFirstMappedPort(); + val vaultAddress = "http://" + vaultContainer.getHost() + ":" + vaultContainer.getFirstMappedPort(); persistence = new VaultSecretPersistence(vaultAddress, "secret/testing", "vault-dev-token-id"); baseCoordinate = "VaultSecretPersistenceIntegrationTest_coordinate_" + RandomUtils.nextInt() % 20000; @@ -33,25 +34,25 @@ void tearDown() { @Test void testReadWriteUpdate() { - final var coordinate1 = new SecretCoordinate(baseCoordinate, 1); + val coordinate1 = new SecretCoordinate(baseCoordinate, 1); // try reading non-existent value - final var firstRead = persistence.read(coordinate1); - assertTrue(firstRead.isEmpty()); + val firstRead = persistence.read(coordinate1); + assertThat(firstRead.isEmpty()).isTrue(); // write - final var firstPayload = "abc"; + val firstPayload = "abc"; persistence.write(coordinate1, firstPayload); - final var secondRead = persistence.read(coordinate1); - assertTrue(secondRead.isPresent()); + val secondRead = persistence.read(coordinate1); + assertThat(secondRead.isPresent()).isTrue(); assertEquals(firstPayload, secondRead.get()); // update - final var secondPayload = "def"; - final var coordinate2 = new SecretCoordinate(baseCoordinate, 2); + val secondPayload = "def"; + val coordinate2 = new SecretCoordinate(baseCoordinate, 2); persistence.write(coordinate2, secondPayload); - final var thirdRead = persistence.read(coordinate2); - assertTrue(thirdRead.isPresent()); + val thirdRead = persistence.read(coordinate2); + assertThat(thirdRead.isPresent()).isTrue(); assertEquals(secondPayload, thirdRead.get()); } } From a7cf4bd5eba89f4cc5291b41fe0fe23d69fbf9a2 Mon Sep 17 00:00:00 2001 From: Amanda Murphy Date: Thu, 16 Jun 2022 16:41:30 -0700 Subject: [PATCH 06/12] make constructor public --- .../persistence/split_secrets/VaultSecretPersistence.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index 1adc7315fc6c..7afad993b2f5 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -28,7 +28,7 @@ public VaultSecretPersistence(final String address, final String prefix) { /** * Constructor for testing */ - protected VaultSecretPersistence(final String address, final String prefix, final String token) { + public VaultSecretPersistence(final String address, final String prefix, final String token) { this.vault = Exceptions.toRuntime(() -> getVaultClient(address, token)); this.pathPrefix = prefix; } From 8a44740bf91247731484aa0916644bf681ba7a2e Mon Sep 17 00:00:00 2001 From: Amanda Murphy Date: Thu, 16 Jun 2022 17:13:25 -0700 Subject: [PATCH 07/12] Add auth method --- docs/operator-guides/configuring-airbyte.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index 4f6146b563f2..b7737ec9c18a 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -48,7 +48,9 @@ The following variables are relevant to both Docker and Kubernetes. 2. `SECRET_STORE_GCP_CREDENTIALS` - Define the JSON credentials used to read/write Airbyte Configuration to Google Secret Manager. These credentials must have Secret Manager Read/Write access. Alpha support. 3. `VAULT_ADDRESS` - Define the vault address to read/write Airbyte Configuration to Hashicorp Vault. Alpha Support. 4. `VAULT_PREFIX` - Define the vault path prefix. Empty by default. Alpha Support. -5. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use Hashicorp Vault. Alpha support. Undefined behavior will result if this is turned on and then off. +5. `VAULT_TOKEN` - The token used for vault authentication. Alpha Support. +6. `VAULT_AUTH_METHOD` - How vault will preform authentication. Currently, only supports Token auth. Defaults to token. Alpha Support. +7. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use Hashicorp Vault. Alpha support. Undefined behavior will result if this is turned on and then off. #### Database 1. `DATABASE_USER` - Define the Jobs Database user. From 20c4bc6218351f33894a10e802f369d0e91f298b Mon Sep 17 00:00:00 2001 From: Amanda Murphy Date: Tue, 21 Jun 2022 14:30:10 -0700 Subject: [PATCH 08/12] Remove the bettercloud vault token environment variable and add one for airbyte --- .../src/main/java/io/airbyte/config/Configs.java | 6 ++++++ .../src/main/java/io/airbyte/config/EnvConfigs.java | 6 ++++++ .../persistence/split_secrets/SecretPersistence.java | 4 ++-- .../persistence/split_secrets/VaultSecretPersistence.java | 8 -------- docs/operator-guides/configuring-airbyte.md | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index 741b4932edb1..39bab8e3ee57 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -119,6 +119,12 @@ public interface Configs { */ String getVaultPrefix(); + /** + * Define the vault token to read/write Airbyte Configuration to Hashicorp Vault. Empty by + * default. Alpha Support. + */ + String getVaultToken(); + // Database /** * Define the Jobs Database user. diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 302be04adc2e..08ab4c5ce05b 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -159,6 +159,7 @@ public class EnvConfigs implements Configs { private static final String VAULT_ADDRESS = "VAULT_ADDRESS"; private static final String VAULT_PREFIX = "VAULT_PREFIX"; + private static final String VAULT_AUTH_TOKEN = "VAULT_AUTH_TOKEN"; public static final long DEFAULT_MAX_SPEC_WORKERS = 5; public static final long DEFAULT_MAX_CHECK_WORKERS = 5; @@ -341,6 +342,11 @@ public String getVaultPrefix() { return getEnvOrDefault(VAULT_PREFIX, ""); } + @Override + public String getVaultToken() { + return getEnvOrDefault(VAULT_AUTH_TOKEN, ""); + } + // Database @Override public String getDatabaseUser() { diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java index 300284a6828d..bd039f170d4e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java @@ -32,7 +32,7 @@ static Optional getLongLived(final DSLContext dslContext, fin return Optional.of(GoogleSecretManagerPersistence.getLongLived(configs.getSecretStoreGcpProjectId(), configs.getSecretStoreGcpCredentials())); } case VAULT -> { - return Optional.of(new VaultSecretPersistence(configs.getVaultAddress(), configs.getVaultPrefix())); + return Optional.of(new VaultSecretPersistence(configs.getVaultAddress(), configs.getVaultPrefix(), configs.getVaultToken())); } default -> { return Optional.empty(); @@ -60,7 +60,7 @@ static Optional getEphemeral(final DSLContext dslContext, fin return Optional.of(GoogleSecretManagerPersistence.getEphemeral(configs.getSecretStoreGcpProjectId(), configs.getSecretStoreGcpCredentials())); } case VAULT -> { - return Optional.of(new VaultSecretPersistence(configs.getVaultAddress(), configs.getVaultPrefix())); + return Optional.of(new VaultSecretPersistence(configs.getVaultAddress(), configs.getVaultPrefix(), configs.getVaultToken())); } default -> { return Optional.empty(); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index 7afad993b2f5..c2ef30456770 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -20,14 +20,6 @@ final public class VaultSecretPersistence implements SecretPersistence { private final Vault vault; private final String pathPrefix; - public VaultSecretPersistence(final String address, final String prefix) { - this.vault = Exceptions.toRuntime(() -> getVaultClient(address)); - this.pathPrefix = prefix; - } - - /** - * Constructor for testing - */ public VaultSecretPersistence(final String address, final String prefix, final String token) { this.vault = Exceptions.toRuntime(() -> getVaultClient(address, token)); this.pathPrefix = prefix; diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index b7737ec9c18a..0ab0da91ae7f 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -48,7 +48,7 @@ The following variables are relevant to both Docker and Kubernetes. 2. `SECRET_STORE_GCP_CREDENTIALS` - Define the JSON credentials used to read/write Airbyte Configuration to Google Secret Manager. These credentials must have Secret Manager Read/Write access. Alpha support. 3. `VAULT_ADDRESS` - Define the vault address to read/write Airbyte Configuration to Hashicorp Vault. Alpha Support. 4. `VAULT_PREFIX` - Define the vault path prefix. Empty by default. Alpha Support. -5. `VAULT_TOKEN` - The token used for vault authentication. Alpha Support. +5. `VAULT_AUTH_TOKEN` - The token used for vault authentication. Alpha Support. 6. `VAULT_AUTH_METHOD` - How vault will preform authentication. Currently, only supports Token auth. Defaults to token. Alpha Support. 7. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use Hashicorp Vault. Alpha support. Undefined behavior will result if this is turned on and then off. From a326740419dc5013a9230612c44a502e2c7f4dc9 Mon Sep 17 00:00:00 2001 From: Amanda Murphy Date: Tue, 21 Jun 2022 15:11:25 -0700 Subject: [PATCH 09/12] Remove unused client --- .../split_secrets/VaultSecretPersistence.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index c2ef30456770..2d7ecfb39003 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -53,20 +53,6 @@ public void write(final SecretCoordinate coordinate, final String payload) { } } - /** - * This creates a vault client using a vault agent which uses AWS IAM for auth using engine version 2. - */ - private static Vault getVaultClient(final String address) throws VaultException { - val config = new VaultConfig() - .address(address) - .engineVersion(2) - .build(); - return new Vault(config); - } - - /** - * Vault client for testing - */ private static Vault getVaultClient(final String address, final String token) throws VaultException { val config = new VaultConfig() .address(address) From 846742f86db5de126d924e560f82deca59775e20 Mon Sep 17 00:00:00 2001 From: Amanda Murphy Date: Tue, 21 Jun 2022 15:14:14 -0700 Subject: [PATCH 10/12] Update VAULT_AUTH_TOKEN to not default to empty string --- .../src/main/java/io/airbyte/config/EnvConfigs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 08ab4c5ce05b..fc65e254cca2 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -344,7 +344,7 @@ public String getVaultPrefix() { @Override public String getVaultToken() { - return getEnvOrDefault(VAULT_AUTH_TOKEN, ""); + return getEnv(VAULT_AUTH_TOKEN); } // Database From bcdf863bea159e6bdd70f2c4b9e53c0ae525d0ac Mon Sep 17 00:00:00 2001 From: A Murphy Date: Tue, 21 Jun 2022 18:36:28 -0700 Subject: [PATCH 11/12] Update docs/operator-guides/configuring-airbyte.md Co-authored-by: Benoit Moriceau --- docs/operator-guides/configuring-airbyte.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index 0ab0da91ae7f..5744b67ac584 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -50,7 +50,7 @@ The following variables are relevant to both Docker and Kubernetes. 4. `VAULT_PREFIX` - Define the vault path prefix. Empty by default. Alpha Support. 5. `VAULT_AUTH_TOKEN` - The token used for vault authentication. Alpha Support. 6. `VAULT_AUTH_METHOD` - How vault will preform authentication. Currently, only supports Token auth. Defaults to token. Alpha Support. -7. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use Hashicorp Vault. Alpha support. Undefined behavior will result if this is turned on and then off. +7. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use Hashicorp Vault, currently only the token based authentication is supported. Alpha support. Undefined behavior will result if this is turned on and then off. #### Database 1. `DATABASE_USER` - Define the Jobs Database user. From 3cae61e20905e12c8a930826abf039b6b62f87b5 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 23 Jun 2022 17:59:37 +0200 Subject: [PATCH 12/12] format --- .../src/main/java/io/airbyte/config/Configs.java | 4 ++-- .../split_secrets/VaultSecretPersistence.java | 1 + .../split_secrets/VaultSecretPersistenceTest.java | 13 +++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index 86e70ef0edaf..c480cefde298 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -120,8 +120,8 @@ public interface Configs { String getVaultPrefix(); /** - * Define the vault token to read/write Airbyte Configuration to Hashicorp Vault. Empty by - * default. Alpha Support. + * Define the vault token to read/write Airbyte Configuration to Hashicorp Vault. Empty by default. + * Alpha Support. */ String getVaultToken(); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index 2d7ecfb39003..066f06f109a6 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -61,4 +61,5 @@ private static Vault getVaultClient(final String address, final String token) th .build(); return new Vault(config); } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java index f1ad31ed4d0c..44251c5b6070 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistenceTest.java @@ -1,5 +1,12 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.config.persistence.split_secrets; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + import lombok.val; import org.apache.commons.lang3.RandomUtils; import org.junit.jupiter.api.AfterEach; @@ -7,10 +14,8 @@ import org.junit.jupiter.api.Test; import org.testcontainers.vault.VaultContainer; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.assertj.core.api.Assertions.assertThat; - public class VaultSecretPersistenceTest { + private VaultSecretPersistence persistence; private String baseCoordinate; @@ -55,5 +60,5 @@ void testReadWriteUpdate() { assertThat(thirdRead.isPresent()).isTrue(); assertEquals(secondPayload, thirdRead.get()); } -} +}