diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicBidiUnbufferedWritableByteChannelTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicBidiUnbufferedWritableByteChannelTest.java index 475ba6b35c9f..11fcb3f450db 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicBidiUnbufferedWritableByteChannelTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicBidiUnbufferedWritableByteChannelTest.java @@ -128,23 +128,23 @@ public void scenario1() throws Exception { BidiWriteCtx writeCtx = new BidiWriteCtx<>(resumableWrite); SettableApiFuture done = SettableApiFuture.create(); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - ByteBuffer bb = DataGenerator.base64Characters().genByteBuffer(_256KiB); - StorageException se = assertThrows(StorageException.class, () -> channel.write(bb)); - assertAll( - () -> assertThat(se.getCode()).isEqualTo(0), - () -> assertThat(se.getReason()).isEqualTo("invalid"), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(0), - () -> assertThat(channel.isOpen()).isFalse()); + GrpcCallContext::createDefault)) { + + ByteBuffer bb = DataGenerator.base64Characters().genByteBuffer(_256KiB); + StorageException se = assertThrows(StorageException.class, () -> channel.write(bb)); + assertAll( + () -> assertThat(se.getCode()).isEqualTo(0), + () -> assertThat(se.getReason()).isEqualTo("invalid"), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(0), + () -> assertThat(channel.isOpen()).isFalse()); + } } } @@ -210,22 +210,22 @@ public void scenario2() throws Exception { writeCtx.getTotalSentBytes().set(_256KiB); writeCtx.getConfirmedBytes().set(_256KiB); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - StorageException se = assertThrows(StorageException.class, channel::close); - assertAll( - () -> assertThat(se.getCode()).isEqualTo(0), - () -> assertThat(se.getReason()).isEqualTo("invalid"), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_256KiB), - () -> assertThat(channel.isOpen()).isFalse()); + GrpcCallContext::createDefault)) { + + StorageException se = assertThrows(StorageException.class, channel::close); + assertAll( + () -> assertThat(se.getCode()).isEqualTo(0), + () -> assertThat(se.getReason()).isEqualTo("invalid"), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_256KiB), + () -> assertThat(channel.isOpen()).isFalse()); + } } } @@ -291,22 +291,22 @@ public void scenario3() throws Exception { writeCtx.getTotalSentBytes().set(_512KiB); writeCtx.getConfirmedBytes().set(_512KiB); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - StorageException se = assertThrows(StorageException.class, channel::close); - assertAll( - () -> assertThat(se.getCode()).isEqualTo(0), - () -> assertThat(se.getReason()).isEqualTo("dataLoss"), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_512KiB), - () -> assertThat(channel.isOpen()).isFalse()); + GrpcCallContext::createDefault)) { + + StorageException se = assertThrows(StorageException.class, channel::close); + assertAll( + () -> assertThat(se.getCode()).isEqualTo(0), + () -> assertThat(se.getReason()).isEqualTo("dataLoss"), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_512KiB), + () -> assertThat(channel.isOpen()).isFalse()); + } } } @@ -374,19 +374,20 @@ public void scenario4() throws Exception { writeCtx.getTotalSentBytes().set(_256KiB); writeCtx.getConfirmedBytes().set(_256KiB); - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); + GrpcCallContext::createDefault)) { - channel.close(); + channel.close(); - BidiWriteObjectResponse BidiWriteObjectResponse = done.get(2, TimeUnit.SECONDS); - assertThat(BidiWriteObjectResponse).isEqualTo(resp1); + BidiWriteObjectResponse BidiWriteObjectResponse = done.get(2, TimeUnit.SECONDS); + assertThat(BidiWriteObjectResponse).isEqualTo(resp1); + } } } @@ -454,22 +455,22 @@ public void scenario4_1() throws Exception { writeCtx.getTotalSentBytes().set(_512KiB); writeCtx.getConfirmedBytes().set(_512KiB); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - StorageException se = assertThrows(StorageException.class, channel::close); - assertAll( - () -> assertThat(se.getCode()).isEqualTo(0), - () -> assertThat(se.getReason()).isEqualTo("dataLoss"), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_512KiB), - () -> assertThat(channel.isOpen()).isFalse()); + GrpcCallContext::createDefault)) { + + StorageException se = assertThrows(StorageException.class, channel::close); + assertAll( + () -> assertThat(se.getCode()).isEqualTo(0), + () -> assertThat(se.getReason()).isEqualTo("dataLoss"), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_512KiB), + () -> assertThat(channel.isOpen()).isFalse()); + } } } @@ -537,22 +538,22 @@ public void scenario4_2() throws Exception { writeCtx.getTotalSentBytes().set(_512KiB); writeCtx.getConfirmedBytes().set(_512KiB); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - StorageException se = assertThrows(StorageException.class, channel::close); - assertAll( - () -> assertThat(se.getCode()).isEqualTo(0), - () -> assertThat(se.getReason()).isEqualTo("dataLoss"), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_512KiB), - () -> assertThat(channel.isOpen()).isFalse()); + GrpcCallContext::createDefault)) { + + StorageException se = assertThrows(StorageException.class, channel::close); + assertAll( + () -> assertThat(se.getCode()).isEqualTo(0), + () -> assertThat(se.getReason()).isEqualTo("dataLoss"), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_512KiB), + () -> assertThat(channel.isOpen()).isFalse()); + } } } @@ -630,23 +631,23 @@ public void scenario5() throws Exception { writeCtx.getTotalSentBytes().set(_256KiB); writeCtx.getConfirmedBytes().set(_256KiB); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - ByteBuffer bb = DataGenerator.base64Characters().genByteBuffer(_256KiB); - StorageException se = assertThrows(StorageException.class, () -> channel.write(bb)); - assertAll( - () -> assertThat(se.getCode()).isEqualTo(0), - () -> assertThat(se.getReason()).isEqualTo("dataLoss"), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_256KiB), - () -> assertThat(channel.isOpen()).isFalse()); + GrpcCallContext::createDefault)) { + + ByteBuffer bb = DataGenerator.base64Characters().genByteBuffer(_256KiB); + StorageException se = assertThrows(StorageException.class, () -> channel.write(bb)); + assertAll( + () -> assertThat(se.getCode()).isEqualTo(0), + () -> assertThat(se.getReason()).isEqualTo("dataLoss"), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_256KiB), + () -> assertThat(channel.isOpen()).isFalse()); + } } } @@ -694,23 +695,23 @@ public void scenario7() throws Exception { BidiResumableWrite resumableWrite = getResumableWrite(uploadId); BidiWriteCtx writeCtx = new BidiWriteCtx<>(resumableWrite); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - ByteBuffer buf = DataGenerator.base64Characters().genByteBuffer(_256KiB); - StorageException se = assertThrows(StorageException.class, () -> channel.write(buf)); - assertAll( - () -> assertThat(se.getCode()).isEqualTo(0), - () -> assertThat(se.getReason()).isEqualTo("dataLoss"), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(0), - () -> assertThat(channel.isOpen()).isFalse()); + GrpcCallContext::createDefault)) { + + ByteBuffer buf = DataGenerator.base64Characters().genByteBuffer(_256KiB); + StorageException se = assertThrows(StorageException.class, () -> channel.write(buf)); + assertAll( + () -> assertThat(se.getCode()).isEqualTo(0), + () -> assertThat(se.getReason()).isEqualTo("dataLoss"), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(0), + () -> assertThat(channel.isOpen()).isFalse()); + } } } @@ -744,23 +745,23 @@ public void incremental_success() throws Exception { BidiResumableWrite resumableWrite = getResumableWrite(uploadId); BidiWriteCtx writeCtx = new BidiWriteCtx<>(resumableWrite); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, CHUNK_SEGMENTER, writeCtx, - GrpcCallContext::createDefault); - - ByteBuffer buf = DataGenerator.base64Characters().genByteBuffer(_256KiB); - int written = channel.write(buf); - assertAll( - () -> assertThat(buf.remaining()).isEqualTo(0), - () -> assertThat(written).isEqualTo(_256KiB), - () -> assertThat(writeCtx.getTotalSentBytes().get()).isEqualTo(_256KiB), - () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_256KiB)); + GrpcCallContext::createDefault)) { + + ByteBuffer buf = DataGenerator.base64Characters().genByteBuffer(_256KiB); + int written = channel.write(buf); + assertAll( + () -> assertThat(buf.remaining()).isEqualTo(0), + () -> assertThat(written).isEqualTo(_256KiB), + () -> assertThat(writeCtx.getTotalSentBytes().get()).isEqualTo(_256KiB), + () -> assertThat(writeCtx.getConfirmedBytes().get()).isEqualTo(_256KiB)); + } } } @@ -796,29 +797,29 @@ public void incremental_partialSuccess() throws Exception { ChunkSegmenter chunkSegmenter = new ChunkSegmenter(Hasher.noop(), ByteStringStrategy.copy(), _512KiB, _256KiB); - //noinspection resource - GapicBidiUnbufferedWritableByteChannel channel = + try (GapicBidiUnbufferedWritableByteChannel channel = new GapicBidiUnbufferedWritableByteChannel( storageClient.bidiWriteObjectCallable(), RetrierWithAlg.attemptOnce(), done, chunkSegmenter, writeCtx, - GrpcCallContext::createDefault); - - ByteBuffer buf = DataGenerator.base64Characters().genByteBuffer(_512KiB); - int written = channel.write(buf); - assertAll( - () -> assertThat(buf.remaining()).isEqualTo(_256KiB), - () -> assertThat(written).isEqualTo(_256KiB), - () -> - assertWithMessage("totalSentBytes") - .that(writeCtx.getTotalSentBytes().get()) - .isEqualTo(_256KiB), - () -> - assertWithMessage("confirmedBytes") - .that(writeCtx.getConfirmedBytes().get()) - .isEqualTo(_256KiB)); + GrpcCallContext::createDefault)) { + + ByteBuffer buf = DataGenerator.base64Characters().genByteBuffer(_512KiB); + int written = channel.write(buf); + assertAll( + () -> assertThat(buf.remaining()).isEqualTo(_256KiB), + () -> assertThat(written).isEqualTo(_256KiB), + () -> + assertWithMessage("totalSentBytes") + .that(writeCtx.getTotalSentBytes().get()) + .isEqualTo(_256KiB), + () -> + assertWithMessage("confirmedBytes") + .that(writeCtx.getConfirmedBytes().get()) + .isEqualTo(_256KiB)); + } } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITOpenTelemetryMPUTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITOpenTelemetryMPUTest.java index e1a83ba6ebda..7a457167353b 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITOpenTelemetryMPUTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITOpenTelemetryMPUTest.java @@ -62,87 +62,88 @@ public final class ITOpenTelemetryMPUTest { public void checkMPUInstrumentation() throws Exception { TestExporter exporter = new TestExporter(); - OpenTelemetrySdk openTelemetrySdk = + try (OpenTelemetrySdk openTelemetrySdk = OpenTelemetrySdk.builder() .setTracerProvider( SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(exporter)) .build()) - .build(); - - HttpStorageOptions httpStorageOptions = (HttpStorageOptions) storage.getOptions(); - StorageOptions storageOptions = - httpStorageOptions.toBuilder().setOpenTelemetry(openTelemetrySdk).build(); - - String objectName = generator.randomObjectName(); - - try (Storage storage = storageOptions.getService()) { - MultipartUploadClient mpuClient = - MultipartUploadClient.create( - MultipartUploadSettings.of((HttpStorageOptions) storage.getOptions())); - - CreateMultipartUploadResponse create = - mpuClient.createMultipartUpload( - CreateMultipartUploadRequest.builder() - .bucket(bucket.getName()) - .key(objectName) - .build()); - - byte[] data = "Hello, World!".getBytes(StandardCharsets.UTF_8); - RequestBody body = RequestBody.of(ByteBuffer.wrap(data)); - UploadPartResponse upload = - mpuClient.uploadPart( - UploadPartRequest.builder() - .bucket(bucket.getName()) - .key(objectName) - .uploadId(create.uploadId()) - .partNumber(1) - .build(), - body); - - mpuClient.completeMultipartUpload( - CompleteMultipartUploadRequest.builder() - .bucket(bucket.getName()) - .key(objectName) - .uploadId(create.uploadId()) - .multipartUpload( - CompletedMultipartUpload.builder() - .parts( - ImmutableList.of( - CompletedPart.builder().partNumber(1).eTag(upload.eTag()).build())) - .build()) - .build()); - - mpuClient.listMultipartUploads( - ListMultipartUploadsRequest.builder().bucket(bucket.getName()).build()); + .build()) { + + HttpStorageOptions httpStorageOptions = (HttpStorageOptions) storage.getOptions(); + StorageOptions storageOptions = + httpStorageOptions.toBuilder().setOpenTelemetry(openTelemetrySdk).build(); + + String objectName = generator.randomObjectName(); + + try (Storage storage = storageOptions.getService()) { + MultipartUploadClient mpuClient = + MultipartUploadClient.create( + MultipartUploadSettings.of((HttpStorageOptions) storage.getOptions())); + + CreateMultipartUploadResponse create = + mpuClient.createMultipartUpload( + CreateMultipartUploadRequest.builder() + .bucket(bucket.getName()) + .key(objectName) + .build()); + + byte[] data = "Hello, World!".getBytes(StandardCharsets.UTF_8); + RequestBody body = RequestBody.of(ByteBuffer.wrap(data)); + UploadPartResponse upload = + mpuClient.uploadPart( + UploadPartRequest.builder() + .bucket(bucket.getName()) + .key(objectName) + .uploadId(create.uploadId()) + .partNumber(1) + .build(), + body); + + mpuClient.completeMultipartUpload( + CompleteMultipartUploadRequest.builder() + .bucket(bucket.getName()) + .key(objectName) + .uploadId(create.uploadId()) + .multipartUpload( + CompletedMultipartUpload.builder() + .parts( + ImmutableList.of( + CompletedPart.builder().partNumber(1).eTag(upload.eTag()).build())) + .build()) + .build()); + + mpuClient.listMultipartUploads( + ListMultipartUploadsRequest.builder().bucket(bucket.getName()).build()); + } + + List spans = exporter.getExportedSpans(); + assertThat(spans).hasSize(4); + + SpanData createSpan = spans.get(0); + assertThat(createSpan.getName()) + .isEqualTo("com.google.cloud.storage.MultipartUploadClient/createMultipartUpload"); + assertThat(createSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) + .isEqualTo(String.format("gs://%s/%s", bucket.getName(), objectName)); + + SpanData uploadSpan = spans.get(1); + assertThat(uploadSpan.getName()) + .isEqualTo("com.google.cloud.storage.MultipartUploadClient/uploadPart"); + assertThat(uploadSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) + .isEqualTo(String.format("gs://%s/%s", bucket.getName(), objectName)); + assertThat(uploadSpan.getAttributes().get(AttributeKey.longKey("partNumber"))).isEqualTo(1); + + SpanData completeSpan = spans.get(2); + assertThat(completeSpan.getName()) + .isEqualTo("com.google.cloud.storage.MultipartUploadClient/completeMultipartUpload"); + assertThat(completeSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) + .isEqualTo(String.format("gs://%s/%s", bucket.getName(), objectName)); + + SpanData listSpan = spans.get(3); + assertThat(listSpan.getName()) + .isEqualTo("com.google.cloud.storage.MultipartUploadClient/listMultipartUploads"); + assertThat(listSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) + .isEqualTo(String.format("gs://%s/", bucket.getName())); } - - List spans = exporter.getExportedSpans(); - assertThat(spans).hasSize(4); - - SpanData createSpan = spans.get(0); - assertThat(createSpan.getName()) - .isEqualTo("com.google.cloud.storage.MultipartUploadClient/createMultipartUpload"); - assertThat(createSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) - .isEqualTo(String.format("gs://%s/%s", bucket.getName(), objectName)); - - SpanData uploadSpan = spans.get(1); - assertThat(uploadSpan.getName()) - .isEqualTo("com.google.cloud.storage.MultipartUploadClient/uploadPart"); - assertThat(uploadSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) - .isEqualTo(String.format("gs://%s/%s", bucket.getName(), objectName)); - assertThat(uploadSpan.getAttributes().get(AttributeKey.longKey("partNumber"))).isEqualTo(1); - - SpanData completeSpan = spans.get(2); - assertThat(completeSpan.getName()) - .isEqualTo("com.google.cloud.storage.MultipartUploadClient/completeMultipartUpload"); - assertThat(completeSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) - .isEqualTo(String.format("gs://%s/%s", bucket.getName(), objectName)); - - SpanData listSpan = spans.get(3); - assertThat(listSpan.getName()) - .isEqualTo("com.google.cloud.storage.MultipartUploadClient/listMultipartUploads"); - assertThat(listSpan.getAttributes().get(AttributeKey.stringKey("gsutil.uri"))) - .isEqualTo(String.format("gs://%s/", bucket.getName())); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java index 1cfd80cd692e..a4d8512a83e6 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java @@ -268,16 +268,18 @@ List getRetryTestCases() throws IOException { private RetryTests loadRetryTestsDefinition() throws IOException { ClassLoader cl = Thread.currentThread().getContextClassLoader(); - InputStream dataJson = cl.getResourceAsStream(retryTestsJsonResourcePath); - assertNotNull( - String.format( - Locale.US, "Unable to load test definition: %s", retryTestsJsonResourcePath), - dataJson); - - InputStreamReader reader = new InputStreamReader(dataJson, Charsets.UTF_8); - RetryTests.Builder testBuilder = RetryTests.newBuilder(); - JsonFormat.parser().merge(reader, testBuilder); - return testBuilder.build(); + try (InputStream dataJson = cl.getResourceAsStream(retryTestsJsonResourcePath)) { + assertNotNull( + String.format( + Locale.US, "Unable to load test definition: %s", retryTestsJsonResourcePath), + dataJson); + + try (InputStreamReader reader = new InputStreamReader(dataJson, Charsets.UTF_8)) { + RetryTests.Builder testBuilder = RetryTests.newBuilder(); + JsonFormat.parser().merge(reader, testBuilder); + return testBuilder.build(); + } + } } /** Permute the RetryTest, Instructions and methods with our mappings */ diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java index 4d9340762093..89c6339d94f4 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/TestBench.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -63,30 +63,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * A {@link ManagedLifecycle} which integrates with the storage-testbench by pulling the - * docker image, starting the container, providing methods for interacting with the {@code - * /retry_test} rest api, stopping the container. - * - *

A single instance of the testbench is expected to be managed by the {@link - * com.google.cloud.storage.it.runner.registry.Registry} which is used by {@link - * com.google.cloud.storage.it.runner.StorageITRunner}. Accessing the testbench can be accomplished - * by doing the following: - * - *

    - *
  1. Annotating your test class {@code @RunWith(StorageITRunner.class)} - *
  2. Configuring the backend for your integration tests to be {@link - * com.google.cloud.storage.it.runner.annotations.Backend#TEST_BENCH} by doing either - *
      - *
    1. Annotating your test class with {@code @SingleBackend(Backend.TEST_BENCH)} - *
    2. Annotating your test class with {@code @CrossRun} and ensuring {@code - * Backend.TEST_BENCH} is included in the {@code backends} parameter - *
    - *
  3. Specifying {@code @Inject public TestBench testBench;} as a field for the instance of - * testbench to be injected to your test - *
- */ public final class TestBench implements ManagedLifecycle { private static final Logger LOGGER = LoggerFactory.getLogger(TestBench.class); @@ -115,7 +91,7 @@ private TestBench( String dockerImageName, String dockerImageTag, String containerName) { - this.ignorePullError = true; + this.ignorePullError = ignorePullError; this.baseUri = baseUri; this.gRPCBaseUri = gRPCBaseUri; this.dockerImageName = dockerImageName; @@ -133,6 +109,8 @@ private TestBench( .getHeaders() .setUserAgent( String.format(Locale.US, "%s/ test-bench/", this.containerName)); + request.setConnectTimeout(15000); + request.setReadTimeout(15000); }); } @@ -210,19 +188,26 @@ public void start() { // expected when the server isn't running already } try { - tempDirectory = Files.createTempDirectory(containerName); - outPath = tempDirectory.resolve("stdout"); - errPath = tempDirectory.resolve("stderr"); + // Route logs to Bazel/Sponge artifact directories so Test Fusion can see them + String bazelOutputDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR"); + Path baseArtifactDir; + if (bazelOutputDir != null && !bazelOutputDir.isEmpty()) { + baseArtifactDir = java.nio.file.Paths.get(bazelOutputDir); + } else { + baseArtifactDir = java.nio.file.Paths.get("target", "testbench-logs"); + } + + tempDirectory = baseArtifactDir.resolve(containerName); + Files.createDirectories(tempDirectory); + outPath = tempDirectory.resolve("gunicorn-stdout.log"); + errPath = tempDirectory.resolve("gunicorn-stderr.log"); File outFile = outPath.toFile(); File errFile = errPath.toFile(); - LOGGER.info("Redirecting server stdout to: {}", outFile.getAbsolutePath()); - LOGGER.info("Redirecting server stderr to: {}", errFile.getAbsolutePath()); + LOGGER.info("Redirecting server stdout to artifact: {}", outFile.getAbsolutePath()); + LOGGER.info("Redirecting server stderr to artifact: {}", errFile.getAbsolutePath()); + String dockerImage = String.format(Locale.US, "%s:%s", dockerImageName, dockerImageTag); - // First try and pull the docker image, this validates docker is available and running - // on the host, as well as gives time for the image to be downloaded independently of - // trying to start the container. (Below, when we first start the container we then attempt - // to issue a call against the api before we yield to run our tests.) try { Process p = new ProcessBuilder() @@ -248,6 +233,7 @@ public void start() { int port = URI.create(baseUri).getPort(); int gRPCPort = URI.create(gRPCBaseUri).getPort(); + final List command = ImmutableList.of( "docker", @@ -262,22 +248,34 @@ public void start() { dockerImage, "gunicorn", "--bind=0.0.0.0:9000", - "--worker-class=sync", - "--threads=10", + "--worker-class=gthread", + "--threads=40", // High threads to prevent deadlock "--access-logfile=-", + "--error-logfile=-", + "--log-level=debug", // Detailed Python output "--keep-alive=0", "testbench:run()"); + process = new ProcessBuilder() .command(command) - .redirectOutput(outFile) + .redirectOutput(outFile) // OS handles writing, NO Java threads used here .redirectError(errFile) .start(); LOGGER.info(command.toString()); + try { - // wait a small amount of time for the server to come up before probing Thread.sleep(500); - // wait for the server to come up + + // Fail fast if container crashed immediately due to a port collision + if (!process.isAlive()) { + dumpServerLogs(outPath, errPath); + throw new IllegalStateException( + "TestBench Docker container died immediately. Exit code: " + + process.exitValue() + + ". Probable port collision."); + } + List existingResources = runWithRetries( TestBench.this::listRetryTests, @@ -299,7 +297,6 @@ public boolean shouldRetry( LOGGER.info( "Test Server already has retry tests in it, is it running outside the tests?"); } - // Start gRPC Service if (!startGRPCServer(gRPCPort)) { throw new IllegalStateException( "Failed to start server within a reasonable amount of time. Host url(gRPC): " @@ -319,7 +316,6 @@ public boolean shouldRetry( @Override public void stop() { if (runningOutsideAlready) { - // if the server was running outside the tests already simply return return; } try { @@ -346,13 +342,11 @@ public void stop() { LOGGER.warn("Container exit value = {}", shutdownProcessExitValue); } - // wait for the server to shutdown runWithRetries( () -> { try { listRetryTests(); } catch (SocketException e) { - // desired result return null; } throw new NotShutdownException(); @@ -370,13 +364,10 @@ public boolean shouldRetry(Throwable previousThrowable, List previousResponse } }, NanoClock.getDefaultClock()); - try { - Files.delete(errPath); - Files.delete(outPath); - Files.delete(tempDirectory); - } catch (IOException e) { - throw new RuntimeException(e); - } + + // Intentionally NOT deleting the log files here so Test Fusion can archive them. + LOGGER.info("Skipping artifact deletion to preserve logs for Test Fusion."); + } catch (InterruptedException | IOException e) { throw new RuntimeException(e); } @@ -393,6 +384,7 @@ private void dumpServerLogs(Path outFile, Path errFile) throws IOException { } private void dumpServerLog(String prefix, File out) throws IOException { + if (!out.exists()) return; try (BufferedReader reader = new BufferedReader(new FileReader(out))) { String line; while ((line = reader.readLine()) != null) { @@ -401,6 +393,16 @@ private void dumpServerLog(String prefix, File out) throws IOException { } } + private static int findFreePort() { + try (java.net.ServerSocket socket = new java.net.ServerSocket()) { + socket.setReuseAddress(true); + socket.bind(new java.net.InetSocketAddress(0)); + return socket.getLocalPort(); + } catch (java.io.IOException e) { + throw new RuntimeException("Failed to find a free port", e); + } + } + static Builder newBuilder() { return new Builder(); } @@ -455,7 +457,6 @@ static final class Builder { InputStream dockerfileText = cl.getResourceAsStream( "com/google/cloud/storage/it/runner/registry/Dockerfile"); - //noinspection UnstableApiUsage return Optional.ofNullable(dockerfileText) .map(is -> new InputStreamReader(is, Charsets.UTF_8)) .flatMap( @@ -497,11 +498,11 @@ static final class Builder { private Builder() { this( false, - DEFAULT_BASE_URI, - DEFAULT_GRPC_BASE_URI, + "http://127.0.0.1:" + findFreePort(), + "http://127.0.0.1:" + findFreePort(), DEFAULT_IMAGE_NAME, DEFAULT_IMAGE_TAG, - DEFAULT_CONTAINER_NAME); + DEFAULT_CONTAINER_NAME + "_" + java.util.UUID.randomUUID().toString().substring(0, 8)); } private Builder( @@ -517,6 +518,12 @@ private Builder( this.dockerImageName = dockerImageName; this.dockerImageTag = dockerImageTag; this.containerName = containerName; + + LOGGER.info( + "DEBUG-BUILDER: Initialized testbench config -> Container: {}, HTTP: {}, GRPC: {}", + containerName, + baseUri, + gRPCBaseUri); } public Builder setIgnorePullError(boolean ignorePullError) {