Fix S3RepositoryThirdPartyTests.testReadFromPositionLargerThanBlobLength (#106466)

The test should use a random operation purpose that is not "Indices", otherwise S3RetryingInputStream retries up to Integer.MAX_VALUE times which causes the test suite to timeout.

Also fixes the progress in the retries log messages.

Closes #106457

Author: tlrx

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java

@@ -7,7 +7,6 @@
  */
 package org.elasticsearch.repositories.s3;
 
-import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
@@ -26,6 +25,7 @@
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.indices.recovery.RecoverySettings;
@@ -44,12 +44,14 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -226,7 +228,6 @@ List<MultipartUpload> listMultipartUploads() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106457")
     public void testReadFromPositionLargerThanBlobLength() {
         final var blobName = randomIdentifier();
         final var blobBytes = randomBytesReference(randomIntBetween(100, 2_000));
@@ -239,9 +240,26 @@ public void testReadFromPositionLargerThanBlobLength() {
 
         long position = randomLongBetween(blobBytes.length(), Long.MAX_VALUE - 1L);
         long length = randomLongBetween(1L, Long.MAX_VALUE - position);
-        var exception = expectThrows(AmazonClientException.class, () -> readBlob(repository, blobName, position, length));
 
-        assertThat(exception, instanceOf(AmazonS3Exception.class));
-        assertThat(((AmazonS3Exception) exception).getStatusCode(), equalTo(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()));
+        var exception = expectThrows(UncategorizedExecutionException.class, () -> readBlob(repository, blobName, position, length));
+        assertThat(exception.getCause(), instanceOf(ExecutionException.class));
+        assertThat(exception.getCause().getCause(), instanceOf(IOException.class));
+        assertThat(
+            exception.getCause().getCause().getMessage(),
+            containsString(
+                "Requested range [start="
+                    + position
+                    + ", end="
+                    + (position + length - 1L)
+                    + ", currentOffset=0] cannot be satisfied for blob object ["
+                    + repository.basePath().buildAsString()
+                    + blobName
+                    + ']'
+            )
+        );
+        assertThat(
+            asInstanceOf(AmazonS3Exception.class, exception.getRootCause()).getStatusCode(),
+            equalTo(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus())
+        );
     }
 }

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.repositories.s3.S3BlobStore.Operation;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -94,16 +95,34 @@ private void openStreamWithRetry() throws IOException {
                         : "requesting beyond end, start = " + start + " offset=" + currentOffset + " end=" + end;
                     getObjectRequest.setRange(Math.addExact(start, currentOffset), end);
                 }
-                final S3Object s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest));
                 this.currentStreamFirstOffset = Math.addExact(start, currentOffset);
+                final S3Object s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest));
                 this.currentStreamLastOffset = Math.addExact(currentStreamFirstOffset, getStreamLength(s3Object));
                 this.currentStream = s3Object.getObjectContent();
                 return;
             } catch (AmazonClientException e) {
-                if (e instanceof AmazonS3Exception amazonS3Exception && 404 == amazonS3Exception.getStatusCode()) {
-                    throw addSuppressedExceptions(
-                        new NoSuchFileException("Blob object [" + blobKey + "] not found: " + amazonS3Exception.getMessage())
-                    );
+                if (e instanceof AmazonS3Exception amazonS3Exception) {
+                    if (amazonS3Exception.getStatusCode() == RestStatus.NOT_FOUND.getStatus()) {
+                        throw addSuppressedExceptions(
+                            new NoSuchFileException("Blob object [" + blobKey + "] not found: " + amazonS3Exception.getMessage())
+                        );
+                    }
+                    if (amazonS3Exception.getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) {
+                        throw addSuppressedExceptions(
+                            new IOException(
+                                "Requested range [start="
+                                    + start
+                                    + ", end="
+                                    + end
+                                    + ", currentOffset="
+                                    + currentOffset
+                                    + "] cannot be satisfied for blob object ["
+                                    + blobKey
+                                    + ']',
+                                amazonS3Exception
+                            )
+                        );
+                    }
                 }
 
                 if (attempt == 1) {