* Add exception mappers to convert storage failures to Iceberg REST c…

…lient exceptions

* Attempt recovery from failed tasks
  if/when they are re-submitted after a configurable delay.

* Add task deadlines (prevent indefinite retries)

* Add AccessCheckHandler to ObjectStorageMock for simulating access failures in tests.

Closes projectnessie#8738

catalog/files/api/src/main/java/org/projectnessie/catalog/files/api/NonRetryableException.java → catalog/files/api/src/main/java/org/projectnessie/catalog/files/api/ObjectIOConfig.java

@@ -15,29 +15,24 @@
  */
 package org.projectnessie.catalog.files.api;
 
-import java.time.Instant;
-import java.util.Optional;
+import java.time.Duration;
 
-public class NonRetryableException extends ObjectIOException {
-  public NonRetryableException(Throwable cause) {
-    super(cause);
-  }
+public interface ObjectIOConfig {
+  /**
+   * Interval after which a request is retried when storage I/O responds with some "retry later"
+   * response.
+   */
+  Duration retryAfterThrottled();
 
-  public NonRetryableException(String message) {
-    super(message);
-  }
+  /**
+   * Interval after which a request is retried in case of networks / routing errors (e.g. "Service
+   * Unavailable").
+   */
+  Duration retryAfterNetworkError();
 
-  public NonRetryableException(String message, Throwable cause) {
-    super(message, cause);
-  }
-
-  @Override
-  public boolean isRetryable() {
-    return false;
-  }
-
-  @Override
-  public Optional<Instant> retryNotBefore() {
-    return Optional.empty();
-  }
+  /**
+   * Interval after which new requests for data that previously failed to be retrieved from storage
+   * can be re-attempted.
+   */
+  Duration reattemptAfterFetchError();
 }

catalog/files/api/src/main/java/org/projectnessie/catalog/files/api/ObjectIOExceptionMapper.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.files.api;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import java.util.OptionalInt;
+import org.projectnessie.nessie.immutables.NessieImmutable;
+
+@NessieImmutable
+public abstract class ObjectIOExceptionMapper {
+
+  protected abstract List<Analyzer> analyzers();
+
+  protected abstract Duration retryAfterThrottled();
+
+  protected abstract Duration retryAfterNetworkError();
+
+  protected abstract Duration reattemptAfterFetchError();
+
+  protected abstract Clock clock();
+
+  public static ImmutableObjectIOExceptionMapper.Builder builder() {
+    return ImmutableObjectIOExceptionMapper.builder();
+  }
+
+  public Optional<ObjectIOStatus> analyze(Throwable ex) {
+    for (Throwable th = ex; th != null; th = th.getCause()) {
+      for (Analyzer analyzer : analyzers()) {
+        OptionalInt statusCode = analyzer.httpStatusCode(th);
+        if (statusCode.isPresent()) {
+          return Optional.of(toStatus(statusCode.getAsInt()));
+        }
+      }
+    }
+
+    return Optional.empty();
+  }
+
+  public OptionalInt httpStatusCode(Throwable ex) {
+    for (Throwable th = ex; th != null; th = th.getCause()) {
+      for (Analyzer analyzer : analyzers()) {
+        OptionalInt statusCode = analyzer.httpStatusCode(th);
+        if (statusCode.isPresent()) {
+          return statusCode;
+        }
+      }
+    }
+
+    return OptionalInt.empty();
+  }
+
+  private Instant retryAfter(Duration delay) {
+    return clock().instant().plus(delay);
+  }
+
+  private ObjectIOStatus toStatus(int httpStatus) {
+    switch (httpStatus) {
+      case 408:
+      case 425:
+      case 429:
+        return ObjectIOStatus.of(httpStatus, true, retryAfter(retryAfterThrottled()));
+
+      case 502:
+      case 503:
+      case 504:
+        return ObjectIOStatus.of(httpStatus, true, retryAfter(retryAfterNetworkError()));
+
+      default:
+        return ObjectIOStatus.of(httpStatus, false, retryAfter(reattemptAfterFetchError()));
+    }
+  }
+
+  public interface Analyzer {
+    OptionalInt httpStatusCode(Throwable th);
+  }
+}

catalog/files/api/src/main/java/org/projectnessie/catalog/files/api/ObjectIOException.java → catalog/files/api/src/main/java/org/projectnessie/catalog/files/api/ObjectIOStatus.java

@@ -15,24 +15,18 @@
  */
 package org.projectnessie.catalog.files.api;
 
-import java.io.IOException;
 import java.time.Instant;
-import java.util.Optional;
+import org.projectnessie.nessie.immutables.NessieImmutable;
 
-public abstract class ObjectIOException extends IOException {
-  public ObjectIOException(Throwable cause) {
-    super(cause);
-  }
+@NessieImmutable
+public interface ObjectIOStatus {
+  int httpStatusCode();
 
-  public ObjectIOException(String message) {
-    super(message);
-  }
+  boolean isRetryable();
 
-  public ObjectIOException(String message, Throwable cause) {
-    super(message, cause);
-  }
+  Instant reattemptAfter();
 
-  public abstract boolean isRetryable();
-
-  public abstract Optional<Instant> retryNotBefore();
+  static ObjectIOStatus of(int httpCode, boolean retryable, Instant reattemptAfter) {
+    return ImmutableObjectIOStatus.of(httpCode, retryable, reattemptAfter);
+  }
 }