Nessie CLI: Add initial Iceberg REST functionality (#8511)

`CONNECT TO` and `USE` working, `SHOW TABLE/VIEW` displays the table/view metadata.

Also adds the `ON initialReference` clause to `CONNECT TO`.

catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergFixtures.java

@@ -45,6 +45,8 @@
 
 import java.util.Iterator;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.IntSupplier;
 import java.util.stream.Stream;
 import org.projectnessie.catalog.formats.iceberg.meta.IcebergBlobMetadata;
 import org.projectnessie.catalog.formats.iceberg.meta.IcebergNestedField;
@@ -59,6 +61,11 @@
 
 public class IcebergFixtures {
   public static Stream<IcebergType> icebergTypes() {
+    AtomicInteger idGen = new AtomicInteger();
+    return icebergTypes(idGen::incrementAndGet);
+  }
+
+  public static Stream<IcebergType> icebergTypes(IntSupplier idSupplier) {
     return Stream.of(
         booleanType(),
         uuidType(),
@@ -70,16 +77,20 @@ public static Stream<IcebergType> icebergTypes() {
         doubleType(),
         dateType(),
         timeType(),
-        structType(singletonList(nestedField(11, "field11", true, stringType(), null)), null),
-        structType(singletonList(nestedField(11, "field11", false, stringType(), null)), null),
-        listType(1, stringType(), true),
-        listType(1, stringType(), false),
-        listType(1, uuidType(), true),
-        listType(1, uuidType(), false),
-        mapType(3, stringType(), 4, dateType(), true),
-        mapType(3, stringType(), 4, dateType(), false),
-        mapType(3, uuidType(), 4, timeType(), true),
-        mapType(3, uuidType(), 4, timeType(), false),
+        structType(
+            singletonList(nestedField(idSupplier.getAsInt(), "field11", true, stringType(), null)),
+            null),
+        structType(
+            singletonList(nestedField(idSupplier.getAsInt(), "field11", false, stringType(), null)),
+            null),
+        listType(idSupplier.getAsInt(), stringType(), true),
+        listType(idSupplier.getAsInt(), stringType(), false),
+        listType(idSupplier.getAsInt(), uuidType(), true),
+        listType(idSupplier.getAsInt(), uuidType(), false),
+        mapType(idSupplier.getAsInt(), stringType(), idSupplier.getAsInt(), dateType(), true),
+        mapType(idSupplier.getAsInt(), stringType(), idSupplier.getAsInt(), dateType(), false),
+        mapType(idSupplier.getAsInt(), uuidType(), idSupplier.getAsInt(), timeType(), true),
+        mapType(idSupplier.getAsInt(), uuidType(), idSupplier.getAsInt(), timeType(), false),
         decimalType(10, 3),
         fixedType(42),
         timestampType(),

catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergGenerateFixtures.java

@@ -32,10 +32,12 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.zip.GZIPOutputStream;
 import org.projectnessie.catalog.formats.iceberg.meta.IcebergJson;
 import org.projectnessie.catalog.formats.iceberg.meta.IcebergSnapshot;
+import org.projectnessie.catalog.formats.iceberg.meta.IcebergSortOrder;
 import org.projectnessie.catalog.formats.iceberg.meta.IcebergTableMetadata;
 
 public class IcebergGenerateFixtures {
@@ -94,6 +96,12 @@ public static String generateSimpleMetadata(ObjectWriter writer, int icebergSpec
 
   public static String generateMetadataWithManifestList(String basePath, ObjectWriter writer)
       throws Exception {
+    return generateMetadataWithManifestList(basePath, writer, m -> {});
+  }
+
+  public static String generateMetadataWithManifestList(
+      String basePath, ObjectWriter writer, Consumer<IcebergTableMetadata> metadataConsumer)
+      throws Exception {
     IcebergSchemaGenerator schemaGenerator =
         IcebergSchemaGenerator.spec().numColumns(10).numPartitionColumns(2).generate();
 
@@ -109,6 +117,7 @@ public void close() throws IOException {
     UUID commitId = randomUUID();
     long snapshotId = 1;
     long sequenceNumber = 1;
+    long timestamp = 1715175169320L;
     IcebergManifestFileGenerator manifestFileGenerator =
         IcebergManifestFileGenerator.builder()
             .addDataFiles(3)
@@ -133,7 +142,7 @@ public void close() throws IOException {
     IcebergSnapshot snapshotWithManifestList =
         IcebergSnapshot.builder()
             .snapshotId(snapshotId)
-            .timestampMs(System.currentTimeMillis())
+            .timestampMs(timestamp)
             .schemaId(schemaGenerator.getIcebergSchema().schemaId())
             .sequenceNumber(sequenceNumber)
             .manifestList(manifestList)
@@ -142,12 +151,17 @@ public void close() throws IOException {
     IcebergTableMetadata icebergMetadataWithManifestList =
         IcebergTableMetadata.builder()
             .from(tableMetadataSimple().formatVersion(2).build())
+            .lastUpdatedMs(timestamp)
             .schemas(singletonList(schemaGenerator.getIcebergSchema()))
             .partitionSpecs(singletonList(schemaGenerator.getIcebergPartitionSpec()))
+            .sortOrders(singletonList(IcebergSortOrder.UNSORTED_ORDER))
             .currentSnapshotId(snapshotId)
             .defaultSpecId(schemaGenerator.getIcebergPartitionSpec().specId())
+            .defaultSortOrderId(IcebergSortOrder.UNSORTED_ORDER.orderId())
+            .lastSequenceNumber(1L)
             .snapshots(singletonList(snapshotWithManifestList))
             .build();
+    metadataConsumer.accept(icebergMetadataWithManifestList);
     return writer.write(
         URI.create("table-metadata-with-manifest-list/table-metadata-with-manifest-list.json"),
         IcebergJson.objectMapper()

cli/cli/build.gradle.kts

@@ -28,6 +28,10 @@ extra["maven.name"] = "Nessie - CLI"
 
 configurations.all { exclude(group = "org.projectnessie.nessie", module = "nessie-model") }
 
+val versionIceberg = libs.versions.iceberg.get()
+
+val nessieQuarkusServer by configurations.creating
+
 dependencies {
   implementation(project(":nessie-model-quarkus"))
   implementation(project(":nessie-client"))
@@ -37,6 +41,10 @@ dependencies {
   implementation(libs.jline)
   implementation(libs.picocli)
 
+  implementation(platform("org.apache.iceberg:iceberg-bom:$versionIceberg"))
+  implementation("org.apache.iceberg:iceberg-core")
+  runtimeOnly(libs.hadoop.common) { isTransitive = false }
+
   compileOnly(libs.immutables.value.annotations)
   annotationProcessor(libs.immutables.value.processor)
 
@@ -56,12 +64,29 @@ dependencies {
 
   testFixturesApi(platform(libs.junit.bom))
   testFixturesApi(libs.bundles.junit.testing)
+  testFixturesImplementation(project(":nessie-client"))
+  testFixturesImplementation(project(":nessie-cli-grammar"))
+  testFixturesImplementation(libs.jline)
 
   testImplementation(project(":nessie-jaxrs-testextension"))
 
   testImplementation(project(":nessie-versioned-storage-inmemory-tests"))
 
   testCompileOnly(libs.immutables.value.annotations)
+
+  intTestImplementation(project(":nessie-object-storage-mock"))
+  intTestImplementation(project(":nessie-catalog-format-iceberg"))
+  intTestImplementation(project(":nessie-catalog-format-iceberg-fixturegen"))
+  intTestImplementation(project(":nessie-catalog-files-api"))
+  intTestImplementation(project(":nessie-catalog-files-impl"))
+  intTestImplementation(libs.nessie.runner.common)
+  intTestImplementation(platform(libs.awssdk.bom))
+  intTestImplementation("software.amazon.awssdk:s3")
+  intTestImplementation("software.amazon.awssdk:apache-client") {
+    exclude("commons-logging", "commons-logging")
+  }
+
+  nessieQuarkusServer(project(":nessie-quarkus", "quarkusRunner"))
 }
 
 tasks.withType<ProcessResources>().configureEach {
@@ -81,3 +106,16 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
 if (Os.isFamily(Os.FAMILY_MAC) && System.getenv("CI") != null) {
   tasks.named<Test>("intTest").configure { this.enabled = false }
 }
+
+tasks.named<Test>("intTest").configure {
+  // Spark keeps a lot of stuff around in the JVM, breaking tests against different Iceberg
+  // catalogs, so give every test class its own JVM
+  forkEvery = 1
+  inputs.files(nessieQuarkusServer)
+  val execJarProvider =
+    configurations.named("nessieQuarkusServer").map { c ->
+      val file = c.fileCollection(*c.dependencies.toTypedArray()).files.first()
+      listOf("-Dnessie.exec-jar=${file.absolutePath}")
+    }
+  jvmArgumentProviders.add(CommandLineArgumentProvider { execJarProvider.get() })
+}