Spec: Add named storage configurations to the management API

api/management-model/build.gradle.kts

@@ -67,6 +67,11 @@ openApiGenerate {
   configOptions.put("generateBuilders", "true")
   configOptions.put("generateConstructorWithAllArgs", "true")
   configOptions.put("hideGenerationTimestamp", "true")
+  // Disable the JsonNullable wrapper for `nullable: true` properties: we only use `nullable:
+  // true` (on Catalog/UpdateCatalogRequest#storageConfigInfos) to make the Java codegen emit a
+  // `null` default for an omitted array instead of its usual `new ArrayList<>()`, and want the
+  // plain `List<StorageConfigInfo>` type it would otherwise generate, not `JsonNullable<...>`.
+  configOptions.put("openApiNullable", "false")
   additionalProperties.put("apiNamePrefix", "Polaris")
   additionalProperties.put("apiNameSuffix", "Api")
   additionalProperties.put("metricsPrefix", "polaris")

extensions/semantic-models/src/test/java/org/apache/polaris/service/catalog/semanticmodel/AbstractSemanticModelCatalogHandlerTest.java

@@ -136,7 +136,8 @@ private void createCatalogNamespaceAndTable() {
             1,
             FileStorageConfigInfo.builder()
                 .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE)
-                .build());
+                .build(),
+            null);
     try (Response response =
         services
             .catalogsApi()

integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java

@@ -220,7 +220,8 @@ public void updateCatalog(Catalog catalog, Map<String, String> catalogProps) {
                     new UpdateCatalogRequest(
                         catalog.getEntityVersion(),
                         catalogProps,
-                        catalog.getStorageConfigInfo())))) {
+                        catalog.getStorageConfigInfo(),
+                        null)))) {
       assertThat(response.getStatus()).isEqualTo(OK.getStatusCode());
     }
   }

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java

@@ -450,7 +450,10 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() {
     // default-base-location.
     UpdateCatalogRequest updateRequest =
         new UpdateCatalogRequest(
-            fetchedCatalog.getEntityVersion(), Map.of("foo", "bar"), null /* storageConfigIno */);
+            fetchedCatalog.getEntityVersion(),
+            Map.of("foo", "bar"),
+            null /* storageConfigIno */,
+            null);
 
     // Successfully update
     Catalog updatedCatalog;
@@ -587,7 +590,8 @@ public void testCreateAndUpdateAzureCatalog() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"),
-            modifiedStorageConfig);
+            modifiedStorageConfig,
+            null);
     try (Response response =
         managementApi.request("v1/catalogs/" + catalogName).put(Entity.json(badUpdateRequest))) {
       assertThat(response)
@@ -604,7 +608,8 @@ public void testCreateAndUpdateAzureCatalog() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"),
-            storageConfig);
+            storageConfig,
+            null);
 
     // 200 successful update
     try (Response response =
@@ -680,7 +685,8 @@ public void testCreateListUpdateAndDeleteCatalog() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "s3://newbucket/"),
-            invalidModifiedStorageConfig);
+            invalidModifiedStorageConfig,
+            null);
     try (Response response =
         managementApi
             .request("v1/catalogs/{cat}", Map.of("cat", catalogName))
@@ -707,7 +713,8 @@ public void testCreateListUpdateAndDeleteCatalog() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "s3://newbucket/"),
-            validModifiedStorageConfig);
+            validModifiedStorageConfig,
+            null);
 
     // 200 successful update
     try (Response response =
@@ -795,7 +802,8 @@ public void testUpdateCatalogChangeAwsAccountIdRejected() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "s3://bucket1/"),
-            differentAccountConfig);
+            differentAccountConfig,
+            null);
 
     try (Response response =
         managementApi
@@ -847,7 +855,8 @@ public void testUpdateCatalogChangeRoleWithinSameAccountAllowed() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "s3://bucket1/"),
-            updatedConfig);
+            updatedConfig,
+            null);
 
     try (Response response =
         managementApi
@@ -898,7 +907,8 @@ public void testUpdateCatalogChangeExternalIdRejected() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "s3://bucket1/"),
-            configWithDifferentExternalId);
+            configWithDifferentExternalId,
+            null);
 
     try (Response response =
         managementApi

runtime/service/src/cloudTest/java/org/apache/polaris/service/it/GcpCatalogFederationIntegrationIT.java

@@ -193,7 +193,8 @@ private void createCatalog() {
                 .setGcsServiceAccount(SERVICE_ACCOUNT)
                 .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS)
                 .setAllowedLocations(List.of(BASE_LOCATION))
-                .build());
+                .build(),
+            null);
 
     managementApi.createCatalog(catalog);
   }

runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java

@@ -218,7 +218,8 @@ public void testUpdateCatalogWithDisallowedStorageConfig() {
         new UpdateCatalogRequest(
             fetchedCatalog.getEntityVersion(),
             Map.of("default-base-location", "file:///tmp/path/to/data/"),
-            fileStorage);
+            fileStorage,
+            null);
 
     // failure to update
     assertThatThrownBy(
@@ -241,7 +242,8 @@ public void testUpdateCatalogWithDisallowedStorageConfig() {
                 .setAllowedLocations(List.of("s3://bucket/path/to/data"))
                 .setRoleArn("arn:aws:iam::123456789012:role/my-role")
                 .setEndpoint("http://example.com")
-                .build());
+                .build(),
+            null);
     assertThatThrownBy(
             () ->
                 services
@@ -525,7 +527,8 @@ public void testUpdateCatalogChangeAwsAccountIdBlockedByDefault() {
             AwsStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.S3)
                 .setAllowedLocations(List.of("s3://bucket/path/to/data"))
                 .setRoleArn("arn:aws:iam::999999999999:role/other-role")
-                .build());
+                .build(),
+            null);
     assertThatThrownBy(
             () ->
                 services
@@ -581,7 +584,8 @@ public void testUpdateCatalogChangesWithinSameAccountAllowed() {
             AwsStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.S3)
                 .setAllowedLocations(List.of("s3://bucket/path/to/data"))
                 .setRoleArn("arn:aws:iam::123456789012:role/other-role")
-                .build());
+                .build(),
+            null);
     try (Response response =
         services
             .catalogsApi()
@@ -635,7 +639,8 @@ public void testUpdateCatalogChangeExternalIdBlockedByDefault() {
                 .setAllowedLocations(List.of("s3://bucket/path/to/data"))
                 .setRoleArn("arn:aws:iam::123456789012:role/my-role")
                 .setExternalId("different-external-id")
-                .build());
+                .build(),
+            null);
     assertThatThrownBy(
             () ->
                 services
@@ -706,7 +711,8 @@ public void testUpdateCatalogStorageConfigChangesAllowedWithFeatureFlag() {
                 .setAllowedLocations(List.of("s3://bucket/path/to/data"))
                 .setRoleArn("arn:aws:iam::999999999999:role/other-role")
                 .setExternalId("different-external-id")
-                .build());
+                .build(),
+            null);
     try (Response response =
         flagEnabledServices
             .catalogsApi()
@@ -901,7 +907,8 @@ public void testDisallowedMechanismIsRejectedOnUpdateToo() {
         new UpdateCatalogRequest(
             fetched.getEntityVersion(),
             Map.of("default-base-location", "s3://second-bucket/base/empty-stay"),
-            secondMechanismConfig().build());
+            secondMechanismConfig().build(),
+            null);
     assertThatThrownBy(
             () ->
                 stsOnlyUnrestricted
@@ -941,7 +948,8 @@ public void theLiteralDefaultIsRefusedLikeAnyUnknownValueAtCreateAndUpdate() {
         new UpdateCatalogRequest(
             fetched.getEntityVersion(),
             Map.of("default-base-location", "s3://second-bucket/base/stays-empty"),
-            namesDefault);
+            namesDefault,
+            null);
     assertThatThrownBy(
             () ->
                 svc.catalogsApi()
@@ -971,7 +979,8 @@ public void anEmptyMechanismCanBeUpdatedToAnExplicitSts() {
         new UpdateCatalogRequest(
             fetched.getEntityVersion(),
             Map.of("default-base-location", "s3://second-bucket/base/empty-then-sts"),
-            explicitSts);
+            explicitSts,
+            null);
     try (Response response =
         svc.catalogsApi()
             .updateCatalog("empty-then-sts", toSts, svc.realmContext(), svc.securityContext())) {
@@ -1017,7 +1026,8 @@ public void anExplicitStsCanBeUpdatedBackToTheEmptyField() {
         new UpdateCatalogRequest(
             fetched.getEntityVersion(),
             Map.of("default-base-location", "s3://second-bucket/base/sts-then-empty"),
-            backToDefault);
+            backToDefault,
+            null);
     try (Response response =
         svc.catalogsApi()
             .updateCatalog("sts-then-empty", toEmpty, svc.realmContext(), svc.securityContext())) {
@@ -1076,7 +1086,8 @@ public void changingTheMechanismIsAcceptedAndValidatedByTheNewMechanism() {
                 .setCredentialVendingMechanism(TEST_MECHANISM)
                 .setAllowedLocations(List.of("s3://second-bucket/base/"))
                 .setEndpoint("https://s3.example.test")
-                .build());
+                .build(),
+            null);
     try (Response response =
         svc.catalogsApi()
             .updateCatalog(
@@ -1117,7 +1128,8 @@ public void emptyMechanismCatalogEndpointStaysMutable() {
                 .setAllowedLocations(List.of("s3://second-bucket/base/"))
                 .setEndpoint("https://s3.other.example.com:1234")
                 .setPathStyleAccess(true)
-                .build());
+                .build(),
+            null);
     try (Response response =
         svc.catalogsApi()
             .updateCatalog(
@@ -1157,7 +1169,8 @@ public void updatingACatalogToAnAllowlistedButUninstalledMechanismIsRefused() {
             AwsStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.S3)
                 .setCredentialVendingMechanism("UNINSTALLED_MECHANISM")
                 .setAllowedLocations(List.of("s3://second-bucket/base/"))
-                .build());
+                .build(),
+            null);
     assertThatThrownBy(
             () ->
                 svc.catalogsApi()
@@ -1223,7 +1236,8 @@ public void theMechanismValidatesTheConfigAtCreateAndAtUpdate() {
         new UpdateCatalogRequest(
             fetched.getEntityVersion(),
             Map.of("default-base-location", "s3://second-bucket/base/needs-endpoint"),
-            base.setEndpoint(null).build());
+            base.setEndpoint(null).build(),
+            null);
     assertThatThrownBy(
             () ->
                 svc.catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java

@@ -94,7 +94,8 @@ private Response createCatalog(
             System.currentTimeMillis(),
             System.currentTimeMillis(),
             1,
-            config);
+            config,
+            null);
     return services
         .catalogsApi()
         .createCatalog(

runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisS3InteroperabilityTest.java

@@ -106,7 +106,8 @@ private PolarisCatalog createCatalog(String catalogName, String scheme) {
             1725487592064L,
             1725487592064L,
             1,
-            config);
+            config,
+            null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/admin/S3CredentialVendingMechanismRoutesTest.java

@@ -341,7 +341,8 @@ void theRealmKillSwitchRefusesTheCatalogWithNotEnabled(boolean skipSubscoping) {
         new UpdateCatalogRequest(
             fetched.getEntityVersion(),
             Map.of("default-base-location", "s3://bucket/base/mechkill"),
-            fetched.getStorageConfigInfo());
+            fetched.getStorageConfigInfo(),
+            null);
     assertThatThrownBy(
             () ->
                 svc.catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/CommitTransactionMetadataCleanupTest.java

@@ -163,7 +163,7 @@ private void createCatalogAndNamespace(TestServices services, String catalogLoca
             .build();
     Catalog catalogObject =
         new Catalog(
-            Catalog.TypeEnum.INTERNAL, catalog, propertiesBuilder.build(), 0L, 0L, 1, config);
+            Catalog.TypeEnum.INTERNAL, catalog, propertiesBuilder.build(), 0L, 0L, 1, config, null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/CommitTransactionTest.java

@@ -217,7 +217,7 @@ private void createCatalogAndNamespace(
             .build();
     Catalog catalogObject =
         new Catalog(
-            Catalog.TypeEnum.INTERNAL, catalog, propertiesBuilder.build(), 0L, 0L, 1, config);
+            Catalog.TypeEnum.INTERNAL, catalog, propertiesBuilder.build(), 0L, 0L, 1, config, null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/EntityIdempotencyUpdateTableTest.java

@@ -207,7 +207,8 @@ private static void createCatalog(TestServices services, String catalogLocation)
             1725487592064L,
             1725487592064L,
             1,
-            storageConfig);
+            storageConfig,
+            null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergAllowedLocationTest.java

@@ -852,7 +852,8 @@ private void createCatalog(
             1725487592064L,
             1725487592064L,
             1,
-            config);
+            config,
+            null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergOverlappingTableTest.java

@@ -186,7 +186,8 @@ private void createCatalogAndNamespace(
             1725487592064L,
             1725487592064L,
             1,
-            config);
+            config,
+            null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergTableLocationTest.java

@@ -114,7 +114,8 @@ private void createCatalogAndNamespace(
             1725487592064L,
             1725487592064L,
             1,
-            config);
+            config,
+            null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/idempotency/EntityIdempotencyCreateTableTest.java

@@ -188,7 +188,8 @@ private static void createCatalog(TestServices services, String catalogLocation)
             1725487592064L,
             1725487592064L,
             1,
-            storageConfig);
+            storageConfig,
+            null);
     try (Response response =
         services
             .catalogsApi()

runtime/service/src/test/java/org/apache/polaris/service/storage/S3CredentialVendingMechanismCdiTest.java

@@ -544,7 +544,8 @@ private static UpdateCatalogRequest updateToUninstalledMechanismRequest(
             .setCredentialVendingMechanism(UNINSTALLED_MECHANISM)
             .setRoleArn("arn:aws:iam::123456789012:role/r")
             .setAllowedLocations(List.of("s3://bucket/base/" + name + "/"))
-            .build());
+            .build(),
+        null);
   }
 
   private static void createTable(CatalogApi catalogApi, String catalog, String ns, String table) {

runtime/service/src/test/java/org/apache/polaris/service/storage/S3CredentialVendingMechanismThirdMechanismCdiTest.java

@@ -188,7 +188,8 @@ void anInstalledThirdMechanismVendsWhenAllowlistedAndIsRefusedWithoutDispatchWhe
                       List.of(
                           "s3://bucket/base/" + catalog + "/",
                           "s3://bucket/base/" + catalog + "-extra/"))
-                  .build());
+                  .build(),
+              null);
       try (Response r =
           managementApi
               .request("v1/catalogs/{name}", Map.of("name", catalog))