From f8e9b6f6582d923a478a59cf56168d106e375b1b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 17 Dec 2025 00:32:59 -0800 Subject: [PATCH 1/6] CORE: Allow table level override for scan planning --- .../apache/iceberg/rest/RESTSessionCatalog.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 61e25d3d4fc6..f2f29863acf1 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -505,9 +505,16 @@ public Table loadTable(SessionContext context, TableIdentifier identifier) { } private RESTTable restTableForScanPlanning( - TableOperations ops, TableIdentifier finalIdentifier, RESTClient restClient) { + TableOperations ops, + TableIdentifier finalIdentifier, + RESTClient restClient, + Map tableConf) { // server supports remote planning endpoint and server / client wants to do server side planning - if (endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN) && restScanPlanningEnabled) { + boolean shouldDoServerSidePlanning = + PropertyUtil.propertyAsBoolean( + tableConf, RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, false); + if (endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN) + && (restScanPlanningEnabled || shouldDoServerSidePlanning)) { return new RESTTable( ops, fullTableName(finalIdentifier), @@ -589,7 +596,7 @@ public Table registerTable( trackFileIO(ops); - RESTTable restTable = restTableForScanPlanning(ops, ident, tableClient); + RESTTable restTable = restTableForScanPlanning(ops, ident, tableClient, tableConf); if (restTable != null) { return restTable; } @@ -858,7 +865,7 @@ public Table create() { trackFileIO(ops); - RESTTable restTable = restTableForScanPlanning(ops, ident, tableClient); + RESTTable restTable = restTableForScanPlanning(ops, ident, tableClient, tableConf); if (restTable != null) { return restTable; } From f65a613561d2dad3dac34a96682d6c26eb7332e2 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Singh Date: Wed, 17 Dec 2025 17:08:27 +0000 Subject: [PATCH 2/6] Add test and spec changes --- .../iceberg/rest/TestRESTScanPlanning.java | 81 +++++++++++++++++++ open-api/rest-catalog-open-api.py | 1 + open-api/rest-catalog-open-api.yaml | 1 + 3 files changed, 83 insertions(+) diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java index f84197b0f16e..ebf4bf5a2ce3 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java @@ -72,6 +72,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.rest.responses.ConfigResponse; import org.apache.iceberg.rest.responses.ErrorResponse; +import org.apache.iceberg.rest.responses.LoadTableResponse; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.servlet.ServletContextHandler; @@ -1020,4 +1021,84 @@ public void serverSupportsPlanningButNotCancellation() throws IOException { // Verify no exception was thrown - cancelPlan returns false when endpoint not supported assertThat(cancelled).isFalse(); } + + @ParameterizedTest + @EnumSource(PlanningMode.class) + public void tableLevelScanPlanningOverride( + Function planMode) + throws IOException { + // Test REST_SCAN_PLANNING_ENABLED in LoadTableResponse.config() overrides catalog setting + configurePlanningBehavior(planMode); + + // Catalog that adds scan planning config to LoadTableResponse (table-level override) + CatalogWithAdapter catalogWithAdapter = + catalogWithTableLevelConfig(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "true"); + + RESTTable table = restTableFor(catalogWithAdapter.catalog, "table_override_test"); + setParserContext(table); + + assertThat(table.newScan().planFiles()).hasSize(1); + + catalogWithAdapter.catalog.close(); + } + + private CatalogWithAdapter catalogWithTableLevelConfig(String configKey, String configValue) { + RESTCatalogAdapter adapter = + Mockito.spy( + new RESTCatalogAdapter(backendCatalog) { + @Override + public T execute( + HTTPRequest request, + Class responseType, + Consumer errorHandler, + Consumer> responseHeaders) { + if (ResourcePaths.config().equals(request.path())) { + return castResponse( + responseType, + ConfigResponse.builder() + .withEndpoints( + Arrays.stream(Route.values()) + .map(r -> Endpoint.create(r.method().name(), r.resourcePath())) + .collect(Collectors.toList())) + .withOverride(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "false") + .build()); + } + + Object body = roundTripSerialize(request.body(), "request"); + T response = + super.execute( + ImmutableHTTPRequest.builder().from(request).body(body).build(), + responseType, + errorHandler, + responseHeaders); + + // Add config to ALL LoadTableResponse + if (response instanceof LoadTableResponse) { + LoadTableResponse load = (LoadTableResponse) response; + return roundTripSerialize( + castResponse( + responseType, + LoadTableResponse.builder() + .withTableMetadata(load.tableMetadata()) + .addConfig(configKey, configValue) + .addAllCredentials(load.credentials()) + .build()), + "response"); + } + + return roundTripSerialize(response, "response"); + } + }); + + RESTCatalog catalog = + new RESTCatalog(SessionCatalog.SessionContext.createEmpty(), (config) -> adapter); + catalog.initialize( + "test", + ImmutableMap.of( + RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, + "false", + CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO")); + return new CatalogWithAdapter(catalog, adapter); + } } diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 1079d277d3c7..3a62fbc12b6a 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1291,6 +1291,7 @@ class LoadTableResult(BaseModel): ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + - `rest-scan-planning-enabled`: The client should use remote scan planning. ## AWS Configurations diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index d322b0c7c7c0..0993b8936641 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3374,6 +3374,7 @@ components: ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + - `rest-scan-planning-enabled`: The client should use remote scan planning. ## AWS Configurations From 9f2923ed7d8daed4a69a387c82c91784dae5f12e Mon Sep 17 00:00:00 2001 From: Prashant Kumar Singh Date: Thu, 18 Dec 2025 08:46:01 +0000 Subject: [PATCH 3/6] Add rest planning mode --- .../iceberg/rest/RESTCatalogProperties.java | 45 +++++++++- .../iceberg/rest/RESTSessionCatalog.java | 84 ++++++++++++++----- .../iceberg/rest/TestRESTScanPlanning.java | 84 +++++++++++++++++-- open-api/rest-catalog-open-api.py | 5 +- open-api/rest-catalog-open-api.yaml | 5 +- 5 files changed, 187 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java b/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java index 79617b2982ff..63cfc779ed51 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java @@ -37,12 +37,51 @@ private RESTCatalogProperties() {} public static final String NAMESPACE_SEPARATOR = "namespace-separator"; - // Enable planning on the REST server side - public static final String REST_SCAN_PLANNING_ENABLED = "rest-scan-planning-enabled"; - public static final boolean REST_SCAN_PLANNING_ENABLED_DEFAULT = false; + // Configure scan planning mode on the REST server side + public static final String REST_SCAN_PLANNING_MODE = "rest-scan-planning-mode"; + public static final String REST_SCAN_PLANNING_MODE_DEFAULT = ScanPlanningMode.NONE.modeName(); public enum SnapshotMode { ALL, REFS } + + /** + * Enum to represent the scan planning mode for REST catalog. + * + *
    + *
  • NONE - Server-side scan planning is not supported/allowed + *
  • OPTIONAL - Client can choose between client-side or server-side planning + *
  • REQUIRED - Server-side planning is required (client-side planning not allowed) + *
+ */ + public enum ScanPlanningMode { + NONE("none"), + OPTIONAL("optional"), + REQUIRED("required"); + + private final String modeName; + + ScanPlanningMode(String modeName) { + this.modeName = modeName; + } + + public String modeName() { + return modeName; + } + + public static ScanPlanningMode fromString(String mode) { + if (mode == null) { + return NONE; + } + for (ScanPlanningMode planningMode : values()) { + if (planningMode.modeName.equalsIgnoreCase(mode)) { + return planningMode; + } + } + throw new IllegalArgumentException( + String.format( + "Invalid scan planning mode: %s. Valid values are: none, optional, required", mode)); + } + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index f2f29863acf1..c234227d982c 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -159,7 +159,7 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog private MetricsReporter reporter = null; private boolean reportingViaRestEnabled; private Integer pageSize = null; - private boolean restScanPlanningEnabled; + private RESTCatalogProperties.ScanPlanningMode restScanPlanningMode; private CloseableGroup closeables = null; private Set endpoints; private Supplier> mutationHeaders = Map::of; @@ -272,11 +272,13 @@ public void initialize(String name, Map unresolved) { RESTCatalogProperties.NAMESPACE_SEPARATOR, RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); - this.restScanPlanningEnabled = - PropertyUtil.propertyAsBoolean( + String scanPlanningConfig = + PropertyUtil.propertyAsString( mergedProps, - RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, - RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED_DEFAULT); + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, + RESTCatalogProperties.REST_SCAN_PLANNING_MODE_DEFAULT); + this.restScanPlanningMode = + RESTCatalogProperties.ScanPlanningMode.fromString(scanPlanningConfig); super.initialize(name, mergedProps); } @@ -486,7 +488,7 @@ public Table loadTable(SessionContext context, TableIdentifier identifier) { // RestTable should only be returned for non-metadata tables, because client would // not have access to metadata files for example manifests, since all it needs is catalog. if (metadataType == null) { - RESTTable restTable = restTableForScanPlanning(ops, finalIdentifier, tableClient); + RESTTable restTable = restTableForScanPlanning(ops, finalIdentifier, tableClient, tableConf); if (restTable != null) { return restTable; } @@ -509,23 +511,59 @@ private RESTTable restTableForScanPlanning( TableIdentifier finalIdentifier, RESTClient restClient, Map tableConf) { - // server supports remote planning endpoint and server / client wants to do server side planning - boolean shouldDoServerSidePlanning = - PropertyUtil.propertyAsBoolean( - tableConf, RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, false); - if (endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN) - && (restScanPlanningEnabled || shouldDoServerSidePlanning)) { - return new RESTTable( - ops, - fullTableName(finalIdentifier), - metricsReporter(paths.metrics(finalIdentifier), restClient), - restClient, - Map::of, - finalIdentifier, - paths, - endpoints); - } - return null; + // Determine the effective scan planning mode (table-level config overrides catalog config) + RESTCatalogProperties.ScanPlanningMode effectiveMode = restScanPlanningMode; + + // Check for table-level override + String tableModeConfig = tableConf.get(RESTCatalogProperties.REST_SCAN_PLANNING_MODE); + if (tableModeConfig != null) { + effectiveMode = RESTCatalogProperties.ScanPlanningMode.fromString(tableModeConfig); + } + + boolean serverSupportsPlanning = endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN); + + // Handle the three modes + switch (effectiveMode) { + case NONE: + // Server-side planning not allowed, use client-side + return null; + + case OPTIONAL: + // Use server-side planning if server supports it + if (serverSupportsPlanning) { + return new RESTTable( + ops, + fullTableName(finalIdentifier), + metricsReporter(paths.metrics(finalIdentifier), restClient), + restClient, + Map::of, + finalIdentifier, + paths, + endpoints); + } + return null; + + case REQUIRED: + // Server-side planning is required + if (!serverSupportsPlanning) { + throw new UnsupportedOperationException( + String.format( + "Server-side scan planning is required but server does not support endpoint: %s", + Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN)); + } + return new RESTTable( + ops, + fullTableName(finalIdentifier), + metricsReporter(paths.metrics(finalIdentifier), restClient), + restClient, + Map::of, + finalIdentifier, + paths, + endpoints); + + default: + throw new IllegalStateException("Unknown scan planning mode: " + effectiveMode); + } } private void trackFileIO(RESTTableOperations ops) { diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java index ebf4bf5a2ce3..58730ed55c45 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java @@ -124,7 +124,7 @@ public T execute( .collect(Collectors.toList())) .withOverrides( ImmutableMap.of( - RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "true")) + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "optional")) .build()); } Object body = roundTripSerialize(request.body(), "request"); @@ -923,8 +923,8 @@ public T execute( ImmutableMap.of( CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO", - RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, - "true")); + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, + "optional")); return new CatalogWithAdapter(catalog, adapter); } @@ -946,6 +946,74 @@ public void serverDoesNotSupportPlanningEndpoint() throws IOException { .isEqualTo(FILE_A.location()); } + @Test + public void scanPlanningModeRequired() throws IOException { + // Test REQUIRED mode - should throw exception when server doesn't support planning + // Create a catalog that doesn't advertise scan planning endpoints + RESTCatalogAdapter adapter = + Mockito.spy( + new RESTCatalogAdapter(backendCatalog) { + @Override + public T execute( + HTTPRequest request, + Class responseType, + Consumer errorHandler, + Consumer> responseHeaders) { + if (ResourcePaths.config().equals(request.path())) { + // Return config without scan planning endpoints + return castResponse( + responseType, + ConfigResponse.builder().withEndpoints(baseCatalogEndpoints()).build()); + } + return super.execute(request, responseType, errorHandler, responseHeaders); + } + }); + + RESTCatalog catalog = + new RESTCatalog(SessionCatalog.SessionContext.createEmpty(), (config) -> adapter); + catalog.initialize( + "test-required", + ImmutableMap.of( + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, + "required", + CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO")); + + TableIdentifier tableId = TableIdentifier.of(NS, "required_mode_test"); + catalog.createNamespace(NS); + + // Should throw UnsupportedOperationException when trying to create/load the table + // because REQUIRED mode needs server-side planning but server doesn't support it + assertThatThrownBy(() -> catalog.createTable(tableId, SCHEMA)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Server-side scan planning is required"); + + catalog.close(); + } + + @Test + public void scanPlanningModeNone() throws IOException { + // Test NONE mode - should use client-side planning even if server supports it + CatalogWithAdapter catalogWithAdapter = + catalogWithTableLevelConfig(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "none"); + + Table table = createTableWithScanPlanning(catalogWithAdapter.catalog, "none_mode_test"); + table.newAppend().appendFile(FILE_A).commit(); + + // Should not be a RESTTable since NONE mode disables server-side planning + assertThat(table).isNotInstanceOf(RESTTable.class); + + // Should still work with client-side planning + assertThat(table.newScan().planFiles()) + .hasSize(1) + .first() + .extracting(ContentScanTask::file) + .extracting(ContentFile::location) + .isEqualTo(FILE_A.location()); + + catalogWithAdapter.catalog.close(); + } + @Test public void serverSupportsPlanningSyncOnlyNotAsync() { // Server supports submit (sync) but not fetch (async polling) @@ -1027,12 +1095,12 @@ public void serverSupportsPlanningButNotCancellation() throws IOException { public void tableLevelScanPlanningOverride( Function planMode) throws IOException { - // Test REST_SCAN_PLANNING_ENABLED in LoadTableResponse.config() overrides catalog setting + // Test REST_SCAN_PLANNING_MODE in LoadTableResponse.config() overrides catalog setting configurePlanningBehavior(planMode); // Catalog that adds scan planning config to LoadTableResponse (table-level override) CatalogWithAdapter catalogWithAdapter = - catalogWithTableLevelConfig(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "true"); + catalogWithTableLevelConfig(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "optional"); RESTTable table = restTableFor(catalogWithAdapter.catalog, "table_override_test"); setParserContext(table); @@ -1060,7 +1128,7 @@ public T execute( Arrays.stream(Route.values()) .map(r -> Endpoint.create(r.method().name(), r.resourcePath())) .collect(Collectors.toList())) - .withOverride(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "false") + .withOverride(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "none") .build()); } @@ -1095,8 +1163,8 @@ public T execute( catalog.initialize( "test", ImmutableMap.of( - RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, - "false", + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, + "none", CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); return new CatalogWithAdapter(catalog, adapter); diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 3a62fbc12b6a..387b8fe4b00a 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1291,7 +1291,10 @@ class LoadTableResult(BaseModel): ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - - `rest-scan-planning-enabled`: The client should use remote scan planning. + - `rest-scan-planning-mode`: Controls the scan planning behavior. Valid values are: + - `none` (default): Server-side scan planning is not supported or allowed. Client must use client-side planning. + - `optional`: Client can choose between client-side or server-side planning based on its capabilities and preferences. + - `required`: Server-side planning is required. Client must not use client-side planning. ## AWS Configurations diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 0993b8936641..411b0c75baef 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3374,7 +3374,10 @@ components: ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - - `rest-scan-planning-enabled`: The client should use remote scan planning. + - `rest-scan-planning-mode`: Controls the scan planning behavior. Valid values are: + - `none` (default): Server-side scan planning is not supported or allowed. Client must use client-side planning. + - `optional`: Client can choose between client-side or server-side planning based on its capabilities and preferences. + - `required`: Server-side planning is required. Client must not use client-side planning. ## AWS Configurations From e5c378e5a117ce6b938f4b10d9713857af9af9a5 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 22 Dec 2025 09:07:17 -0800 Subject: [PATCH 4/6] adjust build --- .../apache/iceberg/spark/extensions/TestRemoteScanPlanning.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java index 14e6c358898c..d9561c6d9695 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java @@ -42,7 +42,7 @@ protected static Object[][] parameters() { .put(CatalogProperties.URI, restCatalog.properties().get(CatalogProperties.URI)) // this flag is typically only set by the server, but we set it from the client for // testing - .put(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "true") + .put(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "required") .build(), SparkCatalogConfig.REST.catalogName() + ".default.binary_table" } From ef9e24f25865f94d4807e546cd8a242b5bd2fa78 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Singh Date: Tue, 23 Dec 2025 01:28:38 +0000 Subject: [PATCH 5/6] Address review feedback --- .../iceberg/rest/RESTCatalogProperties.java | 110 ++++++++- .../iceberg/rest/RESTSessionCatalog.java | 91 ++++---- .../iceberg/rest/ScanPlanningNegotiator.java | 210 ++++++++++++++++++ .../iceberg/rest/TestRESTScanPlanning.java | 25 ++- open-api/rest-catalog-open-api.py | 28 ++- open-api/rest-catalog-open-api.yaml | 28 ++- .../extensions/TestRemoteScanPlanning.java | 2 +- 7 files changed, 417 insertions(+), 77 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java b/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java index 63cfc779ed51..13f56ce018de 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java @@ -37,9 +37,16 @@ private RESTCatalogProperties() {} public static final String NAMESPACE_SEPARATOR = "namespace-separator"; - // Configure scan planning mode on the REST server side + // Configure scan planning mode on the REST catalog/server side public static final String REST_SCAN_PLANNING_MODE = "rest-scan-planning-mode"; - public static final String REST_SCAN_PLANNING_MODE_DEFAULT = ScanPlanningMode.NONE.modeName(); + public static final String REST_SCAN_PLANNING_MODE_DEFAULT = + ScanPlanningMode.CLIENT_PREFERRED.modeName(); + + // Configure client-side scan planning preference + public static final String CLIENT_SCAN_PLANNING_PREFERENCE = + "rest-scan-planning-client-preference"; + public static final String CLIENT_SCAN_PLANNING_PREFERENCE_DEFAULT = + ClientScanPlanningPreference.NONE.preferenceName(); public enum SnapshotMode { ALL, @@ -47,18 +54,32 @@ public enum SnapshotMode { } /** - * Enum to represent the scan planning mode for REST catalog. + * Enum to represent the scan planning mode for REST catalog (server-side configuration). + * + *

This defines what the catalog/server requires or prefers for scan planning using RFC 2119 + * semantics (MUST, SHOULD, MAY): * *

    - *
  • NONE - Server-side scan planning is not supported/allowed - *
  • OPTIONAL - Client can choose between client-side or server-side planning - *
  • REQUIRED - Server-side planning is required (client-side planning not allowed) + *
  • CLIENT_ONLY - Client MUST use client-side planning. Server MUST NOT provide server-side + * planning endpoints. + *
  • CLIENT_PREFERRED (default) - Client SHOULD use client-side planning. Server MAY provide + * server-side planning if explicitly requested. + *
  • CATALOG_PREFERRED - Client SHOULD use server-side planning when available. Server MAY + * fall back to client-side if server doesn't support planning or client explicitly requests + * it. + *
  • CATALOG_ONLY - Client MUST use server-side planning. Server MUST provide server-side + * planning endpoints. Client MUST NOT use client-side planning. *
+ * + *

For the complete decision matrix showing how this negotiates with {@link + * ClientScanPlanningPreference}, see {@link + * org.apache.iceberg.rest.ScanPlanningNegotiator#negotiate}. */ public enum ScanPlanningMode { - NONE("none"), - OPTIONAL("optional"), - REQUIRED("required"); + CLIENT_ONLY("client-only"), + CLIENT_PREFERRED("client-preferred"), + CATALOG_PREFERRED("catalog-preferred"), + CATALOG_ONLY("catalog-only"); private final String modeName; @@ -70,9 +91,25 @@ public String modeName() { return modeName; } + public boolean isClientAllowed() { + return this == CLIENT_ONLY || this == CLIENT_PREFERRED; + } + + public boolean isCatalogAllowed() { + return this == CATALOG_PREFERRED || this == CATALOG_ONLY; + } + + public boolean requiresClient() { + return this == CLIENT_ONLY; + } + + public boolean requiresCatalog() { + return this == CATALOG_ONLY; + } + public static ScanPlanningMode fromString(String mode) { if (mode == null) { - return NONE; + return CLIENT_PREFERRED; } for (ScanPlanningMode planningMode : values()) { if (planningMode.modeName.equalsIgnoreCase(mode)) { @@ -81,7 +118,58 @@ public static ScanPlanningMode fromString(String mode) { } throw new IllegalArgumentException( String.format( - "Invalid scan planning mode: %s. Valid values are: none, optional, required", mode)); + "Invalid scan planning mode: %s. Valid values are: client-only, client-preferred, catalog-preferred, catalog-only", + mode)); + } + } + + /** + * Enum to represent the client-side scan planning preference. + * + *

This defines what the client wants to do for scan planning using RFC 2119 semantics (MUST, + * SHOULD): + * + *

    + *
  • NONE (default) - Client SHOULD follow catalog's {@link ScanPlanningMode} recommendation. + * No explicit preference set. This is the cooperative default behavior. + *
  • CLIENT_PLANNING - Client MUST use client-side planning. Throws {@link + * IllegalStateException} if catalog mode is {@link ScanPlanningMode#CATALOG_ONLY}. + *
  • CATALOG_PLANNING - Client MUST use server-side planning. Throws {@link + * IllegalStateException} if catalog mode is {@link ScanPlanningMode#CLIENT_ONLY}, or {@link + * UnsupportedOperationException} if server doesn't support planning endpoints. + *
+ * + *

For the complete decision matrix showing how this negotiates with {@link ScanPlanningMode}, + * see {@link org.apache.iceberg.rest.ScanPlanningNegotiator#negotiate}. + */ + public enum ClientScanPlanningPreference { + NONE("none"), + CLIENT_PLANNING("client"), + CATALOG_PLANNING("catalog"); + + private final String preferenceName; + + ClientScanPlanningPreference(String preferenceName) { + this.preferenceName = preferenceName; + } + + public String preferenceName() { + return preferenceName; + } + + public static ClientScanPlanningPreference fromString(String pref) { + if (pref == null || pref.trim().isEmpty()) { + return NONE; + } + for (ClientScanPlanningPreference preference : values()) { + if (preference.preferenceName.equalsIgnoreCase(pref)) { + return preference; + } + } + throw new IllegalArgumentException( + String.format( + "Invalid client scan planning preference: %s. Valid values are: none, client, catalog", + pref)); } } } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index c234227d982c..b8cf6e74f932 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -160,6 +160,7 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog private boolean reportingViaRestEnabled; private Integer pageSize = null; private RESTCatalogProperties.ScanPlanningMode restScanPlanningMode; + private RESTCatalogProperties.ClientScanPlanningPreference clientScanPlanningPreference; private CloseableGroup closeables = null; private Set endpoints; private Supplier> mutationHeaders = Map::of; @@ -279,6 +280,15 @@ public void initialize(String name, Map unresolved) { RESTCatalogProperties.REST_SCAN_PLANNING_MODE_DEFAULT); this.restScanPlanningMode = RESTCatalogProperties.ScanPlanningMode.fromString(scanPlanningConfig); + + String clientPreferenceConfig = + PropertyUtil.propertyAsString( + mergedProps, + RESTCatalogProperties.CLIENT_SCAN_PLANNING_PREFERENCE, + RESTCatalogProperties.CLIENT_SCAN_PLANNING_PREFERENCE_DEFAULT); + this.clientScanPlanningPreference = + RESTCatalogProperties.ClientScanPlanningPreference.fromString(clientPreferenceConfig); + super.initialize(name, mergedProps); } @@ -511,59 +521,46 @@ private RESTTable restTableForScanPlanning( TableIdentifier finalIdentifier, RESTClient restClient, Map tableConf) { - // Determine the effective scan planning mode (table-level config overrides catalog config) - RESTCatalogProperties.ScanPlanningMode effectiveMode = restScanPlanningMode; - - // Check for table-level override + // Determine effective catalog mode (table-level config overrides catalog config) + RESTCatalogProperties.ScanPlanningMode effectiveCatalogMode = restScanPlanningMode; String tableModeConfig = tableConf.get(RESTCatalogProperties.REST_SCAN_PLANNING_MODE); if (tableModeConfig != null) { - effectiveMode = RESTCatalogProperties.ScanPlanningMode.fromString(tableModeConfig); + effectiveCatalogMode = RESTCatalogProperties.ScanPlanningMode.fromString(tableModeConfig); } - boolean serverSupportsPlanning = endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN); + // Determine effective client preference (could optionally support table-level override) + RESTCatalogProperties.ClientScanPlanningPreference effectiveClientPref = + clientScanPlanningPreference; + String tableClientPrefConfig = + tableConf.get(RESTCatalogProperties.CLIENT_SCAN_PLANNING_PREFERENCE); + if (tableClientPrefConfig != null) { + effectiveClientPref = + RESTCatalogProperties.ClientScanPlanningPreference.fromString(tableClientPrefConfig); + } - // Handle the three modes - switch (effectiveMode) { - case NONE: - // Server-side planning not allowed, use client-side - return null; - - case OPTIONAL: - // Use server-side planning if server supports it - if (serverSupportsPlanning) { - return new RESTTable( - ops, - fullTableName(finalIdentifier), - metricsReporter(paths.metrics(finalIdentifier), restClient), - restClient, - Map::of, - finalIdentifier, - paths, - endpoints); - } - return null; - - case REQUIRED: - // Server-side planning is required - if (!serverSupportsPlanning) { - throw new UnsupportedOperationException( - String.format( - "Server-side scan planning is required but server does not support endpoint: %s", - Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN)); - } - return new RESTTable( - ops, - fullTableName(finalIdentifier), - metricsReporter(paths.metrics(finalIdentifier), restClient), - restClient, - Map::of, - finalIdentifier, - paths, - endpoints); + // Check server capabilities + boolean serverSupportsPlanning = endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN); - default: - throw new IllegalStateException("Unknown scan planning mode: " + effectiveMode); - } + // Negotiate scan planning strategy + ScanPlanningNegotiator.PlanningDecision decision = + ScanPlanningNegotiator.negotiate( + effectiveCatalogMode, effectiveClientPref, serverSupportsPlanning, finalIdentifier); + + // Apply the decision + if (decision == ScanPlanningNegotiator.PlanningDecision.USE_CATALOG_PLANNING) { + return new RESTTable( + ops, + fullTableName(finalIdentifier), + metricsReporter(paths.metrics(finalIdentifier), restClient), + restClient, + Map::of, + finalIdentifier, + paths, + endpoints); + } + + // USE_CLIENT_PLANNING + return null; } private void trackFileIO(RESTTableOperations ops) { diff --git a/core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java b/core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java new file mode 100644 index 000000000000..928249701901 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTCatalogProperties.ClientScanPlanningPreference; +import org.apache.iceberg.rest.RESTCatalogProperties.ScanPlanningMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Handles negotiation between client scan planning preferences and catalog scan planning mode + * requirements. + * + *

This class encapsulates the decision logic for determining whether to use client-side or + * server-side scan planning based on: + * + *

    + *
  • Catalog mode (CLIENT_ONLY, CLIENT_PREFERRED, CATALOG_PREFERRED, CATALOG_ONLY) + *
  • Client preference (NONE, CLIENT_PLANNING, CATALOG_PLANNING) + *
  • Server capabilities (supports planning endpoint or not) + *
+ * + *

Decision Matrix

+ * + * The following table shows the final decision based on catalog mode and client preference: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Client PreferenceCLIENT_ONLYCLIENT_PREFERREDCATALOG_PREFERREDCATALOG_ONLY
NONE (follow catalog)Client-sideClient-sideServer-side (fallback to client if unavailable)Server-side (fail if unavailable)
CLIENT_PLANNING (MUST client)Client-sideClient-sideClient-sideIllegalStateException
CATALOG_PLANNING (MUST server)IllegalStateExceptionServer-side (fail if unavailable)Server-side (fail if unavailable)Server-side (fail if unavailable)
+ * + *

Notes: + * + *

    + *
  • "Server-side (fallback to client if unavailable)" - Attempts server-side planning with + * graceful fallback to client-side if server doesn't advertise planning endpoints. + *
  • "Server-side (fail if unavailable)" - Throws {@link UnsupportedOperationException} if + * server doesn't advertise planning endpoints. + *
  • "IllegalStateException" - Thrown when client and catalog MUST requirements conflict. + *
+ */ +public class ScanPlanningNegotiator { + private static final Logger LOG = LoggerFactory.getLogger(ScanPlanningNegotiator.class); + + private ScanPlanningNegotiator() {} + + /** Result of scan planning negotiation. */ + public enum PlanningDecision { + USE_CLIENT_PLANNING, + USE_CATALOG_PLANNING + } + + /** + * Negotiates scan planning strategy between client preference and catalog mode. + * + * @param catalogMode the catalog's scan planning mode (from server config or table config) + * @param clientPref the client's scan planning preference + * @param serverSupportsPlanning whether the server advertises the scan planning endpoint + * @param tableIdentifier the table identifier (for error messages and logging) + * @return the negotiated planning decision + * @throws IllegalStateException if client and catalog requirements are incompatible + * @throws UnsupportedOperationException if required server-side planning but server doesn't + * support it + */ + public static PlanningDecision negotiate( + ScanPlanningMode catalogMode, + ClientScanPlanningPreference clientPref, + boolean serverSupportsPlanning, + TableIdentifier tableIdentifier) { + + switch (clientPref) { + case NONE: + // Client follows catalog's decision + return decideByCatalogMode(catalogMode, serverSupportsPlanning, tableIdentifier); + + case CLIENT_PLANNING: + // Client explicitly wants client-side planning + if (catalogMode == ScanPlanningMode.CATALOG_ONLY) { + throw new IllegalStateException( + String.format( + "Client requires client-side planning but catalog requires server-side planning for table %s. " + + "Either change client preference or update catalog mode.", + tableIdentifier)); + } + LOG.debug( + "Using client-side planning for table {} per client preference (catalog mode: {})", + tableIdentifier, + catalogMode); + return PlanningDecision.USE_CLIENT_PLANNING; + + case CATALOG_PLANNING: + // Client explicitly wants server-side planning + if (catalogMode == ScanPlanningMode.CLIENT_ONLY) { + throw new IllegalStateException( + String.format( + "Client requires server-side planning but catalog requires client-side planning for table %s. " + + "Either change client preference or update catalog mode.", + tableIdentifier)); + } + if (!serverSupportsPlanning) { + throw new UnsupportedOperationException( + String.format( + "Client requires server-side planning for table %s but server does not support planning endpoint. " + + "Either change client preference or upgrade server to support scan planning.", + tableIdentifier)); + } + LOG.debug( + "Using server-side planning for table {} per client preference (catalog mode: {})", + tableIdentifier, + catalogMode); + return PlanningDecision.USE_CATALOG_PLANNING; + + default: + throw new IllegalStateException("Unknown client preference: " + clientPref); + } + } + + private static PlanningDecision decideByCatalogMode( + ScanPlanningMode catalogMode, + boolean serverSupportsPlanning, + TableIdentifier tableIdentifier) { + + switch (catalogMode) { + case CLIENT_ONLY: + LOG.debug( + "Using client-side planning for table {} per catalog requirement (mode: CLIENT_ONLY)", + tableIdentifier); + return PlanningDecision.USE_CLIENT_PLANNING; + + case CLIENT_PREFERRED: + LOG.debug( + "Using client-side planning for table {} per catalog preference (mode: CLIENT_PREFERRED)", + tableIdentifier); + return PlanningDecision.USE_CLIENT_PLANNING; + + case CATALOG_PREFERRED: + // Catalog prefers server-side, use it if available + if (!serverSupportsPlanning) { + LOG.warn( + "Table {} prefers server-side planning (mode: CATALOG_PREFERRED) but server doesn't support it. " + + "Falling back to client-side planning. Consider upgrading server or changing mode to CLIENT_PREFERRED.", + tableIdentifier); + return PlanningDecision.USE_CLIENT_PLANNING; + } + LOG.debug( + "Using server-side planning for table {} per catalog preference (mode: CATALOG_PREFERRED)", + tableIdentifier); + return PlanningDecision.USE_CATALOG_PLANNING; + + case CATALOG_ONLY: + if (!serverSupportsPlanning) { + throw new UnsupportedOperationException( + String.format( + "Catalog requires server-side planning (mode: CATALOG_ONLY) for table %s but server does not support planning endpoint. " + + "Either change catalog mode or upgrade server to support scan planning.", + tableIdentifier)); + } + LOG.debug( + "Using server-side planning for table {} per catalog requirement (mode: CATALOG_ONLY)", + tableIdentifier); + return PlanningDecision.USE_CATALOG_PLANNING; + + default: + throw new IllegalStateException("Unknown catalog mode: " + catalogMode); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java index 58730ed55c45..9e8d6ab55373 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java @@ -124,7 +124,8 @@ public T execute( .collect(Collectors.toList())) .withOverrides( ImmutableMap.of( - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "optional")) + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, + "catalog-preferred")) .build()); } Object body = roundTripSerialize(request.body(), "request"); @@ -924,7 +925,7 @@ public T execute( CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO", RESTCatalogProperties.REST_SCAN_PLANNING_MODE, - "optional")); + "catalog-preferred")); return new CatalogWithAdapter(catalog, adapter); } @@ -972,10 +973,10 @@ public T execute( RESTCatalog catalog = new RESTCatalog(SessionCatalog.SessionContext.createEmpty(), (config) -> adapter); catalog.initialize( - "test-required", + "test-catalog-only", ImmutableMap.of( RESTCatalogProperties.REST_SCAN_PLANNING_MODE, - "required", + "catalog-only", CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); @@ -983,10 +984,11 @@ public T execute( catalog.createNamespace(NS); // Should throw UnsupportedOperationException when trying to create/load the table - // because REQUIRED mode needs server-side planning but server doesn't support it + // because CATALOG_ONLY mode requires server-side planning but server doesn't support it assertThatThrownBy(() -> catalog.createTable(tableId, SCHEMA)) .isInstanceOf(UnsupportedOperationException.class) - .hasMessageContaining("Server-side scan planning is required"); + .hasMessageContaining("server-side planning") + .hasMessageContaining("CATALOG_ONLY"); catalog.close(); } @@ -995,7 +997,8 @@ public T execute( public void scanPlanningModeNone() throws IOException { // Test NONE mode - should use client-side planning even if server supports it CatalogWithAdapter catalogWithAdapter = - catalogWithTableLevelConfig(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "none"); + catalogWithTableLevelConfig( + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "client-only"); Table table = createTableWithScanPlanning(catalogWithAdapter.catalog, "none_mode_test"); table.newAppend().appendFile(FILE_A).commit(); @@ -1100,7 +1103,8 @@ public void tableLevelScanPlanningOverride( // Catalog that adds scan planning config to LoadTableResponse (table-level override) CatalogWithAdapter catalogWithAdapter = - catalogWithTableLevelConfig(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "optional"); + catalogWithTableLevelConfig( + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "catalog-preferred"); RESTTable table = restTableFor(catalogWithAdapter.catalog, "table_override_test"); setParserContext(table); @@ -1128,7 +1132,8 @@ public T execute( Arrays.stream(Route.values()) .map(r -> Endpoint.create(r.method().name(), r.resourcePath())) .collect(Collectors.toList())) - .withOverride(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "none") + .withOverride( + RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "client-only") .build()); } @@ -1164,7 +1169,7 @@ public T execute( "test", ImmutableMap.of( RESTCatalogProperties.REST_SCAN_PLANNING_MODE, - "none", + "client-only", CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); return new CatalogWithAdapter(catalog, adapter); diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 387b8fe4b00a..8befc9e1fb0d 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1291,10 +1291,30 @@ class LoadTableResult(BaseModel): ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - - `rest-scan-planning-mode`: Controls the scan planning behavior. Valid values are: - - `none` (default): Server-side scan planning is not supported or allowed. Client must use client-side planning. - - `optional`: Client can choose between client-side or server-side planning based on its capabilities and preferences. - - `required`: Server-side planning is required. Client must not use client-side planning. + - `rest-scan-planning-mode`: Controls the catalog's scan planning mode (server-side configuration). This property uses RFC 2119 semantics (MUST, SHOULD, MAY). Valid values are: + - `client-only`: Client MUST use client-side planning. Server MUST NOT provide server-side planning endpoints or capabilities. + - `client-preferred` (default): Client SHOULD use client-side planning. Server MAY provide server-side planning if explicitly requested by client. + - `catalog-preferred`: Client SHOULD use server-side planning when available. Server MAY fall back to client-side planning if server doesn't support planning endpoints or client explicitly requests it. + - `catalog-only`: Client MUST use server-side planning. Server MUST provide server-side planning endpoints. Client MUST NOT use client-side planning. + - `rest-scan-planning-client-preference`: Controls the client's scan planning preference. This property uses RFC 2119 semantics (MUST, SHOULD). Valid values are: + - `none` (default): Client SHOULD follow the catalog's `rest-scan-planning-mode` recommendation. No explicit preference set. + - `client`: Client MUST use client-side planning. Request fails with IllegalStateException if catalog mode is `catalog-only`. + - `catalog`: Client MUST use server-side planning. Request fails with IllegalStateException if catalog mode is `client-only`, or with UnsupportedOperationException if server doesn't support planning endpoints. + + ### Scan Planning Decision Matrix + + The final scan planning decision is determined by the negotiation between `rest-scan-planning-mode` (catalog requirement) and `rest-scan-planning-client-preference` (client requirement): + + | Client Preference | client-only | client-preferred | catalog-preferred | catalog-only | + |-------------------|-------------|------------------|-------------------|--------------| + | **none** (follow catalog) | Client-side | Client-side | Server-side (fallback to client if unavailable) | Server-side (fail if unavailable) | + | **client** (MUST client) | Client-side | Client-side | Client-side | **IllegalStateException** | + | **catalog** (MUST server) | **IllegalStateException** | Server-side (fail if unavailable) | Server-side (fail if unavailable) | Server-side (fail if unavailable) | + + Notes: + - "Server-side (fallback to client if unavailable)" means the client will attempt server-side planning, but gracefully fall back to client-side if the server doesn't advertise planning endpoints. + - "Server-side (fail if unavailable)" means the client will throw UnsupportedOperationException if the server doesn't advertise planning endpoints. + - "IllegalStateException" is thrown when client and catalog MUST requirements conflict (e.g., client MUST use client-side but catalog MUST use server-side). ## AWS Configurations diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 411b0c75baef..ec6938cc8012 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3374,10 +3374,30 @@ components: ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - - `rest-scan-planning-mode`: Controls the scan planning behavior. Valid values are: - - `none` (default): Server-side scan planning is not supported or allowed. Client must use client-side planning. - - `optional`: Client can choose between client-side or server-side planning based on its capabilities and preferences. - - `required`: Server-side planning is required. Client must not use client-side planning. + - `rest-scan-planning-mode`: Controls the catalog's scan planning mode (server-side configuration). Valid values are: + - `client-only`: Client MUST use client-side planning. Server MUST NOT provide server-side planning endpoints or capabilities. + - `client-preferred` (default): Client SHOULD use client-side planning. Server MAY provide server-side planning if explicitly requested by client. + - `catalog-preferred`: Client SHOULD use server-side planning when available. Server MAY fall back to client-side planning if server doesn't support planning endpoints or client explicitly requests it. + - `catalog-only`: Client MUST use server-side planning. Server MUST provide server-side planning endpoints. Client MUST NOT use client-side planning. + - `rest-scan-planning-client-preference`: Controls the client's scan planning preference. Valid values are: + - `none` (default): Client SHOULD follow the catalog's `rest-scan-planning-mode` recommendation. No explicit preference set. + - `client`: Client MUST use client-side planning. Request fails with IllegalStateException if catalog mode is `catalog-only`. + - `catalog`: Client MUST use server-side planning. Request fails with IllegalStateException if catalog mode is `client-only`, or with UnsupportedOperationException if server doesn't support planning endpoints. + + ### Scan Planning Decision Matrix + + The final scan planning decision is determined by the negotiation between `rest-scan-planning-mode` (catalog requirement) and `rest-scan-planning-client-preference` (client requirement): + + | Client Preference | client-only | client-preferred | catalog-preferred | catalog-only | + |-------------------|-------------|------------------|-------------------|--------------| + | **none** (follow catalog) | Client-side | Client-side | Server-side (fallback to client if unavailable) | Server-side (fail if unavailable) | + | **client** (MUST client) | Client-side | Client-side | Client-side | **IllegalStateException** | + | **catalog** (MUST server) | **IllegalStateException** | Server-side (fail if unavailable) | Server-side (fail if unavailable) | Server-side (fail if unavailable) | + + Notes: + - "Server-side (fallback to client if unavailable)" means the client will attempt server-side planning, but gracefully fall back to client-side if the server doesn't advertise planning endpoints. + - "Server-side (fail if unavailable)" means the client will throw UnsupportedOperationException if the server doesn't advertise planning endpoints. + - "IllegalStateException" is thrown when client and catalog MUST requirements conflict (e.g., client MUST use client-side but catalog MUST use server-side). ## AWS Configurations diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java index d9561c6d9695..65d3a6bf7e6f 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java @@ -42,7 +42,7 @@ protected static Object[][] parameters() { .put(CatalogProperties.URI, restCatalog.properties().get(CatalogProperties.URI)) // this flag is typically only set by the server, but we set it from the client for // testing - .put(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "required") + .put(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "catalog-only") .build(), SparkCatalogConfig.REST.catalogName() + ".default.binary_table" } From 1f9bdcb5ba429e5ed65c62713a1c1fdbad26c15e Mon Sep 17 00:00:00 2001 From: Prashant Kumar Singh Date: Wed, 24 Dec 2025 00:24:58 +0000 Subject: [PATCH 6/6] Adjust preference mode --- .../iceberg/rest/RESTCatalogProperties.java | 141 ++++------ .../iceberg/rest/RESTSessionCatalog.java | 66 ++--- .../iceberg/rest/ScanPlanningNegotiator.java | 260 ++++++++++-------- .../apache/iceberg/rest/TestRESTCatalog.java | 36 +++ .../iceberg/rest/TestRESTScanPlanning.java | 25 +- open-api/rest-catalog-open-api.py | 47 ++-- open-api/rest-catalog-open-api.yaml | 47 ++-- .../extensions/TestRemoteScanPlanning.java | 4 +- .../extensions/TestRemoteScanPlanning.java | 4 +- .../extensions/TestRemoteScanPlanning.java | 4 +- .../extensions/TestRemoteScanPlanning.java | 4 +- 11 files changed, 338 insertions(+), 300 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java b/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java index 13f56ce018de..eb80158b0cab 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTCatalogProperties.java @@ -18,6 +18,9 @@ */ package org.apache.iceberg.rest; +import java.util.Arrays; +import java.util.stream.Collectors; + public final class RESTCatalogProperties { private RESTCatalogProperties() {} @@ -37,43 +40,53 @@ private RESTCatalogProperties() {} public static final String NAMESPACE_SEPARATOR = "namespace-separator"; - // Configure scan planning mode on the REST catalog/server side - public static final String REST_SCAN_PLANNING_MODE = "rest-scan-planning-mode"; - public static final String REST_SCAN_PLANNING_MODE_DEFAULT = + // Configure scan planning mode + // Can be set by server in LoadTableResponse.config() or by client in catalog properties + // Negotiation rules: ONLY beats PREFERRED, both PREFERRED = client wins + // Default when neither client nor server provides: client-preferred + public static final String SCAN_PLANNING_MODE = "scan-planning-mode"; + public static final String SCAN_PLANNING_MODE_DEFAULT = ScanPlanningMode.CLIENT_PREFERRED.modeName(); - // Configure client-side scan planning preference - public static final String CLIENT_SCAN_PLANNING_PREFERENCE = - "rest-scan-planning-client-preference"; - public static final String CLIENT_SCAN_PLANNING_PREFERENCE_DEFAULT = - ClientScanPlanningPreference.NONE.preferenceName(); - public enum SnapshotMode { ALL, REFS } /** - * Enum to represent the scan planning mode for REST catalog (server-side configuration). + * Enum to represent scan planning mode configuration. + * + *

Can be configured by: + * + *

    + *
  • Server: Returned in LoadTableResponse.config() to advertise server preference/requirement + *
  • Client: Set in catalog properties to set client preference/requirement + *
+ * + *

When both client and server configure this property, the values are negotiated: * - *

This defines what the catalog/server requires or prefers for scan planning using RFC 2119 - * semantics (MUST, SHOULD, MAY): + *

Values: * *

    - *
  • CLIENT_ONLY - Client MUST use client-side planning. Server MUST NOT provide server-side - * planning endpoints. - *
  • CLIENT_PREFERRED (default) - Client SHOULD use client-side planning. Server MAY provide - * server-side planning if explicitly requested. - *
  • CATALOG_PREFERRED - Client SHOULD use server-side planning when available. Server MAY - * fall back to client-side if server doesn't support planning or client explicitly requests - * it. - *
  • CATALOG_ONLY - Client MUST use server-side planning. Server MUST provide server-side - * planning endpoints. Client MUST NOT use client-side planning. + *
  • CLIENT_ONLY - MUST use client-side planning. Fails if paired with CATALOG_ONLY from other + * side. + *
  • CLIENT_PREFERRED (default) - Prefer client-side planning but flexible. + *
  • CATALOG_PREFERRED - Prefer server-side planning but flexible. Falls back to client if + * server doesn't support planning endpoints. + *
  • CATALOG_ONLY - MUST use server-side planning. Requires server support. Fails if paired + * with CLIENT_ONLY from other side. *
* - *

For the complete decision matrix showing how this negotiates with {@link - * ClientScanPlanningPreference}, see {@link - * org.apache.iceberg.rest.ScanPlanningNegotiator#negotiate}. + *

Negotiation rules when both sides are configured: + * + *

    + *
  • Incompatible: CLIENT_ONLY + CATALOG_ONLY = FAIL + *
  • ONLY beats PREFERRED: One "ONLY" + opposite "PREFERRED" = ONLY wins (inflexible + * beats flexible) + *
  • Both PREFERRED: Different PREFERRED types = Client config wins + *
  • Both same: Use that planning type + *
  • Only one configured: Use the configured side + *
*/ public enum ScanPlanningMode { CLIENT_ONLY("client-only"), @@ -91,20 +104,28 @@ public String modeName() { return modeName; } - public boolean isClientAllowed() { - return this == CLIENT_ONLY || this == CLIENT_PREFERRED; + public boolean isClientOnly() { + return this == CLIENT_ONLY; } - public boolean isCatalogAllowed() { - return this == CATALOG_PREFERRED || this == CATALOG_ONLY; + public boolean isCatalogOnly() { + return this == CATALOG_ONLY; } - public boolean requiresClient() { - return this == CLIENT_ONLY; + public boolean isOnly() { + return this == CLIENT_ONLY || this == CATALOG_ONLY; } - public boolean requiresCatalog() { - return this == CATALOG_ONLY; + public boolean isPreferred() { + return this == CLIENT_PREFERRED || this == CATALOG_PREFERRED; + } + + public boolean prefersClient() { + return this == CLIENT_ONLY || this == CLIENT_PREFERRED; + } + + public boolean prefersCatalog() { + return this == CATALOG_ONLY || this == CATALOG_PREFERRED; } public static ScanPlanningMode fromString(String mode) { @@ -116,60 +137,10 @@ public static ScanPlanningMode fromString(String mode) { return planningMode; } } + String validModes = + Arrays.stream(values()).map(ScanPlanningMode::modeName).collect(Collectors.joining(", ")); throw new IllegalArgumentException( - String.format( - "Invalid scan planning mode: %s. Valid values are: client-only, client-preferred, catalog-preferred, catalog-only", - mode)); - } - } - - /** - * Enum to represent the client-side scan planning preference. - * - *

This defines what the client wants to do for scan planning using RFC 2119 semantics (MUST, - * SHOULD): - * - *

    - *
  • NONE (default) - Client SHOULD follow catalog's {@link ScanPlanningMode} recommendation. - * No explicit preference set. This is the cooperative default behavior. - *
  • CLIENT_PLANNING - Client MUST use client-side planning. Throws {@link - * IllegalStateException} if catalog mode is {@link ScanPlanningMode#CATALOG_ONLY}. - *
  • CATALOG_PLANNING - Client MUST use server-side planning. Throws {@link - * IllegalStateException} if catalog mode is {@link ScanPlanningMode#CLIENT_ONLY}, or {@link - * UnsupportedOperationException} if server doesn't support planning endpoints. - *
- * - *

For the complete decision matrix showing how this negotiates with {@link ScanPlanningMode}, - * see {@link org.apache.iceberg.rest.ScanPlanningNegotiator#negotiate}. - */ - public enum ClientScanPlanningPreference { - NONE("none"), - CLIENT_PLANNING("client"), - CATALOG_PLANNING("catalog"); - - private final String preferenceName; - - ClientScanPlanningPreference(String preferenceName) { - this.preferenceName = preferenceName; - } - - public String preferenceName() { - return preferenceName; - } - - public static ClientScanPlanningPreference fromString(String pref) { - if (pref == null || pref.trim().isEmpty()) { - return NONE; - } - for (ClientScanPlanningPreference preference : values()) { - if (preference.preferenceName.equalsIgnoreCase(pref)) { - return preference; - } - } - throw new IllegalArgumentException( - String.format( - "Invalid client scan planning preference: %s. Valid values are: none, client, catalog", - pref)); + String.format("Invalid scan planning mode: %s. Valid values are: %s", mode, validModes)); } } } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index b8cf6e74f932..f3eb746f193a 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -159,8 +159,8 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog private MetricsReporter reporter = null; private boolean reportingViaRestEnabled; private Integer pageSize = null; - private RESTCatalogProperties.ScanPlanningMode restScanPlanningMode; - private RESTCatalogProperties.ClientScanPlanningPreference clientScanPlanningPreference; + private RESTCatalogProperties.ScanPlanningMode clientConfiguredScanPlanningMode; + private RESTCatalogProperties.ScanPlanningMode catalogLevelScanPlanningMode; private CloseableGroup closeables = null; private Set endpoints; private Supplier> mutationHeaders = Map::of; @@ -273,21 +273,23 @@ public void initialize(String name, Map unresolved) { RESTCatalogProperties.NAMESPACE_SEPARATOR, RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); - String scanPlanningConfig = - PropertyUtil.propertyAsString( - mergedProps, - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, - RESTCatalogProperties.REST_SCAN_PLANNING_MODE_DEFAULT); - this.restScanPlanningMode = - RESTCatalogProperties.ScanPlanningMode.fromString(scanPlanningConfig); - - String clientPreferenceConfig = - PropertyUtil.propertyAsString( - mergedProps, - RESTCatalogProperties.CLIENT_SCAN_PLANNING_PREFERENCE, - RESTCatalogProperties.CLIENT_SCAN_PLANNING_PREFERENCE_DEFAULT); - this.clientScanPlanningPreference = - RESTCatalogProperties.ClientScanPlanningPreference.fromString(clientPreferenceConfig); + // Client-configured scan planning mode (from catalog properties, not from server config) + // Read from un-merged properties to avoid picking up server-provided defaults + String clientScanPlanningConfig = + PropertyUtil.propertyAsString(props, RESTCatalogProperties.SCAN_PLANNING_MODE, null); + this.clientConfiguredScanPlanningMode = + clientScanPlanningConfig != null + ? RESTCatalogProperties.ScanPlanningMode.fromString(clientScanPlanningConfig) + : null; + + // Also store the catalog-level (possibly server-provided) scan planning mode as a fallback + // This comes from ConfigResponse.overrides() and gets merged into mergedProps + String catalogLevelConfig = + PropertyUtil.propertyAsString(mergedProps, RESTCatalogProperties.SCAN_PLANNING_MODE, null); + this.catalogLevelScanPlanningMode = + catalogLevelConfig != null + ? RESTCatalogProperties.ScanPlanningMode.fromString(catalogLevelConfig) + : null; super.initialize(name, mergedProps); } @@ -521,30 +523,28 @@ private RESTTable restTableForScanPlanning( TableIdentifier finalIdentifier, RESTClient restClient, Map tableConf) { - // Determine effective catalog mode (table-level config overrides catalog config) - RESTCatalogProperties.ScanPlanningMode effectiveCatalogMode = restScanPlanningMode; - String tableModeConfig = tableConf.get(RESTCatalogProperties.REST_SCAN_PLANNING_MODE); - if (tableModeConfig != null) { - effectiveCatalogMode = RESTCatalogProperties.ScanPlanningMode.fromString(tableModeConfig); - } - - // Determine effective client preference (could optionally support table-level override) - RESTCatalogProperties.ClientScanPlanningPreference effectiveClientPref = - clientScanPlanningPreference; - String tableClientPrefConfig = - tableConf.get(RESTCatalogProperties.CLIENT_SCAN_PLANNING_PREFERENCE); - if (tableClientPrefConfig != null) { - effectiveClientPref = - RESTCatalogProperties.ClientScanPlanningPreference.fromString(tableClientPrefConfig); + // Get client-configured mode (set in catalog properties during initialization) + RESTCatalogProperties.ScanPlanningMode clientMode = clientConfiguredScanPlanningMode; + + // Get server-provided mode + // Priority: table-level config > catalog-level config (from ConfigResponse) + String tableLevelModeConfig = tableConf.get(RESTCatalogProperties.SCAN_PLANNING_MODE); + RESTCatalogProperties.ScanPlanningMode serverMode; + if (tableLevelModeConfig != null) { + serverMode = RESTCatalogProperties.ScanPlanningMode.fromString(tableLevelModeConfig); + } else { + // Fall back to catalog-level server config (from ConfigResponse.overrides()) + serverMode = catalogLevelScanPlanningMode; } // Check server capabilities boolean serverSupportsPlanning = endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN); // Negotiate scan planning strategy + // Rules: ONLY beats PREFERRED, both PREFERRED = client wins, one side only = use it ScanPlanningNegotiator.PlanningDecision decision = ScanPlanningNegotiator.negotiate( - effectiveCatalogMode, effectiveClientPref, serverSupportsPlanning, finalIdentifier); + clientMode, serverMode, serverSupportsPlanning, finalIdentifier); // Apply the decision if (decision == ScanPlanningNegotiator.PlanningDecision.USE_CATALOG_PLANNING) { diff --git a/core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java b/core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java index 928249701901..0c07fcc8489f 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java +++ b/core/src/main/java/org/apache/iceberg/rest/ScanPlanningNegotiator.java @@ -19,67 +19,34 @@ package org.apache.iceberg.rest; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.rest.RESTCatalogProperties.ClientScanPlanningPreference; import org.apache.iceberg.rest.RESTCatalogProperties.ScanPlanningMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Handles negotiation between client scan planning preferences and catalog scan planning mode - * requirements. + * Handles negotiation between client and server scan planning mode configurations. * *

This class encapsulates the decision logic for determining whether to use client-side or * server-side scan planning based on: * *

    - *
  • Catalog mode (CLIENT_ONLY, CLIENT_PREFERRED, CATALOG_PREFERRED, CATALOG_ONLY) - *
  • Client preference (NONE, CLIENT_PLANNING, CATALOG_PLANNING) + *
  • Client mode (CLIENT_ONLY, CLIENT_PREFERRED, CATALOG_PREFERRED, CATALOG_ONLY, or null) + *
  • Server mode (CLIENT_ONLY, CLIENT_PREFERRED, CATALOG_PREFERRED, CATALOG_ONLY, or null) *
  • Server capabilities (supports planning endpoint or not) *
* - *

Decision Matrix

+ *

Negotiation Rules * - * The following table shows the final decision based on catalog mode and client preference: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Client PreferenceCLIENT_ONLYCLIENT_PREFERREDCATALOG_PREFERREDCATALOG_ONLY
NONE (follow catalog)Client-sideClient-sideServer-side (fallback to client if unavailable)Server-side (fail if unavailable)
CLIENT_PLANNING (MUST client)Client-sideClient-sideClient-sideIllegalStateException
CATALOG_PLANNING (MUST server)IllegalStateExceptionServer-side (fail if unavailable)Server-side (fail if unavailable)Server-side (fail if unavailable)
- * - *

Notes: + *

When both client and server configure scan planning mode, the values are negotiated: * *

    - *
  • "Server-side (fallback to client if unavailable)" - Attempts server-side planning with - * graceful fallback to client-side if server doesn't advertise planning endpoints. - *
  • "Server-side (fail if unavailable)" - Throws {@link UnsupportedOperationException} if - * server doesn't advertise planning endpoints. - *
  • "IllegalStateException" - Thrown when client and catalog MUST requirements conflict. + *
  • Incompatible hard requirements: CLIENT_ONLY + CATALOG_ONLY = FAIL + *
  • ONLY wins over PREFERRED: When one side has "ONLY" and the other has "PREFERRED", + * the ONLY requirement wins (inflexible beats flexible) + *
  • Both PREFERRED: When both are PREFERRED (different types), client config wins + *
  • Both same: When both have the same value, use that planning type + *
  • Only one configured: Use the configured side (client or server) + *
  • Neither configured: Use default (CLIENT_PREFERRED) *
*/ public class ScanPlanningNegotiator { @@ -94,117 +61,180 @@ public enum PlanningDecision { } /** - * Negotiates scan planning strategy between client preference and catalog mode. + * Negotiates scan planning strategy between client and server configurations. + * + *

Precedence: Client config > Server config > Default (CLIENT_PREFERRED) * - * @param catalogMode the catalog's scan planning mode (from server config or table config) - * @param clientPref the client's scan planning preference + * @param clientMode the client's scan planning mode (from catalog properties), may be null + * @param serverMode the server's scan planning mode (from LoadTableResponse.config()), may be + * null * @param serverSupportsPlanning whether the server advertises the scan planning endpoint * @param tableIdentifier the table identifier (for error messages and logging) * @return the negotiated planning decision - * @throws IllegalStateException if client and catalog requirements are incompatible + * @throws IllegalStateException if client and server requirements are incompatible (CLIENT_ONLY + * vs CATALOG_ONLY) * @throws UnsupportedOperationException if required server-side planning but server doesn't * support it */ public static PlanningDecision negotiate( - ScanPlanningMode catalogMode, - ClientScanPlanningPreference clientPref, + ScanPlanningMode clientMode, + ScanPlanningMode serverMode, boolean serverSupportsPlanning, TableIdentifier tableIdentifier) { - switch (clientPref) { - case NONE: - // Client follows catalog's decision - return decideByCatalogMode(catalogMode, serverSupportsPlanning, tableIdentifier); + // Determine effective mode through negotiation + ScanPlanningMode effectiveMode; + String modeSource; - case CLIENT_PLANNING: - // Client explicitly wants client-side planning - if (catalogMode == ScanPlanningMode.CATALOG_ONLY) { - throw new IllegalStateException( - String.format( - "Client requires client-side planning but catalog requires server-side planning for table %s. " - + "Either change client preference or update catalog mode.", - tableIdentifier)); - } - LOG.debug( - "Using client-side planning for table {} per client preference (catalog mode: {})", - tableIdentifier, - catalogMode); - return PlanningDecision.USE_CLIENT_PLANNING; + if (clientMode != null && serverMode != null) { + // Both client and server have configured modes - negotiate + effectiveMode = negotiateBetweenClientAndServer(clientMode, serverMode, tableIdentifier); + modeSource = + String.format( + "negotiated (client: %s, server: %s)", clientMode.modeName(), serverMode.modeName()); + } else if (clientMode != null) { + // Only client configured + effectiveMode = clientMode; + modeSource = "client config"; + } else if (serverMode != null) { + // Only server configured + effectiveMode = serverMode; + modeSource = "server config"; + } else { + // Neither configured, use default + effectiveMode = ScanPlanningMode.CLIENT_PREFERRED; + modeSource = "default"; + } - case CATALOG_PLANNING: - // Client explicitly wants server-side planning - if (catalogMode == ScanPlanningMode.CLIENT_ONLY) { - throw new IllegalStateException( - String.format( - "Client requires server-side planning but catalog requires client-side planning for table %s. " - + "Either change client preference or update catalog mode.", - tableIdentifier)); - } - if (!serverSupportsPlanning) { - throw new UnsupportedOperationException( - String.format( - "Client requires server-side planning for table %s but server does not support planning endpoint. " - + "Either change client preference or upgrade server to support scan planning.", - tableIdentifier)); - } - LOG.debug( - "Using server-side planning for table {} per client preference (catalog mode: {})", - tableIdentifier, - catalogMode); - return PlanningDecision.USE_CATALOG_PLANNING; + // Apply the effective mode + return applyMode(effectiveMode, serverSupportsPlanning, tableIdentifier, modeSource); + } - default: - throw new IllegalStateException("Unknown client preference: " + clientPref); + /** + * Negotiate between client and server modes when both are configured. + * + *

Rules: + * + *

    + *
  • Both same = Use that mode + *
  • Both ONLY but want opposite planning = FAIL (incompatible) + *
  • One ONLY = ONLY wins (inflexible beats flexible) + *
  • Both PREFERRED = Client config wins + *
+ * + * @return the negotiated mode + * @throws IllegalStateException if both are ONLY but want opposite planning locations + */ + private static ScanPlanningMode negotiateBetweenClientAndServer( + ScanPlanningMode clientMode, ScanPlanningMode serverMode, TableIdentifier tableIdentifier) { + + // Fast path: both are the same - no negotiation needed + if (clientMode == serverMode) { + LOG.debug( + "Client and server agree on scan planning mode {} for table {}", + clientMode.modeName(), + tableIdentifier); + return clientMode; + } + + // Check for incompatible hard requirements: both are ONLY but want opposite planning locations + if (clientMode.isOnly() && serverMode.isOnly()) { + // Both have hard requirements but want different things + throw new IllegalStateException( + String.format( + "Incompatible scan planning requirements for table %s: " + + "client requires %s but server requires %s. " + + "Either change client config or update server mode.", + tableIdentifier, + clientMode.prefersClient() ? "client-side planning" : "server-side planning", + serverMode.prefersClient() ? "client-side planning" : "server-side planning")); } + + // ONLY wins over PREFERRED (inflexible beats flexible) + if (clientMode.isOnly()) { + LOG.debug( + "Client mode {} (hard requirement) wins over server mode {} (flexible) for table {}", + clientMode.modeName(), + serverMode.modeName(), + tableIdentifier); + return clientMode; + } + + if (serverMode.isOnly()) { + LOG.debug( + "Server mode {} (hard requirement) wins over client mode {} (flexible) for table {}", + serverMode.modeName(), + clientMode.modeName(), + tableIdentifier); + return serverMode; + } + + // Both are PREFERRED - client config wins + LOG.debug( + "Both client ({}) and server ({}) are flexible (PREFERRED). Client config wins for table {}", + clientMode.modeName(), + serverMode.modeName(), + tableIdentifier); + return clientMode; } - private static PlanningDecision decideByCatalogMode( - ScanPlanningMode catalogMode, + /** + * Apply the effective mode and determine planning decision. + * + * @return the planning decision based on effective mode and server capabilities + * @throws UnsupportedOperationException if CATALOG_ONLY but server doesn't support planning + */ + private static PlanningDecision applyMode( + ScanPlanningMode effectiveMode, boolean serverSupportsPlanning, - TableIdentifier tableIdentifier) { + TableIdentifier tableIdentifier, + String modeSource) { - switch (catalogMode) { + switch (effectiveMode) { case CLIENT_ONLY: - LOG.debug( - "Using client-side planning for table {} per catalog requirement (mode: CLIENT_ONLY)", - tableIdentifier); - return PlanningDecision.USE_CLIENT_PLANNING; - case CLIENT_PREFERRED: LOG.debug( - "Using client-side planning for table {} per catalog preference (mode: CLIENT_PREFERRED)", - tableIdentifier); + "Using client-side planning for table {} (mode: {}, source: {})", + tableIdentifier, + effectiveMode.modeName(), + modeSource); return PlanningDecision.USE_CLIENT_PLANNING; case CATALOG_PREFERRED: - // Catalog prefers server-side, use it if available + // Prefer server-side, but fall back to client if unavailable if (!serverSupportsPlanning) { LOG.warn( - "Table {} prefers server-side planning (mode: CATALOG_PREFERRED) but server doesn't support it. " - + "Falling back to client-side planning. Consider upgrading server or changing mode to CLIENT_PREFERRED.", - tableIdentifier); + "Table {} prefers server-side planning (mode: CATALOG_PREFERRED from {}) " + + "but server doesn't support it. Falling back to client-side planning. " + + "Consider upgrading server or changing mode to CLIENT_PREFERRED.", + tableIdentifier, + modeSource); return PlanningDecision.USE_CLIENT_PLANNING; } LOG.debug( - "Using server-side planning for table {} per catalog preference (mode: CATALOG_PREFERRED)", - tableIdentifier); + "Using server-side planning for table {} (mode: CATALOG_PREFERRED, source: {})", + tableIdentifier, + modeSource); return PlanningDecision.USE_CATALOG_PLANNING; case CATALOG_ONLY: + // Must use server-side, fail if unavailable if (!serverSupportsPlanning) { throw new UnsupportedOperationException( String.format( - "Catalog requires server-side planning (mode: CATALOG_ONLY) for table %s but server does not support planning endpoint. " - + "Either change catalog mode or upgrade server to support scan planning.", - tableIdentifier)); + "Scan planning mode requires server-side planning (CATALOG_ONLY from %s) " + + "for table %s but server does not support planning endpoint. " + + "Either change scan planning mode or upgrade server to support scan planning.", + modeSource, tableIdentifier)); } LOG.debug( - "Using server-side planning for table {} per catalog requirement (mode: CATALOG_ONLY)", - tableIdentifier); + "Using server-side planning for table {} (mode: CATALOG_ONLY, source: {})", + tableIdentifier, + modeSource); return PlanningDecision.USE_CATALOG_PLANNING; default: - throw new IllegalStateException("Unknown catalog mode: " + catalogMode); + throw new IllegalStateException("Unknown scan planning mode: " + effectiveMode); } } } diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java index df4ba3214aea..4ac59532c89d 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java @@ -3528,4 +3528,40 @@ private static List allRequests(RESTCatalogAdapter adapter) { verify(adapter, atLeastOnce()).execute(captor.capture(), any(), any(), any()); return captor.getAllValues(); } + + @Test + public void scanPlanningModeFromString() { + // Null returns CLIENT_PREFERRED default + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString(null)) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CLIENT_PREFERRED); + + // Valid mode names + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("client-only")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CLIENT_ONLY); + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("client-preferred")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CLIENT_PREFERRED); + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("catalog-preferred")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CATALOG_PREFERRED); + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("catalog-only")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CATALOG_ONLY); + + // Case-insensitive parsing + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("CLIENT-ONLY")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CLIENT_ONLY); + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("Client-Preferred")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CLIENT_PREFERRED); + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("CATALOG-PREFERRED")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CATALOG_PREFERRED); + assertThat(RESTCatalogProperties.ScanPlanningMode.fromString("CaTaLoG-oNlY")) + .isEqualTo(RESTCatalogProperties.ScanPlanningMode.CATALOG_ONLY); + + // Invalid mode throws exception with all valid modes listed + assertThatThrownBy(() -> RESTCatalogProperties.ScanPlanningMode.fromString("invalid-mode")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid scan planning mode: invalid-mode") + .hasMessageContaining("client-only") + .hasMessageContaining("client-preferred") + .hasMessageContaining("catalog-preferred") + .hasMessageContaining("catalog-only"); + } } diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java index 9e8d6ab55373..fbc375766ed2 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java @@ -124,8 +124,7 @@ public T execute( .collect(Collectors.toList())) .withOverrides( ImmutableMap.of( - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, - "catalog-preferred")) + RESTCatalogProperties.SCAN_PLANNING_MODE, "catalog-preferred")) .build()); } Object body = roundTripSerialize(request.body(), "request"); @@ -924,7 +923,7 @@ public T execute( ImmutableMap.of( CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO", - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, + RESTCatalogProperties.SCAN_PLANNING_MODE, "catalog-preferred")); return new CatalogWithAdapter(catalog, adapter); } @@ -975,7 +974,7 @@ public T execute( catalog.initialize( "test-catalog-only", ImmutableMap.of( - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, + RESTCatalogProperties.SCAN_PLANNING_MODE, "catalog-only", CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); @@ -997,8 +996,7 @@ public T execute( public void scanPlanningModeNone() throws IOException { // Test NONE mode - should use client-side planning even if server supports it CatalogWithAdapter catalogWithAdapter = - catalogWithTableLevelConfig( - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "client-only"); + catalogWithTableLevelConfig(RESTCatalogProperties.SCAN_PLANNING_MODE, "client-only"); Table table = createTableWithScanPlanning(catalogWithAdapter.catalog, "none_mode_test"); table.newAppend().appendFile(FILE_A).commit(); @@ -1098,13 +1096,12 @@ public void serverSupportsPlanningButNotCancellation() throws IOException { public void tableLevelScanPlanningOverride( Function planMode) throws IOException { - // Test REST_SCAN_PLANNING_MODE in LoadTableResponse.config() overrides catalog setting + // Test SCAN_PLANNING_MODE in LoadTableResponse.config() overrides catalog setting configurePlanningBehavior(planMode); // Catalog that adds scan planning config to LoadTableResponse (table-level override) CatalogWithAdapter catalogWithAdapter = - catalogWithTableLevelConfig( - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "catalog-preferred"); + catalogWithTableLevelConfig(RESTCatalogProperties.SCAN_PLANNING_MODE, "catalog-preferred"); RESTTable table = restTableFor(catalogWithAdapter.catalog, "table_override_test"); setParserContext(table); @@ -1132,8 +1129,7 @@ public T execute( Arrays.stream(Route.values()) .map(r -> Endpoint.create(r.method().name(), r.resourcePath())) .collect(Collectors.toList())) - .withOverride( - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "client-only") + .withOverride(RESTCatalogProperties.SCAN_PLANNING_MODE, "client-only") .build()); } @@ -1168,10 +1164,9 @@ public T execute( catalog.initialize( "test", ImmutableMap.of( - RESTCatalogProperties.REST_SCAN_PLANNING_MODE, - "client-only", - CatalogProperties.FILE_IO_IMPL, - "org.apache.iceberg.inmemory.InMemoryFileIO")); + // Don't set SCAN_PLANNING_MODE here - let server config control it + // (Setting it here would be client configuration which always takes precedence) + CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); return new CatalogWithAdapter(catalog, adapter); } } diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 8befc9e1fb0d..6b22010607a4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1291,30 +1291,29 @@ class LoadTableResult(BaseModel): ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - - `rest-scan-planning-mode`: Controls the catalog's scan planning mode (server-side configuration). This property uses RFC 2119 semantics (MUST, SHOULD, MAY). Valid values are: - - `client-only`: Client MUST use client-side planning. Server MUST NOT provide server-side planning endpoints or capabilities. - - `client-preferred` (default): Client SHOULD use client-side planning. Server MAY provide server-side planning if explicitly requested by client. - - `catalog-preferred`: Client SHOULD use server-side planning when available. Server MAY fall back to client-side planning if server doesn't support planning endpoints or client explicitly requests it. - - `catalog-only`: Client MUST use server-side planning. Server MUST provide server-side planning endpoints. Client MUST NOT use client-side planning. - - `rest-scan-planning-client-preference`: Controls the client's scan planning preference. This property uses RFC 2119 semantics (MUST, SHOULD). Valid values are: - - `none` (default): Client SHOULD follow the catalog's `rest-scan-planning-mode` recommendation. No explicit preference set. - - `client`: Client MUST use client-side planning. Request fails with IllegalStateException if catalog mode is `catalog-only`. - - `catalog`: Client MUST use server-side planning. Request fails with IllegalStateException if catalog mode is `client-only`, or with UnsupportedOperationException if server doesn't support planning endpoints. - - ### Scan Planning Decision Matrix - - The final scan planning decision is determined by the negotiation between `rest-scan-planning-mode` (catalog requirement) and `rest-scan-planning-client-preference` (client requirement): - - | Client Preference | client-only | client-preferred | catalog-preferred | catalog-only | - |-------------------|-------------|------------------|-------------------|--------------| - | **none** (follow catalog) | Client-side | Client-side | Server-side (fallback to client if unavailable) | Server-side (fail if unavailable) | - | **client** (MUST client) | Client-side | Client-side | Client-side | **IllegalStateException** | - | **catalog** (MUST server) | **IllegalStateException** | Server-side (fail if unavailable) | Server-side (fail if unavailable) | Server-side (fail if unavailable) | - - Notes: - - "Server-side (fallback to client if unavailable)" means the client will attempt server-side planning, but gracefully fall back to client-side if the server doesn't advertise planning endpoints. - - "Server-side (fail if unavailable)" means the client will throw UnsupportedOperationException if the server doesn't advertise planning endpoints. - - "IllegalStateException" is thrown when client and catalog MUST requirements conflict (e.g., client MUST use client-side but catalog MUST use server-side). + - `scan-planning-mode`: Controls scan planning behavior for table operations. This property can be configured by: + - **Server**: Returned in `LoadTableResponse.config()` to advertise server preference/requirement + - **Client**: Set in catalog properties to override server configuration + + **Configuration Precedence**: Client config > Server config > Default (`client-preferred`) + + **Valid values**: + - `client-only`: MUST use client-side planning. Fails if paired with server's `catalog-only`. + - `client-preferred` (default): Prefer client-side planning but flexible. + - `catalog-preferred`: Prefer server-side planning but flexible. Falls back to client if server doesn't support planning endpoints. + - `catalog-only`: MUST use server-side planning. Requires server support. Fails if paired with client's `client-only`. + + ### Scan Planning Negotiation + + When both client and server provide `scan-planning-mode` configuration, the final planning decision is negotiated based on the following rules: + + **Negotiation Rules:** + - **Incompatible requirements**: `client-only` + `catalog-only` = **FAIL** + - **ONLY beats PREFERRED**: When one side has "ONLY" and the other has "PREFERRED", the ONLY requirement wins (inflexible beats flexible) + - **Both PREFERRED**: When both are PREFERRED (different types), client config wins + - **Both same**: When both have the same value, use that planning type + - **Only one configured**: Use the configured side (client or server) + - **Neither configured**: Use default (`client-preferred`) ## AWS Configurations diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index ec6938cc8012..05666f391df1 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3374,30 +3374,29 @@ components: ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - - `rest-scan-planning-mode`: Controls the catalog's scan planning mode (server-side configuration). Valid values are: - - `client-only`: Client MUST use client-side planning. Server MUST NOT provide server-side planning endpoints or capabilities. - - `client-preferred` (default): Client SHOULD use client-side planning. Server MAY provide server-side planning if explicitly requested by client. - - `catalog-preferred`: Client SHOULD use server-side planning when available. Server MAY fall back to client-side planning if server doesn't support planning endpoints or client explicitly requests it. - - `catalog-only`: Client MUST use server-side planning. Server MUST provide server-side planning endpoints. Client MUST NOT use client-side planning. - - `rest-scan-planning-client-preference`: Controls the client's scan planning preference. Valid values are: - - `none` (default): Client SHOULD follow the catalog's `rest-scan-planning-mode` recommendation. No explicit preference set. - - `client`: Client MUST use client-side planning. Request fails with IllegalStateException if catalog mode is `catalog-only`. - - `catalog`: Client MUST use server-side planning. Request fails with IllegalStateException if catalog mode is `client-only`, or with UnsupportedOperationException if server doesn't support planning endpoints. - - ### Scan Planning Decision Matrix - - The final scan planning decision is determined by the negotiation between `rest-scan-planning-mode` (catalog requirement) and `rest-scan-planning-client-preference` (client requirement): - - | Client Preference | client-only | client-preferred | catalog-preferred | catalog-only | - |-------------------|-------------|------------------|-------------------|--------------| - | **none** (follow catalog) | Client-side | Client-side | Server-side (fallback to client if unavailable) | Server-side (fail if unavailable) | - | **client** (MUST client) | Client-side | Client-side | Client-side | **IllegalStateException** | - | **catalog** (MUST server) | **IllegalStateException** | Server-side (fail if unavailable) | Server-side (fail if unavailable) | Server-side (fail if unavailable) | - - Notes: - - "Server-side (fallback to client if unavailable)" means the client will attempt server-side planning, but gracefully fall back to client-side if the server doesn't advertise planning endpoints. - - "Server-side (fail if unavailable)" means the client will throw UnsupportedOperationException if the server doesn't advertise planning endpoints. - - "IllegalStateException" is thrown when client and catalog MUST requirements conflict (e.g., client MUST use client-side but catalog MUST use server-side). + - `scan-planning-mode`: Controls scan planning behavior for table operations. This property can be configured by: + - **Server**: Returned in `LoadTableResponse.config()` to advertise server preference/requirement + - **Client**: Set in catalog properties to override server configuration + + **Configuration Precedence**: Client config > Server config > Default (`client-preferred`) + + **Valid values**: + - `client-only`: MUST use client-side planning. Fails if paired with server's `catalog-only`. + - `client-preferred` (default): Prefer client-side planning but flexible. + - `catalog-preferred`: Prefer server-side planning but flexible. Falls back to client if server doesn't support planning endpoints. + - `catalog-only`: MUST use server-side planning. Requires server support. Fails if paired with client's `client-only`. + + ### Scan Planning Negotiation + + When both client and server provide `scan-planning-mode` configuration, the final planning decision is negotiated based on the following rules: + + **Negotiation Rules:** + - **Incompatible requirements**: `client-only` + `catalog-only` = **FAIL** + - **ONLY beats PREFERRED**: When one side has "ONLY" and the other has "PREFERRED", the ONLY requirement wins (inflexible beats flexible) + - **Both PREFERRED**: When both are PREFERRED (different types), client config wins + - **Both same**: When both have the same value, use that planning type + - **Only one configured**: Use the configured side (client or server) + - **Neither configured**: Use default (`client-preferred`) ## AWS Configurations diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java index 14e6c358898c..83c8a2441c2b 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java @@ -42,7 +42,9 @@ protected static Object[][] parameters() { .put(CatalogProperties.URI, restCatalog.properties().get(CatalogProperties.URI)) // this flag is typically only set by the server, but we set it from the client for // testing - .put(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "true") + .put( + RESTCatalogProperties.SCAN_PLANNING_MODE, + RESTCatalogProperties.ScanPlanningMode.CATALOG_ONLY.modeName()) .build(), SparkCatalogConfig.REST.catalogName() + ".default.binary_table" } diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java index 14e6c358898c..83c8a2441c2b 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java @@ -42,7 +42,9 @@ protected static Object[][] parameters() { .put(CatalogProperties.URI, restCatalog.properties().get(CatalogProperties.URI)) // this flag is typically only set by the server, but we set it from the client for // testing - .put(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "true") + .put( + RESTCatalogProperties.SCAN_PLANNING_MODE, + RESTCatalogProperties.ScanPlanningMode.CATALOG_ONLY.modeName()) .build(), SparkCatalogConfig.REST.catalogName() + ".default.binary_table" } diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java index 65d3a6bf7e6f..83c8a2441c2b 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java @@ -42,7 +42,9 @@ protected static Object[][] parameters() { .put(CatalogProperties.URI, restCatalog.properties().get(CatalogProperties.URI)) // this flag is typically only set by the server, but we set it from the client for // testing - .put(RESTCatalogProperties.REST_SCAN_PLANNING_MODE, "catalog-only") + .put( + RESTCatalogProperties.SCAN_PLANNING_MODE, + RESTCatalogProperties.ScanPlanningMode.CATALOG_ONLY.modeName()) .build(), SparkCatalogConfig.REST.catalogName() + ".default.binary_table" } diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java index 14e6c358898c..83c8a2441c2b 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoteScanPlanning.java @@ -42,7 +42,9 @@ protected static Object[][] parameters() { .put(CatalogProperties.URI, restCatalog.properties().get(CatalogProperties.URI)) // this flag is typically only set by the server, but we set it from the client for // testing - .put(RESTCatalogProperties.REST_SCAN_PLANNING_ENABLED, "true") + .put( + RESTCatalogProperties.SCAN_PLANNING_MODE, + RESTCatalogProperties.ScanPlanningMode.CATALOG_ONLY.modeName()) .build(), SparkCatalogConfig.REST.catalogName() + ".default.binary_table" }