From dee103a7cbffb1367675d373e65fd86877d08c07 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 19 Aug 2026 10:36:54 +0800 Subject: [PATCH 01/38] fix(iceberg): close per-table FileIO on cache eviction --- .../iceberg/IcebergMetadataOps.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 74a3b366f87721..be1b5d61173666 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -75,6 +75,7 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.expressions.Term; +import org.apache.iceberg.io.FileIO; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -128,6 +129,26 @@ public Catalog getCatalog() { return catalog; } + /** + * Returns the catalog-level FileIO for session catalogs that expose one internally (e.g. REST). + * Returns null when the catalog does not have a separate catalog-level FileIO or it cannot be determined. + */ + public FileIO getCatalogFileIO() { + if (catalog == null) { + return null; + } + try { + if (catalog instanceof org.apache.iceberg.rest.RESTSessionCatalog) { + java.lang.reflect.Field field = org.apache.iceberg.rest.RESTSessionCatalog.class.getDeclaredField("io"); + field.setAccessible(true); + return (FileIO) field.get(catalog); + } + } catch (Exception e) { + LOG.warn("Failed to get Iceberg catalog FileIO, table cache entries will not close shared FileIO", e); + } + return null; + } + public ExternalCatalog getExternalCatalog() { return dorisCatalog; } From 6d8e164638fea9a97d9650c8094971d962bf9c68 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 19 Aug 2026 10:53:45 +0800 Subject: [PATCH 02/38] fix(style): fix import order in MetaCacheEntryDef --- .../apache/doris/datasource/metacache/MetaCacheEntryDef.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 25f7a21626af73..da843acae27b0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.metacache; +import com.github.benmanes.caffeine.cache.RemovalListener; + import java.util.Objects; import java.util.function.Function; import javax.annotation.Nullable; From 94684794212a95aa301337455248aa270fffb576 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 19 Aug 2026 11:02:21 +0800 Subject: [PATCH 03/38] fix(hudi): safely close shared file system view with reference counting --- .../hudi/HudiExternalMetaCache.java | 33 ++++++++-- .../datasource/hudi/HudiFsViewCacheValue.java | 62 +++++++++++++++++++ .../datasource/hudi/source/HudiScanNode.java | 20 +++++- 3 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index 6df0ca1bc473b7..bd160afd4b8417 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -29,6 +29,7 @@ import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -79,7 +80,7 @@ public class HudiExternalMetaCache extends AbstractExternalMetaCache { public static final String ENTRY_SCHEMA = "schema"; private final EntryHandle partitionEntry; - private final EntryHandle fsViewEntry; + private final EntryHandle fsViewEntry; private final EntryHandle metaClientEntry; private final EntryHandle schemaEntry; @@ -93,8 +94,9 @@ public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheB TablePartitionValues.class, this::loadPartitionValuesCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiPartitionCacheKey::getNameMapping))); fsViewEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_FS_VIEW, HudiFsViewCacheKey.class, - HoodieTableFileSystemView.class, this::createFsView, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(HudiFsViewCacheKey::getNameMapping))); + HudiFsViewCacheValue.class, this::createFsView, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(HudiFsViewCacheKey::getNameMapping), + this::evictFsView)); metaClientEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_META_CLIENT, HudiMetaClientCacheKey.class, HoodieTableMetaClient.class, this::createHoodieTableMetaClient, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiMetaClientCacheKey::getNameMapping))); @@ -110,7 +112,18 @@ public HoodieTableMetaClient getHoodieTableMetaClient(NameMapping nameMapping) { } public HoodieTableFileSystemView getFsView(NameMapping nameMapping) { - return fsViewEntry.get(nameMapping.getCtlId()).get(HudiFsViewCacheKey.of(nameMapping)); + return fsViewEntry.get(nameMapping.getCtlId()).get(HudiFsViewCacheKey.of(nameMapping)).acquire(); + } + + public void releaseFsView(NameMapping nameMapping) { + MetaCacheEntry entry = + fsViewEntry.getIfInitialized(nameMapping.getCtlId()); + if (entry != null) { + HudiFsViewCacheValue value = entry.getIfPresent(HudiFsViewCacheKey.of(nameMapping)); + if (value != null) { + value.release(); + } + } } public HudiSchemaCacheValue getHudiSchemaCacheValue(NameMapping nameMapping, long timestamp) { @@ -144,12 +157,20 @@ public TablePartitionValues getPartitionValues(HMSExternalTable table, boolean u HudiPartitionCacheKey.of(table.getOrBuildNameMapping(), lastTimestamp, useHiveSyncPartition)); } - private HoodieTableFileSystemView createFsView(HudiFsViewCacheKey key) { + private HudiFsViewCacheValue createFsView(HudiFsViewCacheKey key) { HoodieTableMetaClient tableMetaClient = metaClientEntry.get(key.getNameMapping().getCtlId()) .get(HudiMetaClientCacheKey.of(key.getNameMapping())); HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build(); HoodieLocalEngineContext ctx = new HoodieLocalEngineContext(tableMetaClient.getStorageConf()); - return FileSystemViewManager.createInMemoryFileSystemView(ctx, tableMetaClient, metadataConfig); + return new HudiFsViewCacheValue( + FileSystemViewManager.createInMemoryFileSystemView(ctx, tableMetaClient, metadataConfig)); + } + + private void evictFsView(HudiFsViewCacheKey key, HudiFsViewCacheValue value, + com.github.benmanes.caffeine.cache.RemovalCause cause) { + if (value != null) { + value.evict(); + } } private HoodieTableMetaClient createHoodieTableMetaClient(HudiMetaClientCacheKey key) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java new file mode 100644 index 00000000000000..b245c9442be348 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hudi; + +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Reference-counted wrapper around a shared {@link HoodieTableFileSystemView}. + * + *

The underlying fs view is cached per table and shared by concurrent scan nodes. Closing it while + * another thread is still planning splits is unsafe, so the cache only closes the view after the entry has + * been evicted AND all acquired references have been released. + */ +public class HudiFsViewCacheValue { + private final HoodieTableFileSystemView fsView; + private final AtomicInteger refCount = new AtomicInteger(0); + private volatile boolean evicted = false; + private volatile boolean closed = false; + + public HudiFsViewCacheValue(HoodieTableFileSystemView fsView) { + this.fsView = fsView; + } + + public HoodieTableFileSystemView acquire() { + refCount.incrementAndGet(); + return fsView; + } + + public void evict() { + evicted = true; + maybeClose(); + } + + public void release() { + refCount.decrementAndGet(); + maybeClose(); + } + + private synchronized void maybeClose() { + if (evicted && !closed && refCount.get() == 0) { + closed = true; + fsView.close(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index e82e303fe46da6..c85cbbff53fa1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -92,6 +92,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -127,6 +128,7 @@ public class HudiScanNode extends HiveScanNode { private TableScanParams scanParams; private IncrementalRelation incrementalRelation; private HoodieTableFileSystemView fsView; + private final AtomicBoolean fsViewReleased = new AtomicBoolean(false); // The schema information involved in the current query process (including historical schema). protected ConcurrentHashMap currentQuerySchema = new ConcurrentHashMap<>(); @@ -357,6 +359,14 @@ private boolean canUseNativeReader() { return !sessionVariable.isForceJniScanner() && isCowTable; } + private void releaseFsViewOnce() { + if (fsViewReleased.compareAndSet(false, true) && fsView != null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()) + .releaseFsView(hmsTable.getOrBuildNameMapping()); + } + } + private List getPrunedPartitions(HoodieTableMetaClient metaClient) { NameMapping nameMapping = hmsTable.getOrBuildNameMapping(); List partitionColumnTypes = hmsTable.getPartitionColumnTypes(getRelationSnapshot()); @@ -522,7 +532,11 @@ private void getPartitionsSplits(List partitions, List spl @Override public List getSplits(int numBackends) throws UserException { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { - return getIncrementalSplits(); + try { + return getIncrementalSplits(); + } finally { + releaseFsViewOnce(); + } } initPrunedPartitions(); List splits = Collections.synchronizedList(new ArrayList<>()); @@ -533,6 +547,8 @@ public List getSplits(int numBackends) throws UserException { }); } catch (Exception e) { throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); + } finally { + releaseFsViewOnce(); } return splits; } @@ -558,6 +574,7 @@ private void initPrunedPartitions() throws UserException { public void startSplit(int numBackends) { if (prunedPartitions.isEmpty()) { splitAssignment.finishSchedule(); + releaseFsViewOnce(); return; } AtomicInteger numFinishedPartitions = new AtomicInteger(0); @@ -597,6 +614,7 @@ public void startSplit(int numBackends) { System.currentTimeMillis() - startTime); } splitAssignment.finishSchedule(); + releaseFsViewOnce(); } } }, scheduleExecutor); From 0425c8a7a254a7b83307d88dd417de71a3c294e7 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 20 Aug 2026 18:16:40 +0800 Subject: [PATCH 04/38] fix(hudi): make filesystem view cleanup generation-safe --- .../hudi/HudiExternalMetaCache.java | 26 ++-- .../datasource/hudi/HudiFsViewCacheValue.java | 59 +++++-- .../datasource/hudi/source/HudiScanNode.java | 146 ++++++++++-------- .../iceberg/IcebergMetadataOps.java | 21 --- .../hudi/HudiFsViewCacheValueTest.java | 59 +++++++ 5 files changed, 198 insertions(+), 113 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index bd160afd4b8417..91cdcc6034a5c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -29,7 +29,6 @@ import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; -import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -95,8 +94,8 @@ public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheB MetaCacheEntryInvalidation.forNameMapping(HudiPartitionCacheKey::getNameMapping))); fsViewEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_FS_VIEW, HudiFsViewCacheKey.class, HudiFsViewCacheValue.class, this::createFsView, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(HudiFsViewCacheKey::getNameMapping), - this::evictFsView)); + false, MetaCacheEntryInvalidation.forNameMapping(HudiFsViewCacheKey::getNameMapping)) + .withRemovalListener(value -> value, this::evictFsView)); metaClientEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_META_CLIENT, HudiMetaClientCacheKey.class, HoodieTableMetaClient.class, this::createHoodieTableMetaClient, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiMetaClientCacheKey::getNameMapping))); @@ -111,17 +110,13 @@ public HoodieTableMetaClient getHoodieTableMetaClient(NameMapping nameMapping) { return metaClientEntry.get(nameMapping.getCtlId()).get(HudiMetaClientCacheKey.of(nameMapping)); } - public HoodieTableFileSystemView getFsView(NameMapping nameMapping) { - return fsViewEntry.get(nameMapping.getCtlId()).get(HudiFsViewCacheKey.of(nameMapping)).acquire(); - } - - public void releaseFsView(NameMapping nameMapping) { - MetaCacheEntry entry = - fsViewEntry.getIfInitialized(nameMapping.getCtlId()); - if (entry != null) { - HudiFsViewCacheValue value = entry.getIfPresent(HudiFsViewCacheKey.of(nameMapping)); - if (value != null) { - value.release(); + public HudiFsViewCacheValue.Lease getFsView(NameMapping nameMapping) { + HudiFsViewCacheKey key = HudiFsViewCacheKey.of(nameMapping); + while (true) { + HudiFsViewCacheValue value = fsViewEntry.get(nameMapping.getCtlId()).get(key); + HudiFsViewCacheValue.Lease lease = value.tryAcquire(); + if (lease != null) { + return lease; } } } @@ -166,8 +161,7 @@ private HudiFsViewCacheValue createFsView(HudiFsViewCacheKey key) { FileSystemViewManager.createInMemoryFileSystemView(ctx, tableMetaClient, metadataConfig)); } - private void evictFsView(HudiFsViewCacheKey key, HudiFsViewCacheValue value, - com.github.benmanes.caffeine.cache.RemovalCause cause) { + private void evictFsView(HudiFsViewCacheKey key, HudiFsViewCacheValue value) { if (value != null) { value.evict(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java index b245c9442be348..de80b7985664e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java @@ -19,8 +19,6 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView; -import java.util.concurrent.atomic.AtomicInteger; - /** * Reference-counted wrapper around a shared {@link HoodieTableFileSystemView}. * @@ -30,33 +28,68 @@ */ public class HudiFsViewCacheValue { private final HoodieTableFileSystemView fsView; - private final AtomicInteger refCount = new AtomicInteger(0); - private volatile boolean evicted = false; - private volatile boolean closed = false; + // The loader owns one transferable reference until getFsView hands this exact generation to its first caller. + private int refCount = 1; + private boolean loaderReferenceAvailable = true; + private boolean evicted = false; + private boolean closed = false; public HudiFsViewCacheValue(HoodieTableFileSystemView fsView) { this.fsView = fsView; } - public HoodieTableFileSystemView acquire() { - refCount.incrementAndGet(); - return fsView; + public synchronized Lease tryAcquire() { + if (loaderReferenceAvailable) { + loaderReferenceAvailable = false; + return new Lease(this, fsView); + } + if (evicted) { + return null; + } + refCount++; + return new Lease(this, fsView); } - public void evict() { + public synchronized void evict() { evicted = true; maybeClose(); } - public void release() { - refCount.decrementAndGet(); + private synchronized void release() { + if (refCount <= 0) { + throw new IllegalStateException("Hudi fs view released without a matching acquisition"); + } + refCount--; maybeClose(); } - private synchronized void maybeClose() { - if (evicted && !closed && refCount.get() == 0) { + private void maybeClose() { + if (evicted && !closed && refCount == 0) { closed = true; fsView.close(); } } + + /** A lease pins the exact cache generation until split planning has finished using it. */ + public static class Lease implements AutoCloseable { + private HudiFsViewCacheValue owner; + private final HoodieTableFileSystemView fsView; + + private Lease(HudiFsViewCacheValue owner, HoodieTableFileSystemView fsView) { + this.owner = owner; + this.fsView = fsView; + } + + public HoodieTableFileSystemView get() { + return fsView; + } + + @Override + public synchronized void close() { + if (owner != null) { + owner.release(); + owner = null; + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index c85cbbff53fa1a..07289c5cb739b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -38,6 +38,7 @@ import org.apache.doris.datasource.TableFormatType; import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hive.source.HiveScanNode; +import org.apache.doris.datasource.hudi.HudiFsViewCacheValue; import org.apache.doris.datasource.hudi.HudiPartitionUtils; import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; import org.apache.doris.datasource.hudi.HudiUtils; @@ -88,9 +89,9 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Phaser; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -128,6 +129,7 @@ public class HudiScanNode extends HiveScanNode { private TableScanParams scanParams; private IncrementalRelation incrementalRelation; private HoodieTableFileSystemView fsView; + private HudiFsViewCacheValue.Lease fsViewLease; private final AtomicBoolean fsViewReleased = new AtomicBoolean(false); // The schema information involved in the current query process (including historical schema). @@ -237,10 +239,6 @@ protected void doInitialize() throws UserException { storagePropertiesFingerprint = StorageProperties.combinedFsCacheFingerprint( hmsTable.getStoragePropertiesMap().values()); - fsView = Env.getCurrentEnv() - .getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()) - .getFsView(hmsTable.getOrBuildNameMapping()); } finally { if (getSummaryProfile() != null) { getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis() - tableMetaStartTime); @@ -253,6 +251,11 @@ protected void doInitialize() throws UserException { // and `the file column name`. // Split planning and FE-BE schema transport must describe the same pinned Hudi instant. ExternalUtil.initSchemaInfo(params, -1L, table.getFullSchema(relationSnapshot)); + fsViewLease = Env.getCurrentEnv() + .getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()) + .getFsView(hmsTable.getOrBuildNameMapping()); + fsView = fsViewLease.get(); } @Override @@ -360,10 +363,8 @@ private boolean canUseNativeReader() { } private void releaseFsViewOnce() { - if (fsViewReleased.compareAndSet(false, true) && fsView != null) { - Env.getCurrentEnv().getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()) - .releaseFsView(hmsTable.getOrBuildNameMapping()); + if (fsViewReleased.compareAndSet(false, true) && fsViewLease != null) { + fsViewLease.close(); } } @@ -504,22 +505,30 @@ private List planPartitionSplits(HivePartition partition) throws IOEx private void getPartitionsSplits(List partitions, List splits) { Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); - CountDownLatch countDownLatch = new CountDownLatch(partitions.size()); + Phaser tasks = new Phaser(1); AtomicReference throwable = new AtomicReference<>(); long startTime = System.currentTimeMillis(); - partitions.forEach(partition -> executor.execute(() -> { - try { - getPartitionSplits(partition, splits); - } catch (Throwable t) { - throwable.set(t); - } finally { - countDownLatch.countDown(); - } - })); try { - countDownLatch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e.getMessage(), e); + for (HivePartition partition : partitions) { + tasks.register(); + try { + executor.execute(() -> { + try { + getPartitionSplits(partition, splits); + } catch (Throwable t) { + throwable.compareAndSet(null, t); + } finally { + tasks.arriveAndDeregister(); + } + }); + } catch (RuntimeException e) { + tasks.arriveAndDeregister(); + throw e; + } + } + } finally { + // Phaser waiting is uninterruptible, so every accepted task is terminal before the fs-view lease is freed. + tasks.arriveAndAwaitAdvance(); } if (throwable.get() != null) { throw new RuntimeException(throwable.get().getMessage(), throwable.get()); @@ -538,9 +547,9 @@ public List getSplits(int numBackends) throws UserException { releaseFsViewOnce(); } } - initPrunedPartitions(); List splits = Collections.synchronizedList(new ArrayList<>()); try { + initPrunedPartitions(); hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> { getPartitionsSplits(prunedPartitions, splits); return null; @@ -577,52 +586,62 @@ public void startSplit(int numBackends) { releaseFsViewOnce(); return; } - AtomicInteger numFinishedPartitions = new AtomicInteger(0); ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor(); long startTime = System.currentTimeMillis(); - CompletableFuture.runAsync(() -> { - for (HivePartition partition : prunedPartitions) { - if (batchException.get() != null || splitAssignment.isStop()) { - break; - } - try { - splittersOnFlight.acquire(); - } catch (InterruptedException e) { - batchException.set(new UserException(e.getMessage(), e)); - break; - } - CompletableFuture.runAsync(() -> { + try { + CompletableFuture.runAsync(() -> { + List> submittedTasks = new ArrayList<>(); + for (HivePartition partition : prunedPartitions) { + if (batchException.get() != null || splitAssignment.isStop()) { + break; + } try { - List allFiles = Lists.newArrayList(); - getPartitionSplits(partition, allFiles, false); - if (allFiles.size() > numSplitsPerPartition.get()) { - numSplitsPerPartition.set(allFiles.size()); - } - if (splitAssignment.needMoreSplit()) { - splitAssignment.addToQueue(allFiles); - } - } catch (Exception e) { + splittersOnFlight.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); batchException.set(new UserException(e.getMessage(), e)); - } finally { - splittersOnFlight.release(); - if (batchException.get() != null) { - splitAssignment.setException(batchException.get()); - } - if (numFinishedPartitions.incrementAndGet() == prunedPartitions.size()) { - if (getSummaryProfile() != null) { - getSummaryProfile().addExternalTableGetFileScanTasksTime( - System.currentTimeMillis() - startTime); + break; + } + try { + submittedTasks.add(CompletableFuture.runAsync(() -> { + try { + List allFiles = Lists.newArrayList(); + getPartitionSplits(partition, allFiles, false); + if (allFiles.size() > numSplitsPerPartition.get()) { + numSplitsPerPartition.set(allFiles.size()); + } + if (splitAssignment.needMoreSplit()) { + splitAssignment.addToQueue(allFiles); + } + } catch (Exception e) { + batchException.set(new UserException(e.getMessage(), e)); + } finally { + splittersOnFlight.release(); } - splitAssignment.finishSchedule(); - releaseFsViewOnce(); - } + }, scheduleExecutor)); + } catch (RuntimeException e) { + splittersOnFlight.release(); + batchException.set(new UserException(e.getMessage(), e)); + break; } - }, scheduleExecutor); - } - if (batchException.get() != null) { - splitAssignment.setException(batchException.get()); - } - }, scheduleExecutor); + } + CompletableFuture.allOf(submittedTasks.toArray(new CompletableFuture[0])).whenComplete((ignored, t) -> { + if (batchException.get() != null) { + splitAssignment.setException(batchException.get()); + } + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime( + System.currentTimeMillis() - startTime); + } + splitAssignment.finishSchedule(); + releaseFsViewOnce(); + }); + }, scheduleExecutor); + } catch (RuntimeException e) { + batchException.set(new UserException(e.getMessage(), e)); + splitAssignment.setException(batchException.get()); + releaseFsViewOnce(); + } } @Override @@ -633,6 +652,7 @@ public boolean isBatchMode() { try { initPrunedPartitions(); } catch (UserException e) { + releaseFsViewOnce(); throw new RuntimeException(e.getMessage(), e); } int numPartitions = sessionVariable.getNumPartitionsInBatchMode(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index be1b5d61173666..74a3b366f87721 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -75,7 +75,6 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.expressions.Term; -import org.apache.iceberg.io.FileIO; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -129,26 +128,6 @@ public Catalog getCatalog() { return catalog; } - /** - * Returns the catalog-level FileIO for session catalogs that expose one internally (e.g. REST). - * Returns null when the catalog does not have a separate catalog-level FileIO or it cannot be determined. - */ - public FileIO getCatalogFileIO() { - if (catalog == null) { - return null; - } - try { - if (catalog instanceof org.apache.iceberg.rest.RESTSessionCatalog) { - java.lang.reflect.Field field = org.apache.iceberg.rest.RESTSessionCatalog.class.getDeclaredField("io"); - field.setAccessible(true); - return (FileIO) field.get(catalog); - } - } catch (Exception e) { - LOG.warn("Failed to get Iceberg catalog FileIO, table cache entries will not close shared FileIO", e); - } - return null; - } - public ExternalCatalog getExternalCatalog() { return dorisCatalog; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java new file mode 100644 index 00000000000000..b36ac5eb7b8848 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hudi; + +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class HudiFsViewCacheValueTest { + + @Test + public void testEvictionClosesAfterExactLeaseRelease() { + HoodieTableFileSystemView view = Mockito.mock(HoodieTableFileSystemView.class); + HudiFsViewCacheValue value = new HudiFsViewCacheValue(view); + HudiFsViewCacheValue.Lease lease = value.tryAcquire(); + + Assert.assertNotNull(lease); + Assert.assertSame(view, lease.get()); + value.evict(); + Mockito.verify(view, Mockito.never()).close(); + Assert.assertNull(value.tryAcquire()); + + lease.close(); + Mockito.verify(view).close(); + lease.close(); + Mockito.verifyNoMoreInteractions(view); + } + + @Test + public void testEvictionBeforeLoaderReferenceHandoff() { + HoodieTableFileSystemView view = Mockito.mock(HoodieTableFileSystemView.class); + HudiFsViewCacheValue value = new HudiFsViewCacheValue(view); + + value.evict(); + + Mockito.verify(view, Mockito.never()).close(); + HudiFsViewCacheValue.Lease lease = value.tryAcquire(); + Assert.assertNotNull(lease); + lease.close(); + Mockito.verify(view).close(); + Assert.assertNull(value.tryAcquire()); + } +} From 3418bcf2351b5bfec4c6e41a195739f9f863f5d7 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 20 Aug 2026 19:16:17 +0800 Subject: [PATCH 05/38] fix(metacache): retain external resources for active users --- .../datasource/hudi/HudiFsViewCacheValue.java | 20 +- .../datasource/hudi/source/HudiScanNode.java | 280 ++++++++++++------ .../iceberg/rewrite/RewriteGroupTask.java | 12 +- .../doris/nereids/StatementContext.java | 60 +++- .../hudi/HudiFsViewCacheValueTest.java | 19 +- .../hudi/source/HudiBatchFsViewOwnerTest.java | 67 +++++ .../iceberg/IcebergTableCacheValueTest.java | 124 ++++++++ .../doris/nereids/StatementContextTest.java | 24 ++ 8 files changed, 512 insertions(+), 94 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java index de80b7985664e3..3f3b4feead6085 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java @@ -39,15 +39,25 @@ public HudiFsViewCacheValue(HoodieTableFileSystemView fsView) { } public synchronized Lease tryAcquire() { + Lease lease; if (loaderReferenceAvailable) { loaderReferenceAvailable = false; - return new Lease(this, fsView); - } - if (evicted) { + lease = new Lease(this, fsView); + } else if (evicted) { return null; + } else { + refCount++; + lease = new Lease(this, fsView); + } + try { + // The cache uses expire-after-access without detached refresh. Sync every foreground generation handoff + // so a continuously hot key still observes newly completed commits. + fsView.sync(); + return lease; + } catch (RuntimeException e) { + lease.close(); + throw e; } - refCount++; - return new Lease(this, fsView); } public synchronized void evict() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 07289c5cb739b3..68f16a9044c1c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.ExternalUtil; import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.SplitAssignment; import org.apache.doris.datasource.TableFormatType; import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hive.source.HiveScanNode; @@ -48,6 +49,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; @@ -77,6 +79,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -89,9 +92,9 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Phaser; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -131,6 +134,7 @@ public class HudiScanNode extends HiveScanNode { private HoodieTableFileSystemView fsView; private HudiFsViewCacheValue.Lease fsViewLease; private final AtomicBoolean fsViewReleased = new AtomicBoolean(false); + private final Object batchFsViewResourceKey = new Object(); // The schema information involved in the current query process (including historical schema). protected ConcurrentHashMap currentQuerySchema = new ConcurrentHashMap<>(); @@ -251,11 +255,6 @@ protected void doInitialize() throws UserException { // and `the file column name`. // Split planning and FE-BE schema transport must describe the same pinned Hudi instant. ExternalUtil.initSchemaInfo(params, -1L, table.getFullSchema(relationSnapshot)); - fsViewLease = Env.getCurrentEnv() - .getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()) - .getFsView(hmsTable.getOrBuildNameMapping()); - fsView = fsViewLease.get(); } @Override @@ -368,6 +367,20 @@ private void releaseFsViewOnce() { } } + private synchronized void acquireFsView() { + if (fsViewLease != null) { + return; + } + if (fsViewReleased.get()) { + throw new IllegalStateException("Hudi filesystem-view lease has already been released"); + } + fsViewLease = Env.getCurrentEnv() + .getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()) + .getFsView(hmsTable.getOrBuildNameMapping()); + fsView = fsViewLease.get(); + } + private List getPrunedPartitions(HoodieTableMetaClient metaClient) { NameMapping nameMapping = hmsTable.getOrBuildNameMapping(); List partitionColumnTypes = hmsTable.getPartitionColumnTypes(getRelationSnapshot()); @@ -505,30 +518,29 @@ private List planPartitionSplits(HivePartition partition) throws IOEx private void getPartitionsSplits(List partitions, List splits) { Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); - Phaser tasks = new Phaser(1); + List> acceptedTasks = new ArrayList<>(partitions.size()); AtomicReference throwable = new AtomicReference<>(); + RuntimeException submissionFailure = null; long startTime = System.currentTimeMillis(); - try { - for (HivePartition partition : partitions) { - tasks.register(); - try { - executor.execute(() -> { - try { - getPartitionSplits(partition, splits); - } catch (Throwable t) { - throwable.compareAndSet(null, t); - } finally { - tasks.arriveAndDeregister(); - } - }); - } catch (RuntimeException e) { - tasks.arriveAndDeregister(); - throw e; - } + for (HivePartition partition : partitions) { + try { + acceptedTasks.add(CompletableFuture.runAsync(() -> { + try { + getPartitionSplits(partition, splits); + } catch (Throwable t) { + throwable.compareAndSet(null, t); + } + }, executor)); + } catch (RuntimeException e) { + submissionFailure = e; + break; } - } finally { - // Phaser waiting is uninterruptible, so every accepted task is terminal before the fs-view lease is freed. - tasks.arriveAndAwaitAdvance(); + } + // CompletableFuture.allOf has no Phaser party limit and join is uninterruptible: every accepted task is + // terminal before the caller releases the filesystem-view lease, including submission rejection. + CompletableFuture.allOf(acceptedTasks.toArray(new CompletableFuture[0])).join(); + if (submissionFailure != null) { + throw submissionFailure; } if (throwable.get() != null) { throw new RuntimeException(throwable.get().getMessage(), throwable.get()); @@ -540,26 +552,23 @@ private void getPartitionsSplits(List partitions, List spl @Override public List getSplits(int numBackends) throws UserException { - if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { - try { + acquireFsView(); + try { + if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { return getIncrementalSplits(); - } finally { - releaseFsViewOnce(); } - } - List splits = Collections.synchronizedList(new ArrayList<>()); - try { + List splits = Collections.synchronizedList(new ArrayList<>()); initPrunedPartitions(); hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> { getPartitionsSplits(prunedPartitions, splits); return null; }); + return splits; } catch (Exception e) { throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); } finally { releaseFsViewOnce(); } - return splits; } private void initPrunedPartitions() throws UserException { @@ -586,61 +595,166 @@ public void startSplit(int numBackends) { releaseFsViewOnce(); return; } + acquireFsView(); ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor(); + Executor producerExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); long startTime = System.currentTimeMillis(); + BatchFsViewOwner createdOwner = new BatchFsViewOwner(splitAssignment, fsViewLease); + BatchFsViewOwner batchOwner = createdOwner; + ConnectContext connectContext = ConnectContext.get(); + StatementContext statementContext = connectContext == null ? null : connectContext.getStatementContext(); + if (statementContext != null) { + try { + batchOwner = statementContext.getOrRegisterStatementResource( + batchFsViewResourceKey, () -> createdOwner); + if (batchOwner != createdOwner) { + createdOwner.finish(); + throw new IllegalStateException("Hudi batch split owner was registered twice"); + } + } catch (RuntimeException e) { + createdOwner.finish(); + throw e; + } + } + + BatchFsViewOwner finalBatchOwner = batchOwner; + AtomicInteger pendingTasks = new AtomicInteger(1); // producer reference + Runnable taskFinished = () -> { + if (pendingTasks.decrementAndGet() == 0) { + finishBatchSplit(finalBatchOwner, startTime); + } + }; try { - CompletableFuture.runAsync(() -> { - List> submittedTasks = new ArrayList<>(); - for (HivePartition partition : prunedPartitions) { - if (batchException.get() != null || splitAssignment.isStop()) { - break; - } - try { - splittersOnFlight.acquire(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - batchException.set(new UserException(e.getMessage(), e)); - break; - } - try { - submittedTasks.add(CompletableFuture.runAsync(() -> { - try { - List allFiles = Lists.newArrayList(); - getPartitionSplits(partition, allFiles, false); - if (allFiles.size() > numSplitsPerPartition.get()) { - numSplitsPerPartition.set(allFiles.size()); - } - if (splitAssignment.needMoreSplit()) { - splitAssignment.addToQueue(allFiles); + producerExecutor.execute(() -> { + try { + for (HivePartition partition : prunedPartitions) { + if (batchException.get() != null || splitAssignment.isStop()) { + break; + } + try { + splittersOnFlight.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + recordBatchException(e); + break; + } + if (batchException.get() != null || splitAssignment.isStop()) { + splittersOnFlight.release(); + break; + } + pendingTasks.incrementAndGet(); + try { + scheduleExecutor.execute(() -> { + try { + List allFiles = Lists.newArrayList(); + getPartitionSplits(partition, allFiles, false); + if (allFiles.size() > numSplitsPerPartition.get()) { + numSplitsPerPartition.set(allFiles.size()); + } + if (splitAssignment.needMoreSplit()) { + splitAssignment.addToQueue(allFiles); + } + } catch (Throwable t) { + recordBatchException(t); + } finally { + splittersOnFlight.release(); + taskFinished.run(); } - } catch (Exception e) { - batchException.set(new UserException(e.getMessage(), e)); - } finally { - splittersOnFlight.release(); - } - }, scheduleExecutor)); - } catch (RuntimeException e) { - splittersOnFlight.release(); - batchException.set(new UserException(e.getMessage(), e)); - break; + }); + } catch (RuntimeException e) { + splittersOnFlight.release(); + recordBatchException(e); + taskFinished.run(); + break; + } } + } catch (Throwable t) { + recordBatchException(t); + } finally { + taskFinished.run(); } - CompletableFuture.allOf(submittedTasks.toArray(new CompletableFuture[0])).whenComplete((ignored, t) -> { - if (batchException.get() != null) { - splitAssignment.setException(batchException.get()); - } - if (getSummaryProfile() != null) { - getSummaryProfile().addExternalTableGetFileScanTasksTime( - System.currentTimeMillis() - startTime); - } - splitAssignment.finishSchedule(); - releaseFsViewOnce(); - }); - }, scheduleExecutor); + }); } catch (RuntimeException e) { - batchException.set(new UserException(e.getMessage(), e)); - splitAssignment.setException(batchException.get()); - releaseFsViewOnce(); + recordBatchException(e); + taskFinished.run(); + } + } + + private void recordBatchException(Throwable t) { + batchException.compareAndSet(null, new UserException(t.getMessage(), t)); + } + + private void finishBatchSplit(BatchFsViewOwner batchOwner, long startTime) { + try { + if (batchException.get() != null) { + splitAssignment.setException(batchException.get()); + } + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } + splitAssignment.finishSchedule(); + } finally { + batchOwner.finish(); + } + } + + @VisibleForTesting + static class BatchFsViewOwner implements Closeable { + private final SplitAssignment splitAssignment; + private final HudiFsViewCacheValue.Lease lease; + private final AtomicBoolean finished = new AtomicBoolean(); + private final AtomicReference finishFailure = new AtomicReference<>(); + private final CountDownLatch terminal = new CountDownLatch(1); + + BatchFsViewOwner(SplitAssignment splitAssignment, HudiFsViewCacheValue.Lease lease) { + this.splitAssignment = splitAssignment; + this.lease = lease; + } + + void finish() { + if (finished.compareAndSet(false, true)) { + try { + lease.close(); + } catch (RuntimeException e) { + finishFailure.set(e); + throw e; + } finally { + terminal.countDown(); + } + } + } + + @Override + public void close() { + RuntimeException stopFailure = null; + if (!finished.get()) { + try { + splitAssignment.stop(); + } catch (RuntimeException e) { + stopFailure = e; + } + } + boolean interrupted = false; + while (true) { + try { + terminal.await(); + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + if (stopFailure != null) { + if (finishFailure.get() != null) { + stopFailure.addSuppressed(finishFailure.get()); + } + throw stopFailure; + } + if (finishFailure.get() != null) { + throw finishFailure.get(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java index d20931a83b309c..6476f922bdc970 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java @@ -125,9 +125,10 @@ public void execute() throws JobException { return; } + ConnectContext taskConnectContext = null; try { // Step 1: Create and customize a new ConnectContext for this task - ConnectContext taskConnectContext = buildConnectContext(); + taskConnectContext = buildConnectContext(); // Set target file size for Iceberg write taskConnectContext.getSessionVariable().setIcebergWriteTargetFileSizeBytes(targetFileSizeBytes); // Custom file scan tasks for rewrite operations @@ -160,7 +161,14 @@ public void execute() throws JobException { throw new JobException("Rewrite group execution failed: " + e.getMessage(), e); } finally { - isFinished.set(true); + try { + if (taskConnectContext != null && taskConnectContext.getStatementContext() != null) { + taskConnectContext.getStatementContext().close(); + } + } finally { + ConnectContext.remove(); + isFinished.set(true); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 85bfabf02f412d..040b1265ea9f98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -204,6 +204,11 @@ public enum TableFrom { // table locks private final Stack plannerResources = new Stack<>(); + // Resources that must outlive planning and remain valid until the statement itself finishes. + // Keep these separate from plannerResources: NereidsPlanner releases planner resources as soon as + // physical planning completes, while external split planning can still use statement-scoped objects. + private final Map statementResources = new LinkedHashMap<>(); + private boolean statementResourcesClosed; // placeholder params for prepared statement private List placeholders = new ArrayList<>(); @@ -933,11 +938,56 @@ public synchronized void releasePlannerResources() { } } + /** + * Returns one closeable resource per statement key and closes it when this statement is closed. + * The supplier is invoked at most once for a key. This is intentionally independent from planner locks, + * whose lifetime ends at the end of Nereids planning. + */ + @SuppressWarnings("unchecked") + public synchronized T getOrRegisterStatementResource( + Object resourceKey, java.util.function.Supplier supplier) { + if (statementResourcesClosed) { + throw new IllegalStateException("Statement resources are already closed"); + } + CloseableResource existing = statementResources.get(resourceKey); + if (existing != null) { + return (T) existing.resource; + } + T resource = supplier.get(); + statementResources.put(resourceKey, new CloseableResource( + String.valueOf(resourceKey), Thread.currentThread().getName(), + originStatement == null ? null : originStatement.originStmt, resource)); + return resource; + } + + private synchronized void releaseStatementResources() { + if (statementResourcesClosed) { + return; + } + statementResourcesClosed = true; + Throwable throwable = null; + List resources = new ArrayList<>(statementResources.values()); + statementResources.clear(); + for (int i = resources.size() - 1; i >= 0; i--) { + try { + resources.get(i).close(); + } catch (Throwable t) { + if (throwable == null) { + throwable = t; + } + } + } + if (throwable != null) { + Throwables.throwIfInstanceOf(throwable, RuntimeException.class); + throw new IllegalStateException("Release statement resource failed", throwable); + } + } + // CHECKSTYLE OFF @Override protected void finalize() throws Throwable { - if (!plannerResources.isEmpty()) { - String msg = "Resources leak: " + plannerResources; + if (!plannerResources.isEmpty() || !statementResources.isEmpty()) { + String msg = "Resources leak: planner=" + plannerResources + ", statement=" + statementResources; LOG.error(msg); throw new IllegalStateException(msg); } @@ -947,7 +997,11 @@ protected void finalize() throws Throwable { @Override public void close() { clearExternalScanTasks(); - releasePlannerResources(); + try { + releaseStatementResources(); + } finally { + releasePlannerResources(); + } } public List getPlaceholders() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java index b36ac5eb7b8848..5b280212b6d02e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java @@ -32,6 +32,7 @@ public void testEvictionClosesAfterExactLeaseRelease() { Assert.assertNotNull(lease); Assert.assertSame(view, lease.get()); + Mockito.verify(view).sync(); value.evict(); Mockito.verify(view, Mockito.never()).close(); Assert.assertNull(value.tryAcquire()); @@ -39,7 +40,7 @@ public void testEvictionClosesAfterExactLeaseRelease() { lease.close(); Mockito.verify(view).close(); lease.close(); - Mockito.verifyNoMoreInteractions(view); + Mockito.verify(view, Mockito.times(1)).close(); } @Test @@ -52,8 +53,24 @@ public void testEvictionBeforeLoaderReferenceHandoff() { Mockito.verify(view, Mockito.never()).close(); HudiFsViewCacheValue.Lease lease = value.tryAcquire(); Assert.assertNotNull(lease); + Mockito.verify(view).sync(); lease.close(); Mockito.verify(view).close(); Assert.assertNull(value.tryAcquire()); } + + @Test + public void testLeaseSynchronizesHotCachedView() { + HoodieTableFileSystemView view = Mockito.mock(HoodieTableFileSystemView.class); + HudiFsViewCacheValue value = new HudiFsViewCacheValue(view); + HudiFsViewCacheValue.Lease firstLease = value.tryAcquire(); + + Assert.assertNotNull(firstLease); + firstLease.close(); + HudiFsViewCacheValue.Lease secondLease = value.tryAcquire(); + Assert.assertNotNull(secondLease); + secondLease.close(); + + Mockito.verify(view, Mockito.times(2)).sync(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java new file mode 100644 index 00000000000000..bfb22b74f07426 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hudi.source; + +import org.apache.doris.datasource.SplitAssignment; +import org.apache.doris.datasource.hudi.HudiFsViewCacheValue; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +class HudiBatchFsViewOwnerTest { + + @Test + void statementCloseStopsAndJoinsBeforeReleasingLease() throws Exception { + SplitAssignment assignment = Mockito.mock(SplitAssignment.class); + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future close = executor.submit(owner::close); + Mockito.verify(assignment, Mockito.timeout(3000)).stop(); + Assertions.assertFalse(close.isDone()); + Mockito.verify(lease, Mockito.never()).close(); + + owner.finish(); + + close.get(3, TimeUnit.SECONDS); + Mockito.verify(lease).close(); + } finally { + executor.shutdownNow(); + } + } + + @Test + void normalCompletionDoesNotStopFinishedAssignment() { + SplitAssignment assignment = Mockito.mock(SplitAssignment.class); + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + + owner.finish(); + owner.close(); + + Mockito.verify(assignment, Mockito.never()).stop(); + Mockito.verify(lease).close(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java new file mode 100644 index 00000000000000..7c622d8dd1d0b6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.nereids.StatementContext; + +import org.apache.iceberg.Table; +import org.apache.iceberg.io.FileIO; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Proxy; +import java.util.concurrent.atomic.AtomicInteger; + +class IcebergTableCacheValueTest { + + @Test + void classifiesOnlyPerTableFileIOAsOwned() { + FileIO tableIo = newProxy(FileIO.class); + FileIO catalogIo = newProxy(FileIO.class); + + Assertions.assertTrue(IcebergExternalMetaCache.shouldCloseTableFileIO( + IcebergExternalCatalog.ICEBERG_GLUE, tableIo, null)); + Assertions.assertTrue(IcebergExternalMetaCache.shouldCloseTableFileIO( + IcebergExternalCatalog.ICEBERG_S3_TABLES, tableIo, null)); + Assertions.assertTrue(IcebergExternalMetaCache.shouldCloseTableFileIO( + IcebergExternalCatalog.ICEBERG_REST, tableIo, catalogIo)); + Assertions.assertFalse(IcebergExternalMetaCache.shouldCloseTableFileIO( + IcebergExternalCatalog.ICEBERG_REST, catalogIo, catalogIo)); + Assertions.assertFalse(IcebergExternalMetaCache.shouldCloseTableFileIO( + IcebergExternalCatalog.ICEBERG_REST, tableIo, null)); + Assertions.assertFalse(IcebergExternalMetaCache.shouldCloseTableFileIO( + IcebergExternalCatalog.ICEBERG_DLF, tableIo, null)); + } + + @Test + void evictionWaitsForActiveBorrower() { + AtomicInteger cleanupCount = new AtomicInteger(); + IcebergTableCacheValue value = newValue(cleanupCount); + IcebergTableCacheValue.Lease lease = value.tryAcquire(); + Assertions.assertNotNull(lease); + value.releaseLoaderReference(); + + value.releaseCacheReference(); + Assertions.assertEquals(0, cleanupCount.get()); + + lease.close(); + Assertions.assertEquals(1, cleanupCount.get()); + lease.close(); + Assertions.assertEquals(1, cleanupCount.get()); + } + + @Test + void statementCloseReleasesBorrowerAfterPlannerResources() { + AtomicInteger cleanupCount = new AtomicInteger(); + IcebergTableCacheValue value = newValue(cleanupCount); + IcebergTableCacheValue.Lease lease = value.tryAcquire(); + Assertions.assertNotNull(lease); + value.releaseLoaderReference(); + + StatementContext statementContext = new StatementContext(); + statementContext.getOrRegisterStatementResource("iceberg-table:1\u0000db\u0000tbl", () -> lease); + value.releaseCacheReference(); + statementContext.releasePlannerResources(); + Assertions.assertEquals(0, cleanupCount.get()); + + statementContext.close(); + Assertions.assertEquals(1, cleanupCount.get()); + } + + @Test + void loaderReferenceBridgesEvictionBeforeBorrow() { + AtomicInteger cleanupCount = new AtomicInteger(); + IcebergTableCacheValue value = newValue(cleanupCount); + + value.releaseCacheReference(); + IcebergTableCacheValue.Lease lease = value.tryAcquire(); + Assertions.assertNotNull(lease); + value.releaseLoaderReference(); + Assertions.assertEquals(0, cleanupCount.get()); + + lease.close(); + Assertions.assertEquals(1, cleanupCount.get()); + } + + @Test + void unborrowedRetiredValueClosesExactlyOnce() { + AtomicInteger cleanupCount = new AtomicInteger(); + IcebergTableCacheValue value = newValue(cleanupCount); + + value.releaseCacheReference(); + value.releaseLoaderReference(); + value.releaseCacheReference(); + value.releaseLoaderReference(); + + Assertions.assertEquals(1, cleanupCount.get()); + } + + private IcebergTableCacheValue newValue(AtomicInteger cleanupCount) { + Table table = newProxy(Table.class); + return new IcebergTableCacheValue(table, () -> null, cleanupCount::incrementAndGet); + } + + @SuppressWarnings("unchecked") + private T newProxy(Class type) { + return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, + (proxy, method, args) -> null); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 9bd52a7cb044eb..9892d4685a5b97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -42,6 +42,7 @@ import org.mockito.InOrder; import org.mockito.Mockito; +import java.io.Closeable; import java.util.AbstractList; import java.util.Arrays; import java.util.Collections; @@ -59,6 +60,29 @@ public class StatementContextTest { + @Test + public void testStatementResourceOutlivesPlannerResources() { + StatementContext statementContext = new StatementContext(); + AtomicInteger closed = new AtomicInteger(); + Closeable first = statementContext.getOrRegisterStatementResource("iceberg:1:db:tbl", + () -> closed::incrementAndGet); + Closeable second = statementContext.getOrRegisterStatementResource("iceberg:1:db:tbl", + () -> { + throw new AssertionError("same statement resource must be reused"); + }); + + org.junit.jupiter.api.Assertions.assertSame(first, second); + statementContext.releasePlannerResources(); + org.junit.jupiter.api.Assertions.assertEquals(0, closed.get()); + + statementContext.close(); + org.junit.jupiter.api.Assertions.assertEquals(1, closed.get()); + statementContext.close(); + org.junit.jupiter.api.Assertions.assertEquals(1, closed.get()); + org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, + () -> statementContext.getOrRegisterStatementResource("late", () -> () -> { })); + } + @Test public void testPreloadExternalTablesBeforeLock() { ConnectContext connectContext = Mockito.mock(ConnectContext.class); From b746b1da0ca8769f60d4ee41476c7cb829f5a7ca Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 20 Aug 2026 20:17:52 +0800 Subject: [PATCH 06/38] fix(iceberg): retain catalog resources for active tables --- .../IcebergCatalogResourceTracker.java | 113 ++++++++++++++++++ .../iceberg/IcebergExternalCatalog.java | 68 +++++++++-- .../iceberg/IcebergExternalTable.java | 10 +- .../datasource/iceberg/IcebergUtils.java | 18 ++- .../IcebergCatalogResourceTrackerTest.java | 79 ++++++++++++ .../iceberg/IcebergTableCacheValueTest.java | 21 ++++ 6 files changed, 292 insertions(+), 17 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTrackerTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java new file mode 100644 index 00000000000000..c8c49443d4d047 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** Keeps one catalog generation alive while tables loaded through it still have owners or borrowers. */ +final class IcebergCatalogResourceTracker { + private Generation current = new Generation(); + + synchronized LoadGuard beginLoad() { + current.retain(); + return new LoadGuard(current); + } + + synchronized void retireCurrent(Runnable cleanup) { + Generation retired = current; + current = new Generation(); + retired.retire(cleanup); + } + + static final class LoadGuard implements AutoCloseable { + private final Generation generation; + private final AtomicBoolean transferred = new AtomicBoolean(); + + private LoadGuard(Generation generation) { + this.generation = generation; + } + + ResourceLease promote() { + if (!transferred.compareAndSet(false, true)) { + throw new IllegalStateException("Iceberg catalog load guard was already completed"); + } + return new ResourceLease(generation); + } + + @Override + public void close() { + if (transferred.compareAndSet(false, true)) { + generation.release(); + } + } + } + + static final class ResourceLease implements AutoCloseable { + private final Generation generation; + private final AtomicBoolean closed = new AtomicBoolean(); + + private ResourceLease(Generation generation) { + this.generation = generation; + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + generation.release(); + } + } + } + + private static final class Generation { + private int references; + private boolean retired; + private boolean cleaned; + private Runnable cleanup; + + private synchronized void retain() { + if (cleaned) { + throw new IllegalStateException("Iceberg catalog generation was already cleaned"); + } + references++; + } + + private synchronized void retire(Runnable cleanup) { + if (retired) { + return; + } + retired = true; + this.cleanup = cleanup; + maybeCleanup(); + } + + private synchronized void release() { + if (references <= 0) { + throw new IllegalStateException("Iceberg catalog generation released too many times"); + } + references--; + maybeCleanup(); + } + + private void maybeCleanup() { + if (retired && references == 0 && !cleaned) { + cleaned = true; + cleanup.run(); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index ded5ff679dc18c..1b333b3e5ceb38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.datasource.InitCatalogLog; @@ -33,6 +34,7 @@ import org.apache.doris.datasource.property.metastore.AbstractIcebergProperties; import org.apache.doris.transaction.TransactionManagerFactory; +import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Catalog; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -63,6 +65,7 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog { public static final long DEFAULT_ICEBERG_MANIFEST_CACHE_TTL_SECOND = 48 * 60 * 60; protected String icebergCatalogType; protected Catalog catalog; + private final IcebergCatalogResourceTracker resourceTracker = new IcebergCatalogResourceTracker(); private AbstractIcebergProperties msProperties; @@ -151,6 +154,12 @@ public Catalog getCatalog() { return ((IcebergMetadataOps) metadataOps).getCatalog(); } + synchronized TableLoadContext beginTableLoad() { + makeSureInitialized(); + return new TableLoadContext((IcebergMetadataOps) metadataOps, executionAuthenticator, icebergCatalogType, + resourceTracker.beginLoad()); + } + public String getIcebergCatalogType() { makeSureInitialized(); return icebergCatalogType; @@ -173,17 +182,58 @@ protected List listTableNamesFromRemote(SessionContext ctx, String dbNam } @Override - public void onClose() { + public synchronized void onClose() { super.onClose(); - if (null != catalog) { - try { - if (catalog instanceof AutoCloseable) { - ((AutoCloseable) catalog).close(); - } - catalog = null; - } catch (Exception e) { - LOG.warn("Failed to close iceberg catalog: {}", getName(), e); + Catalog retiredCatalog = catalog; + catalog = null; + if (retiredCatalog != null) { + resourceTracker.retireCurrent(() -> closeCatalog(retiredCatalog)); + } + } + + private void closeCatalog(Catalog retiredCatalog) { + try { + if (retiredCatalog instanceof AutoCloseable) { + ((AutoCloseable) retiredCatalog).close(); } + } catch (Exception e) { + LOG.warn("Failed to close iceberg catalog: {}", getName(), e); + } + } + + final class TableLoadContext implements AutoCloseable { + private final IcebergMetadataOps ops; + private final ExecutionAuthenticator authenticator; + private final String catalogType; + private final IcebergCatalogResourceTracker.LoadGuard guard; + + private TableLoadContext(IcebergMetadataOps ops, ExecutionAuthenticator authenticator, String catalogType, + IcebergCatalogResourceTracker.LoadGuard guard) { + this.ops = ops; + this.authenticator = authenticator; + this.catalogType = catalogType; + this.guard = guard; + } + + IcebergMetadataOps getOps() { + return ops; + } + + Table loadTable(String dbName, String tableName) throws Exception { + return authenticator.execute(() -> ops.loadTable(dbName, tableName)); + } + + String getCatalogType() { + return catalogType; + } + + IcebergCatalogResourceTracker.ResourceLease promote() { + return guard.promote(); + } + + @Override + public void close() { + guard.close(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 9c9ee6d53b6416..7adda3f51aecae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -243,14 +243,20 @@ public boolean isPartitionColumnAllowNull() { * And the column couldn't change to another column during partition evolution. */ @Override - public boolean isValidRelatedTable() { + public synchronized boolean isValidRelatedTable() { makeSureInitialized(); + if (isValidRelatedTableCached) { + return isValidRelatedTable; + } + return IcebergUtils.withIcebergTable(this, this::isValidRelatedTable); + } + + synchronized boolean isValidRelatedTable(Table table) { if (isValidRelatedTableCached) { return isValidRelatedTable; } isValidRelatedTable = false; Set allFields = Sets.newHashSet(); - Table table = getIcebergTable(); for (PartitionSpec spec : table.specs().values()) { if (spec == null) { isValidRelatedTableCached = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 7c56314c569e92..9828370864fa5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -158,6 +158,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -1151,6 +1152,11 @@ public static Table getWritableIcebergTable(ExternalTable dorisTable, IcebergMet return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable, expectedOps); } + /** The action must return derived metadata rather than retain the supplied table. */ + static T withIcebergTable(ExternalTable dorisTable, Function action) { + return icebergExternalMetaCache(dorisTable).withIcebergTable(dorisTable, action); + } + private static IcebergExternalMetaCache icebergExternalMetaCache(ExternalCatalog catalog) { Preconditions.checkNotNull(catalog, "catalog can not be null"); return Env.getCurrentEnv().getExtMetaCacheMgr().iceberg(catalog.getId()); @@ -1607,10 +1613,10 @@ public static long getCountFromSummary(Map summary, boolean igno * @return estimated row count */ public static long getIcebergRowCount(ExternalTable tbl) { - // the table may be null when the iceberg metadata cache is not loaded.But I don't think it's a problem, - // because the NPE would be caught in the caller and return the default value -1. - // Meanwhile, it will trigger iceberg metadata cache to load the table, so we can get it next time. - Table icebergTable = getIcebergTable(tbl); + return withIcebergTable(tbl, icebergTable -> getIcebergRowCount(tbl, icebergTable)); + } + + private static long getIcebergRowCount(ExternalTable tbl, Table icebergTable) { Snapshot snapshot = icebergTable.currentSnapshot(); if (snapshot == null) { LOG.info("Iceberg table {}.{}.{} is empty, return -1.", @@ -2288,8 +2294,8 @@ private static Optional loadViewSchemaCacheValue(ExternalTable } private static Optional loadTableSchemaCacheValue(ExternalTable dorisTable, long schemaId) { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - return Optional.of(buildTableSchemaCacheValue(dorisTable, schemaId, icebergTable)); + return Optional.of(IcebergUtils.withIcebergTable(dorisTable, + icebergTable -> buildTableSchemaCacheValue(dorisTable, schemaId, icebergTable))); } private static IcebergSchemaCacheValue buildTableSchemaCacheValue(ExternalTable dorisTable, long schemaId, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTrackerTest.java new file mode 100644 index 00000000000000..7418e47594c9e5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTrackerTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +class IcebergCatalogResourceTrackerTest { + + @Test + void catalogRetirementWaitsForLoadedTableOwner() { + IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); + IcebergCatalogResourceTracker.LoadGuard guard = tracker.beginLoad(); + IcebergCatalogResourceTracker.ResourceLease lease = guard.promote(); + guard.close(); + AtomicInteger closeCalls = new AtomicInteger(); + + tracker.retireCurrent(closeCalls::incrementAndGet); + Assertions.assertEquals(0, closeCalls.get()); + + lease.close(); + Assertions.assertEquals(1, closeCalls.get()); + lease.close(); + Assertions.assertEquals(1, closeCalls.get()); + } + + @Test + void failedLoadReleasesRetiredGeneration() { + IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); + IcebergCatalogResourceTracker.LoadGuard guard = tracker.beginLoad(); + AtomicInteger closeCalls = new AtomicInteger(); + + tracker.retireCurrent(closeCalls::incrementAndGet); + Assertions.assertEquals(0, closeCalls.get()); + + guard.close(); + Assertions.assertEquals(1, closeCalls.get()); + } + + @Test + void consecutiveCatalogGenerationsRetireIndependently() { + IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); + IcebergCatalogResourceTracker.LoadGuard first = tracker.beginLoad(); + IcebergCatalogResourceTracker.ResourceLease firstLease = first.promote(); + first.close(); + AtomicInteger firstCloseCalls = new AtomicInteger(); + tracker.retireCurrent(firstCloseCalls::incrementAndGet); + + IcebergCatalogResourceTracker.LoadGuard second = tracker.beginLoad(); + IcebergCatalogResourceTracker.ResourceLease secondLease = second.promote(); + second.close(); + AtomicInteger secondCloseCalls = new AtomicInteger(); + tracker.retireCurrent(secondCloseCalls::incrementAndGet); + + secondLease.close(); + Assertions.assertEquals(0, firstCloseCalls.get()); + Assertions.assertEquals(1, secondCloseCalls.get()); + + firstLease.close(); + Assertions.assertEquals(1, firstCloseCalls.get()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java index 7c622d8dd1d0b6..1e7796cafc5c7b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -111,6 +111,27 @@ void unborrowedRetiredValueClosesExactlyOnce() { Assertions.assertEquals(1, cleanupCount.get()); } + @Test + void catalogRetirementWaitsForTableEvictionAndBorrower() { + IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); + IcebergCatalogResourceTracker.LoadGuard guard = tracker.beginLoad(); + IcebergCatalogResourceTracker.ResourceLease catalogLease = guard.promote(); + guard.close(); + AtomicInteger catalogCloseCount = new AtomicInteger(); + IcebergTableCacheValue value = new IcebergTableCacheValue(newProxy(Table.class), () -> null, + catalogLease::close); + IcebergTableCacheValue.Lease borrower = value.tryAcquire(); + Assertions.assertNotNull(borrower); + value.releaseLoaderReference(); + + tracker.retireCurrent(catalogCloseCount::incrementAndGet); + value.releaseCacheReference(); + Assertions.assertEquals(0, catalogCloseCount.get()); + + borrower.close(); + Assertions.assertEquals(1, catalogCloseCount.get()); + } + private IcebergTableCacheValue newValue(AtomicInteger cleanupCount) { Table table = newProxy(Table.class); return new IcebergTableCacheValue(table, () -> null, cleanupCount::incrementAndGet); From 6365e1474a462711782baaaaff80390632bae899 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 20 Aug 2026 20:34:26 +0800 Subject: [PATCH 07/38] fix(external): align resources with execution lifetime --- .../doris/nereids/StatementContext.java | 57 ++++++++++++++++ .../trees/plans/commands/ExecuteCommand.java | 3 + .../org/apache/doris/qe/StmtExecutor.java | 55 ++++++++++++++- .../iceberg/IcebergExternalMetaCacheTest.java | 1 + .../iceberg/IcebergTableCacheValueTest.java | 68 +++++++++++++++++-- .../doris/nereids/StatementContextTest.java | 32 +++++++++ .../plans/commands/ExecuteCommandTest.java | 10 +++ .../org/apache/doris/qe/StmtExecutorTest.java | 21 ++++++ 8 files changed, 240 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 040b1265ea9f98..a1c814a2507e4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -960,6 +960,29 @@ public synchronized T getOrRegisterStatementResource( return resource; } + /** Start a new execution-scoped resource generation for a retained prepared statement context. */ + public synchronized void beginStatementResourceGeneration() { + if (!statementResources.isEmpty()) { + throw new IllegalStateException("Previous statement resources are still active"); + } + statementResourcesClosed = false; + } + + /** + * Transfers the current statement resources to a later completion boundary. The returned handle owns + * exactly this generation and is idempotent, while {@link #close()} will no longer release the resources. + */ + public synchronized Closeable detachStatementResources() { + if (statementResourcesClosed || statementResources.isEmpty()) { + statementResourcesClosed = true; + return () -> { }; + } + statementResourcesClosed = true; + List resources = new ArrayList<>(statementResources.values()); + statementResources.clear(); + return new DetachedStatementResources(resources); + } + private synchronized void releaseStatementResources() { if (statementResourcesClosed) { return; @@ -983,6 +1006,40 @@ private synchronized void releaseStatementResources() { } } + private static class DetachedStatementResources implements Closeable { + private final List resources; + private boolean closed; + + private DetachedStatementResources(List resources) { + this.resources = resources; + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + closed = true; + Throwable throwable = null; + for (int i = resources.size() - 1; i >= 0; i--) { + try { + resources.get(i).close(); + } catch (Throwable t) { + if (throwable == null) { + throwable = t; + } else { + throwable.addSuppressed(t); + } + } + } + resources.clear(); + if (throwable != null) { + Throwables.throwIfInstanceOf(throwable, RuntimeException.class); + throw new IllegalStateException("Release detached statement resource failed", throwable); + } + } + } + // CHECKSTYLE OFF @Override protected void finalize() throws Throwable { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index 6d0fc555e7824f..7ddeb5b6a677c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -73,6 +73,9 @@ public R accept(PlanVisitor visitor, C context) { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { StatementContext statementContext = ctx.getStatementContext(); + // PREPARE retains this StatementContext, but ConnectProcessor closes the resources from each + // COM_STMT_EXECUTE. Reopen an empty generation before the next execution starts planning. + statementContext.beginStatementResourceGeneration(); statementContext.setPrepareStage(false); statementContext.setIsInsert(false); statementContext.resetMvccSnapshots(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 5c6004fcd10286..c0e9dad355fbc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -151,6 +151,7 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.TSerializer; +import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -194,6 +195,7 @@ public class StmtExecutor { // is finalized later by ConnectContext (see #62259), so the eager close in executeAndSendResult // is skipped. private volatile boolean deferredForArrowFlight = false; + private Closeable deferredArrowFlightStatementResources; private MasterOpExecutor masterOpExecutor = null; private RedirectStatus redirectStatus = null; private Planner planner; @@ -972,16 +974,64 @@ public boolean isDeferredForArrowFlight() { return deferredForArrowFlight; } + void deferArrowFlightQuery() { + Closeable resources = statementContext.detachStatementResources(); + deferredArrowFlightStatementResources = resources; + deferredForArrowFlight = true; + try { + context.addFlightSqlDeferredExecutor(this); + } catch (RuntimeException | Error t) { + deferredForArrowFlight = false; + deferredArrowFlightStatementResources = null; + try { + resources.close(); + } catch (Throwable closeFailure) { + t.addSuppressed(closeFailure); + } + throw t; + } + } + // Finalize an Arrow Flight query whose coordinator was kept alive across the // GetFlightInfo -> DoGet phases: close the coordinator (releasing external-table batch // SplitSources and the query queue slot) and then unregister the query. See #62259. public void finalizeArrowFlightQuery() { + Throwable failure = null; try { if (coord != null) { coord.close(); } - } finally { + } catch (Throwable t) { + failure = t; + } + try { + if (deferredArrowFlightStatementResources != null) { + deferredArrowFlightStatementResources.close(); + } + } catch (Throwable t) { + if (failure == null) { + failure = t; + } else { + failure.addSuppressed(t); + } + } + try { finalizeQuery(); + } catch (Throwable t) { + if (failure == null) { + failure = t; + } else { + failure.addSuppressed(t); + } + } + if (failure != null) { + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure instanceof Error) { + throw (Error) failure; + } + throw new IllegalStateException("Failed to finalize Arrow Flight query", failure); } } @@ -1434,8 +1484,7 @@ public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, // at the end of GetFlightInfo. Point queries use a different coordBase (not // deferred). See #62259. if (coordBase == coord) { - deferredForArrowFlight = true; - context.addFlightSqlDeferredExecutor(this); + deferArrowFlightQuery(); } return; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 7d559a23b075ae..a6975eb020ed79 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -2613,6 +2613,7 @@ public void testInvalidateDbAndStats() { Map stats = cache.stats(catalogId); Assert.assertTrue(stats.containsKey(IcebergExternalMetaCache.ENTRY_TABLE)); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_TABLE).isAutoRefresh()); Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_MANIFEST).isConfigEnabled()); Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_MANIFEST).isEffectiveEnabled()); Assert.assertFalse(stats.get(IcebergExternalMetaCache.ENTRY_MANIFEST).isAutoRefresh()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java index 1e7796cafc5c7b..a1f543433940f7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -17,14 +17,23 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.nereids.StatementContext; +import com.github.benmanes.caffeine.cache.LoadingCache; import org.apache.iceberg.Table; import org.apache.iceberg.io.FileIO; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.lang.reflect.Field; import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; class IcebergTableCacheValueTest { @@ -103,14 +112,57 @@ void unborrowedRetiredValueClosesExactlyOnce() { AtomicInteger cleanupCount = new AtomicInteger(); IcebergTableCacheValue value = newValue(cleanupCount); - value.releaseCacheReference(); - value.releaseLoaderReference(); - value.releaseCacheReference(); - value.releaseLoaderReference(); + value.retire(); + value.retire(); Assertions.assertEquals(1, cleanupCount.get()); } + @Test + void refreshedValueCanRetireBeforeItsFirstBorrow() { + AtomicInteger cleanupCount = new AtomicInteger(); + IcebergTableCacheValue refreshedValue = newValue(cleanupCount); + + refreshedValue.retire(); + + Assertions.assertEquals(1, cleanupCount.get()); + Assertions.assertNull(refreshedValue.tryAcquire()); + } + + @Test + void refreshPublishesNewGenerationWithoutClosingActiveOldBorrower() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + List cleanupCounts = new ArrayList<>(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>("iceberg-table", key -> { + AtomicInteger cleanupCount = new AtomicInteger(); + cleanupCounts.add(cleanupCount); + return newValue(cleanupCount); + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, true, false, + (key, value) -> value.retire()); + IcebergTableCacheValue first = entry.get("table"); + IcebergTableCacheValue.Lease oldBorrower = first.tryAcquire(); + Assertions.assertNotNull(oldBorrower); + first.releaseLoaderReference(); + + extractLoadingCache(entry).refresh("table"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assertions.assertEquals(2, cleanupCounts.size()); + IcebergTableCacheValue second = entry.getIfPresent("table"); + Assertions.assertNotSame(first, second); + Assertions.assertEquals(0, cleanupCounts.get(0).get()); + + oldBorrower.close(); + Assertions.assertEquals(1, cleanupCounts.get(0).get()); + entry.invalidateKey("table"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assertions.assertEquals(1, cleanupCounts.get(1).get()); + } finally { + refreshExecutor.shutdownNow(); + } + } + @Test void catalogRetirementWaitsForTableEvictionAndBorrower() { IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); @@ -137,6 +189,14 @@ private IcebergTableCacheValue newValue(AtomicInteger cleanupCount) { return new IcebergTableCacheValue(table, () -> null, cleanupCount::incrementAndGet); } + @SuppressWarnings("unchecked") + private LoadingCache extractLoadingCache( + MetaCacheEntry entry) throws Exception { + Field field = MetaCacheEntry.class.getDeclaredField("loadingData"); + field.setAccessible(true); + return (LoadingCache) field.get(entry); + } + @SuppressWarnings("unchecked") private T newProxy(Class type) { return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 9892d4685a5b97..1de4a2dcd90c6f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -83,6 +83,38 @@ public void testStatementResourceOutlivesPlannerResources() { () -> statementContext.getOrRegisterStatementResource("late", () -> () -> { })); } + @Test + public void testPreparedStatementStartsFreshResourceGenerationForEveryExecute() { + StatementContext statementContext = new StatementContext(); + AtomicInteger firstClosed = new AtomicInteger(); + AtomicInteger secondClosed = new AtomicInteger(); + + statementContext.getOrRegisterStatementResource("table", () -> firstClosed::incrementAndGet); + statementContext.close(); + org.junit.jupiter.api.Assertions.assertEquals(1, firstClosed.get()); + + statementContext.beginStatementResourceGeneration(); + statementContext.getOrRegisterStatementResource("table", () -> secondClosed::incrementAndGet); + statementContext.close(); + org.junit.jupiter.api.Assertions.assertEquals(1, firstClosed.get()); + org.junit.jupiter.api.Assertions.assertEquals(1, secondClosed.get()); + } + + @Test + public void testDetachedStatementResourcesOutliveStatementContext() throws Exception { + StatementContext statementContext = new StatementContext(); + AtomicInteger closed = new AtomicInteger(); + statementContext.getOrRegisterStatementResource("arrow-flight", () -> closed::incrementAndGet); + + Closeable detached = statementContext.detachStatementResources(); + statementContext.close(); + org.junit.jupiter.api.Assertions.assertEquals(0, closed.get()); + + detached.close(); + detached.close(); + org.junit.jupiter.api.Assertions.assertEquals(1, closed.get()); + } + @Test public void testPreloadExternalTablesBeforeLock() { ConnectContext connectContext = Mockito.mock(ConnectContext.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java index ca0d8306f535c9..8cb06f8cfec182 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java @@ -72,9 +72,19 @@ public void testResolvedScanOptionsAreResetForEveryExecute() throws Exception { new ExecuteCommand("stmt", prepareCommand, statementContext).run(connectContext, executor); Assertions.assertEquals("2", resolveNextSnapshot(scanParams, snapshotId)); + AtomicInteger firstResourceClosed = new AtomicInteger(); + statementContext.getOrRegisterStatementResource("iceberg-table", + () -> firstResourceClosed::incrementAndGet); + statementContext.close(); + Assertions.assertEquals(1, firstResourceClosed.get()); new ExecuteCommand("stmt", prepareCommand, statementContext).run(connectContext, executor); Assertions.assertEquals("3", resolveNextSnapshot(scanParams, snapshotId)); + AtomicInteger secondResourceClosed = new AtomicInteger(); + statementContext.getOrRegisterStatementResource("iceberg-table", + () -> secondResourceClosed::incrementAndGet); + statementContext.close(); + Assertions.assertEquals(1, secondResourceClosed.get()); Mockito.verify(executor, Mockito.times(2)).execute(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 607cea3b40a302..e82ff89a464825 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -24,6 +24,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.ResultFileSink; @@ -107,6 +108,26 @@ public void testFinalizeArrowFlightQueryUnregistersQueryEvenIfCoordCloseThrows() Assert.assertNull(QeProcessorImpl.INSTANCE.getCoordinator(queryId)); } + @Test + public void testArrowFlightDefersStatementResourcesUntilDoGetCompletion() { + StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); + StatementContext statementContext = connectContext.getStatementContext(); + AtomicInteger resourceCloseCount = new AtomicInteger(); + statementContext.getOrRegisterStatementResource("hudi-batch-owner", + () -> resourceCloseCount::incrementAndGet); + Coordinator coord = Mockito.mock(Coordinator.class); + Mockito.when(coord.getQueryOptions()).thenReturn(new TQueryOptions()); + stmtExecutor.setCoord(coord); + + stmtExecutor.deferArrowFlightQuery(); + statementContext.close(); + Assert.assertEquals(0, resourceCloseCount.get()); + + stmtExecutor.finalizeArrowFlightQuery(); + Assert.assertEquals(1, resourceCloseCount.get()); + Mockito.verify(coord).close(); + } + @Test public void testKill() throws Exception { StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); From 187b26a1dfaaa4c8e881876394fda38b3f46041b Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 21 Aug 2026 15:15:50 +0800 Subject: [PATCH 08/38] fix(external): close branch 4.1 lifecycle gaps --- .../datasource/hudi/source/HudiScanNode.java | 137 ++++++++++++------ .../iceberg/IcebergExternalCatalog.java | 25 +++- .../iceberg/IcebergMetadataOps.java | 4 +- .../datasource/iceberg/IcebergUtils.java | 5 + .../iceberg/source/IcebergScanNode.java | 18 ++- .../doris/qe/AutoCloseConnectContext.java | 17 ++- .../org/apache/doris/qe/ConnectProcessor.java | 13 +- .../doris/qe/MysqlConnectProcessor.java | 2 +- .../hudi/source/HudiBatchFsViewOwnerTest.java | 16 ++ .../iceberg/IcebergExternalTableTest.java | 12 +- .../iceberg/IcebergTableCacheValueTest.java | 17 +++ .../doris/qe/AutoCloseConnectContextTest.java | 52 +++++++ 12 files changed, 251 insertions(+), 67 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 68f16a9044c1c5..c579cc36cabfa5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -92,9 +92,11 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.concurrent.FutureTask; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -624,58 +626,90 @@ public void startSplit(int numBackends) { finishBatchSplit(finalBatchOwner, startTime); } }; - try { - producerExecutor.execute(() -> { - try { - for (HivePartition partition : prunedPartitions) { - if (batchException.get() != null || splitAssignment.isStop()) { - break; - } - try { - splittersOnFlight.acquire(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - recordBatchException(e); - break; - } - if (batchException.get() != null || splitAssignment.isStop()) { - splittersOnFlight.release(); - break; - } - pendingTasks.incrementAndGet(); + TerminalTask producerTask = terminalTask(() -> { + try { + for (HivePartition partition : prunedPartitions) { + if (batchException.get() != null || splitAssignment.isStop()) { + break; + } + try { + splittersOnFlight.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + recordBatchException(e); + break; + } + if (batchException.get() != null || splitAssignment.isStop()) { + splittersOnFlight.release(); + break; + } + pendingTasks.incrementAndGet(); + TerminalTask partitionTask = terminalTask(() -> { try { - scheduleExecutor.execute(() -> { - try { - List allFiles = Lists.newArrayList(); - getPartitionSplits(partition, allFiles, false); - if (allFiles.size() > numSplitsPerPartition.get()) { - numSplitsPerPartition.set(allFiles.size()); - } - if (splitAssignment.needMoreSplit()) { - splitAssignment.addToQueue(allFiles); - } - } catch (Throwable t) { - recordBatchException(t); - } finally { - splittersOnFlight.release(); - taskFinished.run(); - } - }); - } catch (RuntimeException e) { - splittersOnFlight.release(); - recordBatchException(e); - taskFinished.run(); - break; + List allFiles = Lists.newArrayList(); + getPartitionSplits(partition, allFiles, false); + if (allFiles.size() > numSplitsPerPartition.get()) { + numSplitsPerPartition.set(allFiles.size()); + } + if (splitAssignment.needMoreSplit()) { + splitAssignment.addToQueue(allFiles); + } + } catch (Throwable t) { + recordBatchException(t); } + }, () -> { + splittersOnFlight.release(); + taskFinished.run(); + }); + finalBatchOwner.track(partitionTask); + try { + scheduleExecutor.execute(partitionTask); + } catch (RuntimeException e) { + recordBatchException(e); + partitionTask.cancelBeforeStart(); + break; } - } catch (Throwable t) { - recordBatchException(t); - } finally { - taskFinished.run(); } - }); + } catch (Throwable t) { + recordBatchException(t); + } + }, taskFinished); + finalBatchOwner.track(producerTask); + try { + producerExecutor.execute(producerTask); } catch (RuntimeException e) { recordBatchException(e); + producerTask.cancelBeforeStart(); + } + } + + private TerminalTask terminalTask(Runnable task, Runnable taskFinished) { + return new TerminalTask(task, taskFinished); + } + + @VisibleForTesting + static class TerminalTask extends FutureTask { + private final AtomicBoolean started = new AtomicBoolean(); + private final Runnable taskFinished; + + TerminalTask(Runnable task, Runnable taskFinished) { + super(task, null); + this.taskFinished = taskFinished; + } + + @Override + public void run() { + if (started.compareAndSet(false, true)) { + super.run(); + } + } + + boolean cancelBeforeStart() { + return started.compareAndSet(false, true) && cancel(false); + } + + @Override + protected void done() { taskFinished.run(); } } @@ -705,6 +739,8 @@ static class BatchFsViewOwner implements Closeable { private final AtomicBoolean finished = new AtomicBoolean(); private final AtomicReference finishFailure = new AtomicReference<>(); private final CountDownLatch terminal = new CountDownLatch(1); + private final ConcurrentLinkedQueue tasks = new ConcurrentLinkedQueue<>(); + private final AtomicBoolean stopping = new AtomicBoolean(); BatchFsViewOwner(SplitAssignment splitAssignment, HudiFsViewCacheValue.Lease lease) { this.splitAssignment = splitAssignment; @@ -724,15 +760,24 @@ void finish() { } } + void track(TerminalTask task) { + tasks.add(task); + if (stopping.get()) { + task.cancelBeforeStart(); + } + } + @Override public void close() { RuntimeException stopFailure = null; if (!finished.get()) { + stopping.set(true); try { splitAssignment.stop(); } catch (RuntimeException e) { stopFailure = e; } + tasks.forEach(TerminalTask::cancelBeforeStart); } boolean interrupted = false; while (true) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 1b333b3e5ceb38..5260bb0bbee028 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; public abstract class IcebergExternalCatalog extends ExternalCatalog { @@ -126,14 +127,14 @@ protected synchronized void initPreExecutionAuthenticator() { protected void initLocalObjectsImpl() { initCatalog(); initPreExecutionAuthenticator(); - IcebergMetadataOps ops = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog); - transactionManager = TransactionManagerFactory.createIcebergTransactionManager(ops); threadPoolWithPreAuth = ThreadPoolManager.newDaemonFixedThreadPoolWithPreAuth( ICEBERG_CATALOG_EXECUTOR_THREAD_NUM, Integer.MAX_VALUE, String.format("iceberg_catalog_%s_executor_pool", name), true, executionAuthenticator); + IcebergMetadataOps ops = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog); + transactionManager = TransactionManagerFactory.createIcebergTransactionManager(ops); metadataOps = ops; } @@ -183,15 +184,29 @@ protected List listTableNamesFromRemote(SessionContext ctx, String dbNam @Override public synchronized void onClose() { + ThreadPoolExecutor retiredExecutor = threadPoolWithPreAuth; + threadPoolWithPreAuth = null; super.onClose(); Catalog retiredCatalog = catalog; catalog = null; - if (retiredCatalog != null) { - resourceTracker.retireCurrent(() -> closeCatalog(retiredCatalog)); - } + resourceTracker.retireCurrent(() -> { + closeCatalog(retiredCatalog); + if (retiredExecutor != null) { + ThreadPoolManager.shutdownExecutorService(retiredExecutor); + } + }); + } + + @Override + public synchronized void resetToUninitialized(boolean invalidCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + super.resetToUninitialized(invalidCache); } private void closeCatalog(Catalog retiredCatalog) { + if (retiredCatalog == null) { + return; + } try { if (retiredCatalog instanceof AutoCloseable) { ((AutoCloseable) retiredCatalog).close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 74a3b366f87721..6cbb51ab62fdbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -105,6 +105,7 @@ public class IcebergMetadataOps implements ExternalMetadataOps { protected ExternalCatalog dorisCatalog; protected SupportsNamespaces nsCatalog; private ExecutionAuthenticator executionAuthenticator; + private final ThreadPoolExecutor threadPoolWithPreAuth; // Generally, there should be only two levels under the catalog, namely ., // but the REST type catalog is obtained from an external server, // and the level provided by the external server may be three levels, ..
. @@ -117,6 +118,7 @@ public IcebergMetadataOps(ExternalCatalog dorisCatalog, Catalog catalog) { this.catalog = catalog; nsCatalog = (SupportsNamespaces) catalog; this.executionAuthenticator = dorisCatalog.getExecutionAuthenticator(); + this.threadPoolWithPreAuth = dorisCatalog.getThreadPoolWithPreAuth(); if (dorisCatalog.getProperties().containsKey(IcebergExternalCatalog.EXTERNAL_CATALOG_NAME)) { externalCatalogName = @@ -1928,7 +1930,7 @@ private boolean isViewCatalogEnabled() { } public ThreadPoolExecutor getThreadPoolWithPreAuth() { - return dorisCatalog.getThreadPoolWithPreAuth(); + return threadPoolWithPreAuth; } private void performDropView(String remoteDbName, String remoteViewName) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 9828370864fa5b..518301c4753e4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -158,6 +158,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -1152,6 +1153,10 @@ public static Table getWritableIcebergTable(ExternalTable dorisTable, IcebergMet return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable, expectedOps); } + public static ThreadPoolExecutor getIcebergTableExecutor(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getIcebergTableExecutor(dorisTable); + } + /** The action must return derived metadata rather than retain the supplied table. */ static T withIcebergTable(ExternalTable dorisTable, Function action) { return icebergExternalMetaCache(dorisTable).withIcebergTable(dorisTable, action); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 2a45a1b749f794..da1e772148cf3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -160,6 +160,7 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Supplier; @@ -174,6 +175,7 @@ public class IcebergScanNode extends FileQueryScanNode { private IcebergSource source; private Table icebergTable; + private ThreadPoolExecutor planningExecutor; private List pushdownIcebergPredicates = Lists.newArrayList(); // If tableLevelPushDownCount is true, means we can do count push down opt at table level. // which means all splits have no position/equality delete files, @@ -296,6 +298,7 @@ protected void doInitialize() throws UserException { getRelationSnapshot(); icebergTable = source.getIcebergTable(); icebergTable = useFrozenTableGeneration(icebergTable); + planningExecutor = getPlanningExecutor(); partitionMapInfos = new HashMap<>(); initializePartitionMetadata(); isPartitionedTable = icebergTable.spec().isPartitioned(); @@ -1754,7 +1757,7 @@ public TableScan createTableScan() throws UserException { this.pushdownIcebergPredicates.add(predicate.toString()); } - icebergTableScan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); + icebergTableScan = scan.planWith(planningExecutor); return icebergTableScan; } @@ -1799,6 +1802,17 @@ private Table useFrozenTableGeneration(Table currentTable) { return currentTable; } + private ThreadPoolExecutor getPlanningExecutor() { + TableIf targetTable = source.getTargetTable(); + if (targetTable instanceof IcebergSysExternalTable) { + targetTable = ((IcebergSysExternalTable) targetTable).getSourceTable(); + } + if (targetTable instanceof IcebergExternalTable) { + return IcebergUtils.getIcebergTableExecutor((IcebergExternalTable) targetTable); + } + return source.getCatalog().getThreadPoolWithPreAuth(); + } + @VisibleForTesting Schema getSystemTableProjectedSchema(List expressions, boolean caseSensitive) throws UserException { @@ -2701,7 +2715,7 @@ private List doGetPositionDeletesSystemTableSplits() throws UserException } long startTime = System.currentTimeMillis(); - scan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); + scan = scan.planWith(planningExecutor); BatchScan plannedScan = scan; try { positionDeleteTasks = getOrPlanPositionDeleteTasks(plannedScan, () -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java index 0c400950c58052..3fc06a23220993 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java @@ -36,10 +36,19 @@ public void call() { @Override public void close() { - connectContext.clear(); - ConnectContext.remove(); - if (previousContext != null) { - previousContext.setThreadLocalInfo(); + try { + if (connectContext.getStatementContext() != null) { + connectContext.getStatementContext().close(); + } + } finally { + try { + connectContext.clear(); + } finally { + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 90e0f6e8a62eef..5db854fe642561 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -773,8 +773,17 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException LOG.warn("Process one query failed because unknown reason: ", e); ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, "Unexpected exception: " + e.getMessage()); } - // no matter the master execute success or fail, the master must transfer the result to follower - // and tell the follower the current journalID. + try { + return buildProxyResult(request, executor); + } finally { + if (ctx.getStatementContext() != null) { + ctx.getStatementContext().close(); + } + } + } + + private TMasterOpResult buildProxyResult(TMasterOpRequest request, StmtExecutor executor) { + // No matter whether execution succeeds or fails, return the result and current journal ID to the follower. TMasterOpResult result = new TMasterOpResult(); if (ctx.queryId() != null // If none master FE not set query id or query id was reset in StmtExecutor diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 5bda3026fa6e49..a5285afc6c0d71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -198,7 +198,7 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare AuditLogHelper.updateMetrics(ctx); } } finally { - prepCtx.statementContext.clearExternalScanTasks(); + prepCtx.statementContext.close(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index bfb22b74f07426..865b4f213c1d5b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -64,4 +64,20 @@ void normalCompletionDoesNotStopFinishedAssignment() { Mockito.verify(assignment, Mockito.never()).stop(); Mockito.verify(lease).close(); } + + @Test + void statementCloseCancelsAcceptedTaskBeforeItStarts() { + SplitAssignment assignment = Mockito.mock(SplitAssignment.class); + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask( + () -> Assertions.fail("cancelled task must not run"), owner::finish); + owner.track(task); + + owner.close(); + + Assertions.assertTrue(task.isCancelled()); + Mockito.verify(assignment).stop(); + Mockito.verify(lease).close(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java index 0a5a4ab11d4621..9e6a02854497f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java @@ -79,7 +79,7 @@ public void testIsSupportedPartitionTable() { Mockito.when(icebergTable.specs()).thenReturn(specs); Assertions.assertFalse(spyTable.isValidRelatedTableCached()); - Assertions.assertFalse(spyTable.isValidRelatedTable()); + Assertions.assertFalse(spyTable.isValidRelatedTable(icebergTable)); Mockito.verify(icebergTable, Mockito.times(1)).specs(); Assertions.assertTrue(spyTable.isValidRelatedTableCached()); @@ -94,7 +94,7 @@ public void testIsSupportedPartitionTable() { List fields = Lists.newArrayList(); Mockito.when(spec.fields()).thenReturn(fields); - Assertions.assertFalse(spyTable.isValidRelatedTable()); + Assertions.assertFalse(spyTable.isValidRelatedTable(icebergTable)); Mockito.verify(spec, Mockito.times(1)).fields(); Assertions.assertTrue(spyTable.isValidRelatedTableCached()); Assertions.assertFalse(spyTable.validRelatedTableCache()); @@ -109,7 +109,7 @@ public void testIsSupportedPartitionTable() { fields.add(null); Mockito.when(spec.fields()).thenReturn(fields); - Assertions.assertFalse(spyTable.isValidRelatedTable()); + Assertions.assertFalse(spyTable.isValidRelatedTable(icebergTable)); Mockito.verify(spec, Mockito.times(2)).fields(); Assertions.assertTrue(spyTable.isValidRelatedTableCached()); Assertions.assertFalse(spyTable.validRelatedTableCache()); @@ -125,7 +125,7 @@ public void testIsSupportedPartitionTable() { Mockito.doReturn(mockTransform("hour")).when(field).transform(); Mockito.when(field.sourceId()).thenReturn(1); - Assertions.assertTrue(spyTable.isValidRelatedTable()); + Assertions.assertTrue(spyTable.isValidRelatedTable(icebergTable)); Assertions.assertTrue(spyTable.isValidRelatedTableCached()); Assertions.assertTrue(spyTable.validRelatedTableCache()); Mockito.verify(schema, Mockito.times(1)).findColumnName(ArgumentMatchers.anyInt()); @@ -134,13 +134,13 @@ public void testIsSupportedPartitionTable() { Mockito.when(field.sourceId()).thenReturn(1); spyTable.setIsValidRelatedTableCached(false); Assertions.assertFalse(spyTable.isValidRelatedTableCached()); - Assertions.assertTrue(spyTable.isValidRelatedTable()); + Assertions.assertTrue(spyTable.isValidRelatedTable(icebergTable)); Mockito.doReturn(mockTransform("month")).when(field).transform(); Mockito.when(field.sourceId()).thenReturn(1); spyTable.setIsValidRelatedTableCached(false); Assertions.assertFalse(spyTable.isValidRelatedTableCached()); - Assertions.assertTrue(spyTable.isValidRelatedTable()); + Assertions.assertTrue(spyTable.isValidRelatedTable(icebergTable)); Assertions.assertTrue(spyTable.isValidRelatedTableCached()); Assertions.assertTrue(spyTable.validRelatedTableCache()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java index a1f543433940f7..10c74149b2b0c9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -33,11 +33,28 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; class IcebergTableCacheValueTest { + @Test + void leaseKeepsExecutorFromItsTableGeneration() { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + try { + IcebergTableCacheValue value = new IcebergTableCacheValue( + newProxy(Table.class), executor, () -> null, () -> { }); + IcebergTableCacheValue.Lease lease = value.tryAcquire(); + Assertions.assertNotNull(lease); + Assertions.assertSame(executor, lease.getPlanningExecutor()); + lease.close(); + value.retire(); + } finally { + executor.shutdownNow(); + } + } + @Test void classifiesOnlyPerTableFileIOAsOwned() { FileIO tableIo = newProxy(FileIO.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java new file mode 100644 index 00000000000000..367ecbe0d65d38 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.nereids.StatementContext; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicBoolean; + +class AutoCloseConnectContextTest { + + @AfterEach + void tearDown() { + ConnectContext.remove(); + } + + @Test + void closeReleasesStatementResourcesAndRestoresPreviousContext() { + ConnectContext previous = new ConnectContext(); + previous.setThreadLocalInfo(); + ConnectContext current = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + current.setStatementContext(statementContext); + AtomicBoolean closed = new AtomicBoolean(); + statementContext.getOrRegisterStatementResource("resource", () -> () -> closed.set(true)); + + try (AutoCloseConnectContext ignored = new AutoCloseConnectContext(current)) { + Assertions.assertSame(current, ConnectContext.get()); + } + + Assertions.assertTrue(closed.get()); + Assertions.assertSame(previous, ConnectContext.get()); + } +} From 0b1a7100d1caf786e5ed136b72fe855ae20e1a6f Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 21 Aug 2026 17:47:46 +0800 Subject: [PATCH 09/38] [fix](iceberg) close remaining runtime lifecycle gaps --- .../datasource/hive/HMSExternalCatalog.java | 80 ++++++++++++- .../datasource/hudi/source/HudiScanNode.java | 76 +++++++------ .../IcebergCatalogResourceTracker.java | 18 +-- .../iceberg/source/IcebergScanNode.java | 8 ++ .../StaleMetaCacheEntryException.java | 25 ++++ .../java/org/apache/doris/mtmv/MTMVCache.java | 107 +++++++++--------- .../org/apache/doris/qe/ConnectProcessor.java | 4 + .../hudi/source/HudiBatchFsViewOwnerTest.java | 43 ++++++- .../iceberg/source/IcebergScanNodeTest.java | 25 ++++ .../doris/nereids/mv/MTMVCacheTest.java | 12 ++ 10 files changed, 297 insertions(+), 101 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 883a38780149d7..3c07afb52ab4f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -28,6 +28,9 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; +import org.apache.doris.datasource.hudi.HudiExternalMetaCache; +import org.apache.doris.datasource.iceberg.IcebergCatalogResourceTracker; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.operations.ExternalMetadataOperations; @@ -39,6 +42,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.iceberg.Table; import org.apache.iceberg.hive.HiveCatalog; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -73,6 +77,7 @@ public class HMSExternalCatalog extends ExternalCatalog { //for "type" = "hms" , but is iceberg table. private IcebergMetadataOps icebergMetadataOps; + private final IcebergCatalogResourceTracker icebergResourceTracker = new IcebergCatalogResourceTracker(); private volatile AbstractHiveProperties hmsProperties; @@ -155,7 +160,11 @@ protected void initLocalObjectsImpl() { } @Override - public void onClose() { + public synchronized void onClose() { + ThreadPoolExecutor retiredExecutor = threadPoolWithPreAuth; + threadPoolWithPreAuth = null; + IcebergMetadataOps retiredIcebergMetadataOps = icebergMetadataOps; + icebergMetadataOps = null; super.onClose(); if (null != fileSystemExecutor) { ThreadPoolManager.shutdownExecutorService(fileSystemExecutor); @@ -164,10 +173,14 @@ public void onClose() { metadataOps.close(); metadataOps = null; } - if (null != icebergMetadataOps) { - icebergMetadataOps.close(); - icebergMetadataOps = null; - } + icebergResourceTracker.retireCurrent(() -> { + if (retiredIcebergMetadataOps != null) { + retiredIcebergMetadataOps.close(); + } + if (retiredExecutor != null) { + ThreadPoolManager.shutdownExecutorService(retiredExecutor); + } + }); } @Override @@ -216,6 +229,14 @@ public void notifyPropertiesUpdated(Map updatedProps) { if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) { Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); } + if (updatedProps.keySet().stream() + .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) { + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); + } + if (updatedProps.keySet().stream() + .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) { + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + } } @Override @@ -227,7 +248,7 @@ public void setDefaultPropsIfMissing(boolean isReplay) { } } - public IcebergMetadataOps getIcebergMetadataOps() { + public synchronized IcebergMetadataOps getIcebergMetadataOps() { makeSureInitialized(); if (icebergMetadataOps == null) { HiveCatalog icebergHiveCatalog = IcebergUtils.createIcebergHiveCatalog(this, getName()); @@ -235,4 +256,51 @@ public IcebergMetadataOps getIcebergMetadataOps() { } return icebergMetadataOps; } + + /** Retains the exact HMS Iceberg runtime while a table cache generation is being loaded or borrowed. */ + public synchronized IcebergTableLoadContext beginIcebergTableLoad() { + makeSureInitialized(); + IcebergMetadataOps ops = getIcebergMetadataOps(); + return new IcebergTableLoadContext(ops, threadPoolWithPreAuth, icebergResourceTracker.beginLoad()); + } + + @Override + public synchronized void resetToUninitialized(boolean invalidCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + super.resetToUninitialized(invalidCache); + } + + public final class IcebergTableLoadContext implements AutoCloseable { + private final IcebergMetadataOps ops; + private final ThreadPoolExecutor executor; + private final IcebergCatalogResourceTracker.LoadGuard guard; + + private IcebergTableLoadContext(IcebergMetadataOps ops, ThreadPoolExecutor executor, + IcebergCatalogResourceTracker.LoadGuard guard) { + this.ops = ops; + this.executor = executor; + this.guard = guard; + } + + public IcebergMetadataOps getOps() { + return ops; + } + + public ThreadPoolExecutor getExecutor() { + return executor; + } + + public Table loadTable(String dbName, String tableName) throws Exception { + return executionAuthenticator.execute(() -> ops.loadTable(dbName, tableName)); + } + + public IcebergCatalogResourceTracker.ResourceLease promote() { + return guard.promote(); + } + + @Override + public void close() { + guard.close(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index c579cc36cabfa5..5f184aafe6492a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -93,7 +93,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; @@ -690,7 +689,10 @@ private TerminalTask terminalTask(Runnable task, Runnable taskFinished) { @VisibleForTesting static class TerminalTask extends FutureTask { private final AtomicBoolean started = new AtomicBoolean(); + private final AtomicBoolean interruptRequested = new AtomicBoolean(); private final Runnable taskFinished; + private volatile Thread runner; + private volatile Runnable ownerDone = () -> { }; TerminalTask(Runnable task, Runnable taskFinished) { super(task, null); @@ -700,7 +702,15 @@ static class TerminalTask extends FutureTask { @Override public void run() { if (started.compareAndSet(false, true)) { - super.run(); + runner = Thread.currentThread(); + if (interruptRequested.get()) { + runner.interrupt(); + } + try { + super.run(); + } finally { + runner = null; + } } } @@ -708,9 +718,28 @@ boolean cancelBeforeStart() { return started.compareAndSet(false, true) && cancel(false); } + void requestStop() { + if (cancelBeforeStart()) { + return; + } + interruptRequested.set(true); + Thread runningThread = runner; + if (runningThread != null) { + runningThread.interrupt(); + } + } + + void setOwnerDone(Runnable ownerDone) { + this.ownerDone = ownerDone; + } + @Override protected void done() { - taskFinished.run(); + try { + taskFinished.run(); + } finally { + ownerDone.run(); + } } } @@ -737,8 +766,6 @@ static class BatchFsViewOwner implements Closeable { private final SplitAssignment splitAssignment; private final HudiFsViewCacheValue.Lease lease; private final AtomicBoolean finished = new AtomicBoolean(); - private final AtomicReference finishFailure = new AtomicReference<>(); - private final CountDownLatch terminal = new CountDownLatch(1); private final ConcurrentLinkedQueue tasks = new ConcurrentLinkedQueue<>(); private final AtomicBoolean stopping = new AtomicBoolean(); @@ -752,54 +779,35 @@ void finish() { try { lease.close(); } catch (RuntimeException e) { - finishFailure.set(e); - throw e; - } finally { - terminal.countDown(); + LOG.warn("Failed to release Hudi fs-view lease after batch tasks terminated", e); } } } void track(TerminalTask task) { + task.setOwnerDone(() -> tasks.remove(task)); tasks.add(task); if (stopping.get()) { - task.cancelBeforeStart(); + task.requestStop(); } } @Override public void close() { - RuntimeException stopFailure = null; if (!finished.get()) { stopping.set(true); try { splitAssignment.stop(); } catch (RuntimeException e) { - stopFailure = e; - } - tasks.forEach(TerminalTask::cancelBeforeStart); - } - boolean interrupted = false; - while (true) { - try { - terminal.await(); - break; - } catch (InterruptedException e) { - interrupted = true; - } - } - if (interrupted) { - Thread.currentThread().interrupt(); - } - if (stopFailure != null) { - if (finishFailure.get() != null) { - stopFailure.addSuppressed(finishFailure.get()); + tasks.forEach(TerminalTask::requestStop); + throw e; } - throw stopFailure; - } - if (finishFailure.get() != null) { - throw finishFailure.get(); + tasks.forEach(TerminalTask::requestStop); } + // Already-started filesystem calls may be blocked in storage code that does not respond to + // interruption. Their TerminalTask.done callbacks retain exact task accounting and eventually call + // finish(), which releases the fs-view lease only after the last task exits. Cancellation must return + // promptly instead of waiting here and wedging statement/Arrow cleanup behind remote storage. } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java index c8c49443d4d047..8c63053807a181 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java @@ -20,21 +20,21 @@ import java.util.concurrent.atomic.AtomicBoolean; /** Keeps one catalog generation alive while tables loaded through it still have owners or borrowers. */ -final class IcebergCatalogResourceTracker { +public final class IcebergCatalogResourceTracker { private Generation current = new Generation(); - synchronized LoadGuard beginLoad() { + public synchronized LoadGuard beginLoad() { current.retain(); return new LoadGuard(current); } - synchronized void retireCurrent(Runnable cleanup) { + public synchronized void retireCurrent(Runnable cleanup) { Generation retired = current; current = new Generation(); retired.retire(cleanup); } - static final class LoadGuard implements AutoCloseable { + public static final class LoadGuard implements AutoCloseable { private final Generation generation; private final AtomicBoolean transferred = new AtomicBoolean(); @@ -42,7 +42,7 @@ private LoadGuard(Generation generation) { this.generation = generation; } - ResourceLease promote() { + public ResourceLease promote() { if (!transferred.compareAndSet(false, true)) { throw new IllegalStateException("Iceberg catalog load guard was already completed"); } @@ -57,7 +57,7 @@ public void close() { } } - static final class ResourceLease implements AutoCloseable { + public static final class ResourceLease implements AutoCloseable { private final Generation generation; private final AtomicBoolean closed = new AtomicBoolean(); @@ -106,7 +106,11 @@ private synchronized void release() { private void maybeCleanup() { if (retired && references == 0 && !cleaned) { cleaned = true; - cleanup.run(); + try { + cleanup.run(); + } finally { + cleanup = null; + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index da1e772148cf3b..d6921f9c9e2947 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -1803,6 +1803,14 @@ private Table useFrozenTableGeneration(Table currentTable) { } private ThreadPoolExecutor getPlanningExecutor() { + Optional snapshot = getPinnedRelationSnapshot(); + if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { + ThreadPoolExecutor frozenExecutor = ((IcebergMvccSnapshot) snapshot.get()) + .getSnapshotCacheValue().getPlanningExecutor(); + if (frozenExecutor != null) { + return frozenExecutor; + } + } TableIf targetTable = source.getTargetTable(); if (targetTable instanceof IcebergSysExternalTable) { targetTable = ((IcebergSysExternalTable) targetTable).getSourceTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java new file mode 100644 index 00000000000000..c7e19360e15cb2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +/** Signals that a caller raced catalog-group retirement and must resolve the current entry again. */ +public class StaleMetaCacheEntryException extends IllegalStateException { + public StaleMetaCacheEntryException(String message) { + super(message); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 391a9546500e2d..d9c022504d68bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -104,59 +104,64 @@ public static MTMVCache from(String defSql, boolean needCost, boolean needLock, ConnectContext currentContext, boolean addSessionVarGuard) throws AnalysisException { - StatementContext mvSqlStatementContext = new StatementContext(createCacheContext, - new OriginStatement(defSql, 0)); - if (!needLock) { - mvSqlStatementContext.setNeedLockTables(false); - } - if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) { - mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext); - } - createCacheContext.getStatementContext().setForceRecordTmpPlan(true); - mvSqlStatementContext.setForceRecordTmpPlan(true); - boolean originalRewriteFlag = createCacheContext.getSessionVariable().enableMaterializedViewRewrite; - createCacheContext.getSessionVariable().enableMaterializedViewRewrite = false; - LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(defSql); - NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); - try { - // Can not convert to table sink, because use the same column from different table when self join - // the out slot is wrong - if (needCost) { - // Only in mv rewrite, we need plan with eliminated cost which is used for mv chosen - planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); - } else { - // No need cost for performance - planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); + StatementContext originalStatementContext = createCacheContext.getStatementContext(); + try (StatementContext mvSqlStatementContext = new StatementContext(createCacheContext, + new OriginStatement(defSql, 0))) { + if (!needLock) { + mvSqlStatementContext.setNeedLockTables(false); } - CascadesContext cascadesContext = planner.getCascadesContext(); - Plan rewritePlan = cascadesContext.getRewritePlan(); + createCacheContext.setStatementContext(mvSqlStatementContext); + createCacheContext.getStatementContext().setForceRecordTmpPlan(true); + mvSqlStatementContext.setForceRecordTmpPlan(true); + boolean originalRewriteFlag = createCacheContext.getSessionVariable().enableMaterializedViewRewrite; + createCacheContext.getSessionVariable().enableMaterializedViewRewrite = false; + try { + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(defSql); + NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); + // Can not convert to table sink, because use the same column from different table when self join + // the out slot is wrong + if (needCost) { + // Only in mv rewrite, we need plan with eliminated cost which is used for mv chosen + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + } else { + // No need cost for performance + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); + } + CascadesContext cascadesContext = planner.getCascadesContext(); + Plan rewritePlan = cascadesContext.getRewritePlan(); - // Only add SessionVarGuardExpr if requested - Optional exprRewriter = addSessionVarGuard - ? Optional.of(new SessionVarGuardRewriter( - ConnectContextUtil.getAffectQueryResultInPlanVariables(createCacheContext), - cascadesContext)) - : Optional.empty(); - Plan addGuardRewritePlan = exprRewriter - .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, rewritePlan)) - .orElse(rewritePlan); - Pair finalPlanStructInfoPair = constructPlanAndStructInfo( - addGuardRewritePlan, cascadesContext); - List> tmpPlanUsedForRewrite = new ArrayList<>(); - for (Plan plan : cascadesContext.getStatementContext().getTmpPlanForMvRewrite()) { - Plan addGuardplan = exprRewriter - .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, plan)) - .orElse(plan); - tmpPlanUsedForRewrite.add(constructPlanAndStructInfo(addGuardplan, cascadesContext)); - } - return new MTMVCache(finalPlanStructInfoPair, addGuardRewritePlan, needCost - ? cascadesContext.getMemo().getRoot().getStatistics() : null, tmpPlanUsedForRewrite); - } finally { - createCacheContext.getStatementContext().setForceRecordTmpPlan(false); - mvSqlStatementContext.setForceRecordTmpPlan(false); - createCacheContext.getSessionVariable().enableMaterializedViewRewrite = originalRewriteFlag; - if (currentContext != null) { - currentContext.setThreadLocalInfo(); + // Only add SessionVarGuardExpr if requested + Optional exprRewriter = addSessionVarGuard + ? Optional.of(new SessionVarGuardRewriter( + ConnectContextUtil.getAffectQueryResultInPlanVariables(createCacheContext), + cascadesContext)) + : Optional.empty(); + Plan addGuardRewritePlan = exprRewriter + .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, rewritePlan)) + .orElse(rewritePlan); + Pair finalPlanStructInfoPair = constructPlanAndStructInfo( + addGuardRewritePlan, cascadesContext); + List> tmpPlanUsedForRewrite = new ArrayList<>(); + for (Plan plan : cascadesContext.getStatementContext().getTmpPlanForMvRewrite()) { + Plan addGuardplan = exprRewriter + .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, plan)) + .orElse(plan); + tmpPlanUsedForRewrite.add(constructPlanAndStructInfo(addGuardplan, cascadesContext)); + } + MTMVCache cache = new MTMVCache(finalPlanStructInfoPair, addGuardRewritePlan, needCost + ? cascadesContext.getMemo().getRoot().getStatistics() : null, tmpPlanUsedForRewrite); + return cache; + } finally { + // Runtime-bearing MVCC snapshots are planning scratch state, not part of the returned logical plans. + // Drop them on both success and failure before the temporary statement releases its runtime leases. + mvSqlStatementContext.resetMvccSnapshots(); + createCacheContext.getStatementContext().setForceRecordTmpPlan(false); + mvSqlStatementContext.setForceRecordTmpPlan(false); + createCacheContext.getSessionVariable().enableMaterializedViewRewrite = originalRewriteFlag; + createCacheContext.setStatementContext(originalStatementContext); + if (currentContext != null) { + currentContext.setThreadLocalInfo(); + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 5db854fe642561..49caaf0a2eb4e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -757,6 +757,10 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException } throw new RuntimeException("Prepare failed when proxy execute"); } + // Forwarded PREPARE and EXECUTE share the retained StatementContext, but they are distinct + // resource generations. Release any catalog/table leases acquired while analyzing PREPARE + // before ExecuteCommand opens the execution generation. + ctx.getStatementContext().detachStatementResources().close(); handleExecute(preparedStatementContext.command, Long.parseLong(preparedStmtId), preparedStatementContext, ByteBuffer.wrap(request.getPrepareExecuteBuffer()).order(ByteOrder.LITTLE_ENDIAN), queryId); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index 865b4f213c1d5b..6f753b8a455c8b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -32,7 +33,7 @@ class HudiBatchFsViewOwnerTest { @Test - void statementCloseStopsAndJoinsBeforeReleasingLease() throws Exception { + void statementCloseReturnsWhileRunningTaskKeepsLeasePinned() throws Exception { SplitAssignment assignment = Mockito.mock(SplitAssignment.class); HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); @@ -40,12 +41,11 @@ void statementCloseStopsAndJoinsBeforeReleasingLease() throws Exception { try { Future close = executor.submit(owner::close); Mockito.verify(assignment, Mockito.timeout(3000)).stop(); - Assertions.assertFalse(close.isDone()); + close.get(3, TimeUnit.SECONDS); Mockito.verify(lease, Mockito.never()).close(); owner.finish(); - close.get(3, TimeUnit.SECONDS); Mockito.verify(lease).close(); } finally { executor.shutdownNow(); @@ -80,4 +80,41 @@ void statementCloseCancelsAcceptedTaskBeforeItStarts() { Mockito.verify(assignment).stop(); Mockito.verify(lease).close(); } + + @Test + void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { + SplitAssignment assignment = Mockito.mock(SplitAssignment.class); + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask(() -> { + started.countDown(); + while (release.getCount() > 0) { + try { + release.await(3, TimeUnit.SECONDS); + } catch (InterruptedException e) { + interrupted.countDown(); + } + } + }, owner::finish); + owner.track(task); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + executor.execute(task); + Assertions.assertTrue(started.await(3, TimeUnit.SECONDS)); + + owner.close(); + + Mockito.verify(assignment).stop(); + Assertions.assertTrue(interrupted.await(3, TimeUnit.SECONDS)); + Mockito.verify(lease, Mockito.never()).close(); + release.countDown(); + Mockito.verify(lease, Mockito.timeout(3000)).close(); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 941f30c4dd7019..2525dd21ba6b67 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -129,6 +129,8 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -551,6 +553,29 @@ public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefres } } + @Test + public void testPlanningExecutorComesFromPinnedSnapshotGeneration() throws Exception { + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergSource(node, source); + + ThreadPoolExecutor frozenExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + Table frozenTable = Mockito.mock(Table.class); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, 1L), Optional.empty(), frozenTable, frozenExecutor)))); + try { + Method method = IcebergScanNode.class.getDeclaredMethod("getPlanningExecutor"); + method.setAccessible(true); + Assert.assertSame(frozenExecutor, method.invoke(node)); + } finally { + frozenExecutor.shutdownNow(); + } + } + private static class CountPlanningIcebergScanNode extends IcebergScanNode { private final TableScan tableScan; private final long snapshotCount; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java index 5cb57665d59d84..9aca076d7898fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java @@ -48,6 +48,18 @@ */ public class MTMVCacheTest extends SqlTestBase { + @Test + void testFailureRestoresCallerStatementContextAndSessionFlag() { + org.apache.doris.nereids.StatementContext original = connectContext.getStatementContext(); + connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + + Assertions.assertThrows(Exception.class, () -> MTMVCache.from( + "select from", connectContext, true, false, connectContext, false)); + + Assertions.assertSame(original, connectContext.getStatementContext()); + Assertions.assertTrue(connectContext.getSessionVariable().enableMaterializedViewRewrite); + } + @Test void testMTMVCacheIsCorrect() throws Exception { connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); From c99db1ae405c3faa9ac10a7449376fd334a9ea0d Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 21 Aug 2026 19:44:54 +0800 Subject: [PATCH 10/38] [fix](hive) fence scan and session teardown races --- .../datasource/hive/HMSExternalCatalog.java | 17 ++++++++++ .../datasource/hive/source/HiveScanNode.java | 29 ++++++++++++++-- .../datasource/hudi/source/HudiScanNode.java | 33 +++++++++++++++++-- .../org/apache/doris/qe/ConnectContext.java | 19 ++++++++++- .../org/apache/doris/qe/StmtExecutor.java | 9 ++++- .../sessions/FlightSqlConnectPoolMgr.java | 2 +- .../hive/source/HiveScanNodeTest.java | 4 +-- .../hudi/source/HudiScanNodeTest.java | 4 +-- .../apache/doris/qe/ConnectContextTest.java | 13 ++++++++ .../org/apache/doris/qe/StmtExecutorTest.java | 21 ++++++++++++ .../sessions/FlightSqlConnectPoolMgrTest.java | 4 +-- 11 files changed, 140 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 3c07afb52ab4f3..e343a02dfc7706 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicLong; /** * External catalog for hive metastore compatible data sources. @@ -80,6 +81,19 @@ public class HMSExternalCatalog extends ExternalCatalog { private final IcebergCatalogResourceTracker icebergResourceTracker = new IcebergCatalogResourceTracker(); private volatile AbstractHiveProperties hmsProperties; + private final AtomicLong runtimeGeneration = new AtomicLong(); + + public long getRuntimeGeneration() { + return runtimeGeneration.get(); + } + + @Override + public synchronized void modifyCatalogProps(Map props) { + // Fence scans before the mutable CatalogProperty is changed. super invokes resetToUninitialized while + // this monitor is still held, so no scan can capture the new properties with the old generation. + runtimeGeneration.incrementAndGet(); + super.modifyCatalogProps(props); + } /** * Lazily initializes HMSProperties from catalog properties. @@ -266,6 +280,9 @@ public synchronized IcebergTableLoadContext beginIcebergTableLoad() { @Override public synchronized void resetToUninitialized(boolean invalidCache) { + runtimeGeneration.incrementAndGet(); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); super.resetToUninitialized(invalidCache); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 1e45cbb3f7c2e9..d9d15c095a60e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -93,6 +93,7 @@ public class HiveScanNode extends FileQueryScanNode { private static final Logger LOG = LogManager.getLogger(HiveScanNode.class); protected final HMSExternalTable hmsTable; + private final long hmsRuntimeGeneration; private HiveTransaction hiveTransaction = null; // will only be set in Nereids, for lagency planner, it should be null @@ -125,6 +126,7 @@ public HiveScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, DirectoryLister directoryLister, ScanContext scanContext) { super(id, desc, planNodeName, statisticalType, scanContext, needCheckColumnPriv, sv); hmsTable = (HMSExternalTable) desc.getTable(); + hmsRuntimeGeneration = ((HMSExternalCatalog) hmsTable.getCatalog()).getRuntimeGeneration(); brokerName = hmsTable.getCatalog().bindBrokerName(); this.directoryLister = directoryLister; } @@ -136,6 +138,7 @@ public void setSelectedPartitions(SelectedPartitions selectedPartitions) { @Override protected void doInitialize() throws UserException { + ensureHmsRuntimeGeneration(); super.doInitialize(); if (hmsTable.isHiveTransactionalTable()) { @@ -145,6 +148,13 @@ protected void doInitialize() throws UserException { Env.getCurrentHiveTransactionMgr().register(hiveTransaction); skipCheckingAcidVersionFile = sessionVariable.skipCheckingAcidVersionFile; } + ensureHmsRuntimeGeneration(); + } + + private void ensureHmsRuntimeGeneration() { + if (((HMSExternalCatalog) hmsTable.getCatalog()).getRuntimeGeneration() != hmsRuntimeGeneration) { + throw new IllegalStateException("HMS catalog properties changed while planning the Hive scan; retry"); + } } static void markTransactionalHiveScanParams(TFileScanRangeParams scanParams) { @@ -156,6 +166,7 @@ static void markTransactionalHiveScanParams(TFileScanRangeParams scanParams) { } protected List getPartitions() throws AnalysisException { + ensureHmsRuntimeGeneration(); long startTime = System.currentTimeMillis(); List resPartitions = Lists.newArrayList(); try { @@ -194,6 +205,7 @@ protected List getPartitions() throws AnalysisException { getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); getSummaryProfile().setGetPartitionsFinishTime(); } + ensureHmsRuntimeGeneration(); return resPartitions; } catch (RuntimeException e) { if (getSummaryProfile() != null) { @@ -205,6 +217,7 @@ protected List getPartitions() throws AnalysisException { @Override public List getSplits(int numBackends) throws UserException { + ensureHmsRuntimeGeneration(); long start = System.currentTimeMillis(); try { if (!partitionInit) { @@ -224,6 +237,7 @@ public List getSplits(int numBackends) throws UserException { allFiles.size(), hmsTable.getDbName(), hmsTable.getName(), (System.currentTimeMillis() - start)); } + ensureHmsRuntimeGeneration(); return allFiles; } catch (Throwable t) { LOG.warn("get file split failed for table: {}", hmsTable.getName(), t); @@ -235,6 +249,7 @@ public List getSplits(int numBackends) throws UserException { @Override public void startSplit(int numBackends) { + ensureHmsRuntimeGeneration(); if (prunedPartitions.isEmpty()) { splitAssignment.finishSchedule(); return; @@ -254,6 +269,7 @@ public void startSplit(int numBackends) { splittersOnFlight.acquire(); CompletableFuture.runAsync(() -> { try { + ensureHmsRuntimeGeneration(); List allFiles = Lists.newArrayList(); getFileSplitByPartitions( cache, Collections.singletonList(partition), allFiles, bindBrokerName, @@ -262,6 +278,7 @@ public void startSplit(int numBackends) { numSplitsPerPartition.set(allFiles.size()); } if (splitAssignment.needMoreSplit()) { + ensureHmsRuntimeGeneration(); splitAssignment.addToQueue(allFiles); } } catch (Exception e) { @@ -338,7 +355,7 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List currentFileCaches = cache.getFilesByPartitions(partitions, true, partitions.size() > 1, directoryLister, hmsTable); HiveFileScanTaskCacheKey cacheKey = new HiveFileScanTaskCacheKey( - hmsTable.getCatalog().getId(), hmsTable.getId(), partitions, + hmsTable.getCatalog().getId(), hmsTable.getId(), hmsRuntimeGeneration, partitions, cache.getFileCacheInvalidationGeneration(hmsTable.getCatalog().getId()), currentFileCaches); try { fileCaches = getOrLoadExternalScanTasks(cacheKey, @@ -519,14 +536,17 @@ private static final class HiveFileScanTaskCacheKey implements ExternalScanTaskCacheKey { private final long catalogId; private final long tableId; + private final long hmsRuntimeGeneration; private final List partitions; private final long fileCacheInvalidationGeneration; private final List fileCacheValueGenerations; - private HiveFileScanTaskCacheKey(long catalogId, long tableId, List partitions, + private HiveFileScanTaskCacheKey(long catalogId, long tableId, long hmsRuntimeGeneration, + List partitions, long fileCacheInvalidationGeneration, List fileCaches) { this.catalogId = catalogId; this.tableId = tableId; + this.hmsRuntimeGeneration = hmsRuntimeGeneration; this.partitions = partitions.stream() .map(HivePartitionCacheKey::new) .collect(Collectors.toList()); @@ -547,6 +567,7 @@ public boolean equals(Object object) { HiveFileScanTaskCacheKey that = (HiveFileScanTaskCacheKey) object; return catalogId == that.catalogId && tableId == that.tableId + && hmsRuntimeGeneration == that.hmsRuntimeGeneration && fileCacheInvalidationGeneration == that.fileCacheInvalidationGeneration && fileCacheValueGenerations.equals(that.fileCacheValueGenerations) && partitions.equals(that.partitions); @@ -554,7 +575,7 @@ public boolean equals(Object object) { @Override public int hashCode() { - return Objects.hash(catalogId, tableId, partitions, fileCacheInvalidationGeneration, + return Objects.hash(catalogId, tableId, hmsRuntimeGeneration, partitions, fileCacheInvalidationGeneration, fileCacheValueGenerations); } } @@ -625,6 +646,7 @@ public TFileFormatType getFileFormatType() throws UserException { @Override protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { + ensureHmsRuntimeGeneration(); if (split instanceof HiveSplit) { HiveSplit hiveSplit = (HiveSplit) split; if (hiveSplit.isACID()) { @@ -687,6 +709,7 @@ protected List getDeleteFiles(TFileRangeDesc rangeDesc) { @Override protected Map getLocationProperties() { + ensureHmsRuntimeGeneration(); return hmsTable.getBackendStorageProperties(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 5f184aafe6492a..80deaa295f6817 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -37,6 +37,7 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SplitAssignment; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hive.source.HiveScanNode; import org.apache.doris.datasource.hudi.HudiFsViewCacheValue; @@ -119,6 +120,7 @@ public class HudiScanNode extends HiveScanNode { private List columnTypes; private List partitionColumnNames; private String storagePropertiesFingerprint; + private final long hmsRuntimeGeneration; private boolean partitionInit = false; private HoodieTimeline timeline; @@ -153,6 +155,7 @@ public HudiScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumn SessionVariable sv, DirectoryLister directoryLister, ScanContext scanContext) { super(id, desc, "HUDI_SCAN_NODE", StatisticalType.HUDI_SCAN_NODE, needCheckColumnPriv, sv, directoryLister, scanContext); + hmsRuntimeGeneration = ((HMSExternalCatalog) hmsTable.getCatalog()).getRuntimeGeneration(); isCowTable = hmsTable.isHoodieCowTable(); if (LOG.isDebugEnabled()) { if (isCowTable) { @@ -179,6 +182,7 @@ public TFileFormatType getFileFormatType() throws UserException { @Override protected void doInitialize() throws UserException { + ensureHmsRuntimeGeneration(); ExternalTable table = (ExternalTable) desc.getTable(); Optional relationSnapshot = getRelationSnapshot(); if (table.isView()) { @@ -256,10 +260,18 @@ protected void doInitialize() throws UserException { // and `the file column name`. // Split planning and FE-BE schema transport must describe the same pinned Hudi instant. ExternalUtil.initSchemaInfo(params, -1L, table.getFullSchema(relationSnapshot)); + ensureHmsRuntimeGeneration(); + } + + private void ensureHmsRuntimeGeneration() { + if (((HMSExternalCatalog) hmsTable.getCatalog()).getRuntimeGeneration() != hmsRuntimeGeneration) { + throw new IllegalStateException("HMS catalog properties changed while planning the Hudi scan; retry"); + } } @Override protected Map getLocationProperties() { + ensureHmsRuntimeGeneration(); if (incrementalRead) { return incrementalRelation.getHoodieParams(); } else { @@ -276,6 +288,7 @@ protected Map getLocationProperties() { @Override protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { + ensureHmsRuntimeGeneration(); if (split instanceof HudiSplit) { HudiSplit hudiSplit = (HudiSplit) split; if (rangeDesc.getFormatType() == TFileFormatType.FORMAT_JNI @@ -456,7 +469,7 @@ private void getPartitionSplits( List plannedSplits; if (useStatementCache) { HudiFileScanTaskCacheKey cacheKey = new HudiFileScanTaskCacheKey( - hmsTable.getCatalog().getId(), hmsTable.getId(), queryInstant, + hmsTable.getCatalog().getId(), hmsTable.getId(), hmsRuntimeGeneration, queryInstant, canUseNativeReader(), sessionVariable.isEnableRuntimeFilterPartitionPrune(), basePath, inputFormat, serdeLib, columnNames, columnTypes, partitionColumnNames, storagePropertiesFingerprint, partition); @@ -527,7 +540,9 @@ private void getPartitionsSplits(List partitions, List spl try { acceptedTasks.add(CompletableFuture.runAsync(() -> { try { + ensureHmsRuntimeGeneration(); getPartitionSplits(partition, splits); + ensureHmsRuntimeGeneration(); } catch (Throwable t) { throwable.compareAndSet(null, t); } @@ -553,6 +568,7 @@ private void getPartitionsSplits(List partitions, List spl @Override public List getSplits(int numBackends) throws UserException { + ensureHmsRuntimeGeneration(); acquireFsView(); try { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { @@ -564,6 +580,7 @@ public List getSplits(int numBackends) throws UserException { getPartitionsSplits(prunedPartitions, splits); return null; }); + ensureHmsRuntimeGeneration(); return splits; } catch (Exception e) { throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); @@ -573,6 +590,7 @@ public List getSplits(int numBackends) throws UserException { } private void initPrunedPartitions() throws UserException { + ensureHmsRuntimeGeneration(); if (partitionInit) { return; } @@ -587,10 +605,12 @@ private void initPrunedPartitions() throws UserException { throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); } partitionInit = true; + ensureHmsRuntimeGeneration(); } @Override public void startSplit(int numBackends) { + ensureHmsRuntimeGeneration(); if (prunedPartitions.isEmpty()) { splitAssignment.finishSchedule(); releaseFsViewOnce(); @@ -627,6 +647,7 @@ public void startSplit(int numBackends) { }; TerminalTask producerTask = terminalTask(() -> { try { + ensureHmsRuntimeGeneration(); for (HivePartition partition : prunedPartitions) { if (batchException.get() != null || splitAssignment.isStop()) { break; @@ -645,12 +666,14 @@ public void startSplit(int numBackends) { pendingTasks.incrementAndGet(); TerminalTask partitionTask = terminalTask(() -> { try { + ensureHmsRuntimeGeneration(); List allFiles = Lists.newArrayList(); getPartitionSplits(partition, allFiles, false); if (allFiles.size() > numSplitsPerPartition.get()) { numSplitsPerPartition.set(allFiles.size()); } if (splitAssignment.needMoreSplit()) { + ensureHmsRuntimeGeneration(); splitAssignment.addToQueue(allFiles); } } catch (Throwable t) { @@ -832,6 +855,7 @@ public int numApproximateSplits() { } private HudiSplit generateHudiSplit(FileSlice fileSlice, List partitionValues, String queryInstant) { + ensureHmsRuntimeGeneration(); Optional baseFile = fileSlice.getBaseFile().toJavaOptional(); String filePath = baseFile.map(BaseFile::getPath).orElse(""); long fileSize = baseFile.map(BaseFile::getFileSize).orElse(0L); @@ -893,6 +917,7 @@ private static final class HudiFileScanTaskCacheKey implements ExternalScanTaskCacheKey { private final long catalogId; private final long tableId; + private final long hmsRuntimeGeneration; private final String queryInstant; private final boolean nativeReader; private final boolean runtimePartitionPrune; @@ -908,12 +933,13 @@ private static final class HudiFileScanTaskCacheKey private final List partitionValues; private HudiFileScanTaskCacheKey( - long catalogId, long tableId, String queryInstant, boolean nativeReader, + long catalogId, long tableId, long hmsRuntimeGeneration, String queryInstant, boolean nativeReader, boolean runtimePartitionPrune, String basePath, String tableInputFormat, String serdeLib, List columnNames, List columnTypes, List partitionColumnNames, String storagePropertiesFingerprint, HivePartition partition) { this.catalogId = catalogId; this.tableId = tableId; + this.hmsRuntimeGeneration = hmsRuntimeGeneration; this.queryInstant = queryInstant; this.nativeReader = nativeReader; this.runtimePartitionPrune = runtimePartitionPrune; @@ -941,6 +967,7 @@ public boolean equals(Object object) { HudiFileScanTaskCacheKey that = (HudiFileScanTaskCacheKey) object; return catalogId == that.catalogId && tableId == that.tableId + && hmsRuntimeGeneration == that.hmsRuntimeGeneration && nativeReader == that.nativeReader && runtimePartitionPrune == that.runtimePartitionPrune && Objects.equals(queryInstant, that.queryInstant) @@ -959,7 +986,7 @@ public boolean equals(Object object) { @Override public int hashCode() { return Objects.hash( - catalogId, tableId, queryInstant, nativeReader, runtimePartitionPrune, + catalogId, tableId, hmsRuntimeGeneration, queryInstant, nativeReader, runtimePartitionPrune, basePath, tableInputFormat, serdeLib, columnNames, columnTypes, partitionColumnNames, storagePropertiesFingerprint, inputFormat, path, partitionValues); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 1ad07cae935c46..f75b7f997ecf2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -987,16 +987,33 @@ public PlSqlOperation getPlSqlOperation() { // with "Split source X is released". These executors are finalized when the next query starts // on this connection, or when the connection is torn down. See #62259. private final List flightSqlDeferredExecutors = new ArrayList<>(); + private boolean flightSqlDeferredExecutorsSealed; - public void addFlightSqlDeferredExecutor(StmtExecutor executor) { + public boolean addFlightSqlDeferredExecutor(StmtExecutor executor) { synchronized (flightSqlDeferredExecutors) { + if (flightSqlDeferredExecutorsSealed) { + return false; + } flightSqlDeferredExecutors.add(executor); + return true; } } public void closeFlightSqlDeferredExecutors() { + closeFlightSqlDeferredExecutors(false); + } + + /** Prevents a session teardown race from accepting an executor after the final drain. */ + public void sealAndCloseFlightSqlDeferredExecutors() { + closeFlightSqlDeferredExecutors(true); + } + + private void closeFlightSqlDeferredExecutors(boolean seal) { List toClose; synchronized (flightSqlDeferredExecutors) { + if (seal) { + flightSqlDeferredExecutorsSealed = true; + } if (flightSqlDeferredExecutors.isEmpty()) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index c0e9dad355fbc8..9abe85ea2f0f99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -978,8 +978,9 @@ void deferArrowFlightQuery() { Closeable resources = statementContext.detachStatementResources(); deferredArrowFlightStatementResources = resources; deferredForArrowFlight = true; + boolean registered; try { - context.addFlightSqlDeferredExecutor(this); + registered = context.addFlightSqlDeferredExecutor(this); } catch (RuntimeException | Error t) { deferredForArrowFlight = false; deferredArrowFlightStatementResources = null; @@ -990,6 +991,12 @@ void deferArrowFlightQuery() { } throw t; } + if (!registered) { + // Session teardown sealed and drained the registry between detachment and registration. Finalize + // directly: no later owner can reach this executor, and the deferred flag keeps the statement's + // ordinary finally block from closing the same coordinator a second time. + finalizeArrowFlightQuery(); + } } // Finalize an Arrow Flight query whose coordinator was kept alive across the diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java index c8854507e00114..9315ee9ef5cda5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java @@ -77,7 +77,7 @@ public void unregisterConnection(ConnectContext ctx) { // Finalize any Arrow Flight query whose coordinator was kept alive across the // GetFlightInfo -> DoGet phases (see #62259), releasing its resources (e.g. external-table // batch SplitSources and the query queue slot). - ctx.closeFlightSqlDeferredExecutors(); + ctx.sealAndCloseFlightSqlDeferredExecutors(); ctx.closeTxn(); if (connectionMap.remove(ctx.getConnectionId()) != null) { numberConnection.decrementAndGet(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java index a194e8f67c6080..d5dfb2d44cabc0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java @@ -556,9 +556,9 @@ private Object newHiveFileScanTaskCacheKey( HivePartition partition, HiveExternalMetaCache.FileCacheValue fileCacheValue) throws Exception { Class keyClass = Class.forName(HiveScanNode.class.getName() + "$HiveFileScanTaskCacheKey"); Constructor constructor = keyClass.getDeclaredConstructor( - long.class, long.class, List.class, long.class, List.class); + long.class, long.class, long.class, List.class, long.class, List.class); constructor.setAccessible(true); - return constructor.newInstance(1L, 2L, Collections.singletonList(partition), 0L, + return constructor.newInstance(1L, 2L, 3L, Collections.singletonList(partition), 0L, Collections.singletonList(fileCacheValue)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java index 418bb7787cdce0..f955d8ed8fa68e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java @@ -387,12 +387,12 @@ private static Object newPartitionCacheKey( throws Exception { Class keyClass = Class.forName(HudiScanNode.class.getName() + "$HudiFileScanTaskCacheKey"); Constructor constructor = keyClass.getDeclaredConstructor( - long.class, long.class, String.class, boolean.class, boolean.class, + long.class, long.class, long.class, String.class, boolean.class, boolean.class, String.class, String.class, String.class, List.class, List.class, List.class, String.class, HivePartition.class); constructor.setAccessible(true); return constructor.newInstance( - 1L, 2L, instant, nativeReader, runtimePrune, + 1L, 2L, 3L, instant, nativeReader, runtimePrune, "file:///table", "parquet", serdeLib, Collections.singletonList("id"), Collections.singletonList("int"), partitionColumnNames, storagePropertiesFingerprint, partition); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index b626f6acf8a92e..b0d789655abeb3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -929,4 +929,17 @@ public void testCloseFlightSqlDeferredExecutorsFinalizesRemainingWhenOneFails() Mockito.verify(failing, Mockito.times(1)).finalizeArrowFlightQuery(); Mockito.verify(healthy, Mockito.times(1)).finalizeArrowFlightQuery(); } + + @Test + public void testSessionTeardownRejectsLateDeferredExecutorRegistration() { + ConnectContext ctx = new ConnectContext(); + StmtExecutor late = Mockito.mock(StmtExecutor.class); + + ctx.sealAndCloseFlightSqlDeferredExecutors(); + + Assert.assertFalse("an executor detached after the final teardown drain must not become unreachable", + ctx.addFlightSqlDeferredExecutor(late)); + ctx.closeFlightSqlDeferredExecutors(); + Mockito.verifyNoInteractions(late); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index e82ff89a464825..57aba92754e2f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -128,6 +128,27 @@ public void testArrowFlightDefersStatementResourcesUntilDoGetCompletion() { Mockito.verify(coord).close(); } + @Test + public void testArrowFlightFinalizesImmediatelyWhenSessionTeardownSealedRegistration() { + StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); + StatementContext statementContext = connectContext.getStatementContext(); + AtomicInteger resourceCloseCount = new AtomicInteger(); + statementContext.getOrRegisterStatementResource("hudi-batch-owner", + () -> resourceCloseCount::incrementAndGet); + Coordinator coord = Mockito.mock(Coordinator.class); + Mockito.when(coord.getQueryOptions()).thenReturn(new TQueryOptions()); + stmtExecutor.setCoord(coord); + + connectContext.sealAndCloseFlightSqlDeferredExecutors(); + stmtExecutor.deferArrowFlightQuery(); + + Assert.assertEquals(1, resourceCloseCount.get()); + Mockito.verify(coord).close(); + statementContext.close(); + Assert.assertEquals("the ordinary statement cleanup must not double-close detached resources", + 1, resourceCloseCount.get()); + } + @Test public void testKill() throws Exception { StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java index 0513221569d9e7..5b02e297a884f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java @@ -43,7 +43,7 @@ public void testUnregisterConnectionFinalizesDeferredExecutors() { // The deferred coordinators must be released on teardown even though this connection was // never registered in the pool (an abandoned connection is still cleaned up, not leaked). Mockito.verify(channel).close(); - Mockito.verify(ctx).closeFlightSqlDeferredExecutors(); + Mockito.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); } // Cleanup must run before the connection bookkeeping (closeTxn / map removal), so that a failure @@ -63,7 +63,7 @@ public void testUnregisterRegisteredConnectionFinalizesDeferredExecutors() { poolMgr.unregisterConnection(ctx); Mockito.verify(channel).close(); - Mockito.verify(ctx).closeFlightSqlDeferredExecutors(); + Mockito.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); Assert.assertNull(poolMgr.getConnectionMap().get(7)); } } From ef15b9df06b99cd093e7d71302f3f0d4cc045607 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 21 Aug 2026 22:05:25 +0800 Subject: [PATCH 11/38] [fix](iceberg) close streaming and flight lifecycles --- .../insert/streaming/StreamingInsertTask.java | 18 ++++-- .../org/apache/doris/qe/ConnectContext.java | 58 +++++++++++++++++-- .../arrowflight/DorisFlightSqlProducer.java | 27 +++++++-- .../StreamingInsertTaskResourceTest.java | 57 ++++++++++++++++++ .../DorisFlightSqlProducerTest.java | 54 +++++++++++++++++ 5 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java index df23f10724049f..539dae8b63d090 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java @@ -182,15 +182,21 @@ public void cancel(boolean needWaitCancelComplete) { } @Override - public void closeOrReleaseResources() { - if (null != stmtExecutor) { + public synchronized void closeOrReleaseResources() { + ConnectContext taskContext = ctx; + try { + if (taskContext != null && taskContext.getStatementContext() != null) { + taskContext.getStatementContext().close(); + } + } finally { stmtExecutor = null; - } - if (null != taskCommand) { taskCommand = null; - } - if (null != ctx) { ctx = null; + // before() installs this attempt's context on the scheduler worker. Remove only that exact + // context: cancellation may invoke cleanup from a different thread while the worker is unwinding. + if (ConnectContext.get() == taskContext) { + ConnectContext.remove(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index f75b7f997ecf2a..8f1482b2cdad05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -988,6 +988,7 @@ public PlSqlOperation getPlSqlOperation() { // on this connection, or when the connection is torn down. See #62259. private final List flightSqlDeferredExecutors = new ArrayList<>(); private boolean flightSqlDeferredExecutorsSealed; + private int flightSqlResultPublishers; public boolean addFlightSqlDeferredExecutor(StmtExecutor executor) { synchronized (flightSqlDeferredExecutors) { @@ -999,6 +1000,33 @@ public boolean addFlightSqlDeferredExecutor(StmtExecutor executor) { } } + /** Linearizes GetFlightInfo publication with the terminal session seal. */ + public boolean canPublishFlightSqlResult() { + synchronized (flightSqlDeferredExecutors) { + return !flightSqlDeferredExecutorsSealed; + } + } + + public boolean beginFlightSqlResultPublication() { + synchronized (flightSqlDeferredExecutors) { + if (flightSqlDeferredExecutorsSealed) { + return false; + } + flightSqlResultPublishers++; + return true; + } + } + + public void endFlightSqlResultPublication() { + List toClose = null; + synchronized (flightSqlDeferredExecutors) { + if (--flightSqlResultPublishers == 0 && flightSqlDeferredExecutorsSealed) { + toClose = drainFlightSqlDeferredExecutors(); + } + } + finalizeFlightSqlDeferredExecutors(toClose); + } + public void closeFlightSqlDeferredExecutors() { closeFlightSqlDeferredExecutors(false); } @@ -1009,16 +1037,34 @@ public void sealAndCloseFlightSqlDeferredExecutors() { } private void closeFlightSqlDeferredExecutors(boolean seal) { - List toClose; + List toClose = null; synchronized (flightSqlDeferredExecutors) { if (seal) { flightSqlDeferredExecutorsSealed = true; + // An in-flight GetFlightInfo owns the coordinator until it either publishes its result or + // observes the seal and fails. Let its terminal path perform the drain so teardown cannot + // release the query resources while a successful ticket is still being constructed. + if (flightSqlResultPublishers != 0) { + return; + } } - if (flightSqlDeferredExecutors.isEmpty()) { - return; - } - toClose = new ArrayList<>(flightSqlDeferredExecutors); - flightSqlDeferredExecutors.clear(); + toClose = drainFlightSqlDeferredExecutors(); + } + finalizeFlightSqlDeferredExecutors(toClose); + } + + private List drainFlightSqlDeferredExecutors() { + if (flightSqlDeferredExecutors.isEmpty()) { + return null; + } + List toClose = new ArrayList<>(flightSqlDeferredExecutors); + flightSqlDeferredExecutors.clear(); + return toClose; + } + + private void finalizeFlightSqlDeferredExecutors(List toClose) { + if (toClose == null) { + return; } for (StmtExecutor deferredExecutor : toClose) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index e64690a54cb10f..5620593af71ae7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -186,9 +186,13 @@ public void closePreparedStatement(final ActionClosePreparedStatementRequest req private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext connectContext, String query, final FlightDescriptor descriptor) { + boolean resultPublisher = false; try { Preconditions.checkState(null != connectContext); Preconditions.checkState(!query.isEmpty()); + resultPublisher = connectContext.beginFlightSqlResultPublication(); + Preconditions.checkState(resultPublisher, + "Arrow Flight SQL session is already torn down"); // Finalize the previous query's coordinator on this connection whose close was // deferred (Arrow Flight keeps it alive across GetFlightInfo -> DoGet so the BE can // fetch external-table splits during DoGet). By now the previous DoGet is done. #62259 @@ -212,9 +216,9 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con final ByteString handle = ByteString.copyFromUtf8(peerIdentity + ":" + queryId); TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder() .setStatementHandle(handle).build(); - return getFlightInfoForSchema(ticketStatement, descriptor, + return publishFlightInfo(connectContext, getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel().getResult(queryId).getVectorSchemaRoot() - .getSchema()); + .getSchema())); } else { // A Flight Sql request can only contain one statement that returns result, // otherwise expected thrown exception during execution. @@ -228,9 +232,10 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con peerIdentity + ":" + DebugUtil.printId(connectContext.queryId())); TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder() .setStatementHandle(handle).build(); - return getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel() - .getResult(DebugUtil.printId(connectContext.queryId())).getVectorSchemaRoot() - .getSchema()); + return publishFlightInfo(connectContext, + getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel() + .getResult(DebugUtil.printId(connectContext.queryId())).getVectorSchemaRoot() + .getSchema())); } } else { // Now only query stmt will pull results from BE. @@ -280,7 +285,8 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con endpoints.add(new FlightEndpoint(ticket, location)); } // TODO Set in BE callback after query end, Client will not callback. - return new FlightInfo(flightSQLConnectProcessor.getArrowSchema(), descriptor, endpoints, -1, -1); + return publishFlightInfo(connectContext, + new FlightInfo(flightSQLConnectProcessor.getArrowSchema(), descriptor, endpoints, -1, -1)); } } } catch (Throwable e) { @@ -298,10 +304,19 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con LOG.error(errMsg, e); throw CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException(); } finally { + if (resultPublisher) { + connectContext.endFlightSqlResultPublication(); + } connectContext.setCommand(MysqlCommand.COM_SLEEP); } } + private FlightInfo publishFlightInfo(ConnectContext connectContext, FlightInfo flightInfo) { + Preconditions.checkState(connectContext.canPublishFlightSqlResult(), + "Arrow Flight SQL session was torn down before GetFlightInfo completed"); + return flightInfo; + } + @Override public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, final CallContext context, final FlightDescriptor descriptor) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java new file mode 100644 index 00000000000000..f72e5f144055fb --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.extensions.insert.streaming; + +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.qe.ConnectContext; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.util.Collections; + +class StreamingInsertTaskResourceTest { + + @AfterEach + void tearDown() { + ConnectContext.remove(); + } + + @Test + void closeReleasesExactAttemptStatementContextAndWorkerContext() throws Exception { + StreamingInsertTask task = new StreamingInsertTask( + 1L, 2L, "", null, "", null, Collections.emptyMap(), null, null); + ConnectContext taskContext = new ConnectContext(); + StatementContext statementContext = Mockito.mock(StatementContext.class); + taskContext.setStatementContext(statementContext); + taskContext.setThreadLocalInfo(); + Field contextField = StreamingInsertTask.class.getDeclaredField("ctx"); + contextField.setAccessible(true); + contextField.set(task, taskContext); + + task.closeOrReleaseResources(); + task.closeOrReleaseResources(); + + Mockito.verify(statementContext).close(); + Assertions.assertNull(task.getCtx()); + Assertions.assertNull(ConnectContext.get()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java index eede12688517be..e557ca3c2194a1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -31,6 +31,7 @@ import org.apache.arrow.flight.Result; import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest; import org.apache.arrow.flight.sql.impl.FlightSql.CommandStatementQuery; +import org.apache.arrow.vector.types.pojo.Schema; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -38,6 +39,7 @@ import org.mockito.MockedConstruction; import org.mockito.Mockito; +import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -207,4 +209,56 @@ public void testGetFlightInfoFinalizesDeferredExecutorWhenSchemaFetchFails() thr producer.close(); } } + + @Test + public void testGetFlightInfoFailsWhenTeardownDrainsRegisteredQueryBeforePublication() throws Exception { + assertTeardownPreventsFlightInfoPublication(true); + } + + @Test + public void testGetFlightInfoFailsWhenTeardownSealsBeforeQueryRegistration() throws Exception { + assertTeardownPreventsFlightInfoPublication(false); + } + + private void assertTeardownPreventsFlightInfoPublication(boolean registerBeforeSeal) throws Exception { + ConnectContext ctx = Mockito.spy(new ConnectContext()); + Mockito.doReturn(Mockito.mock(FlightSqlChannel.class)).when(ctx).getFlightSqlChannel(); + StmtExecutor deferred = Mockito.mock(StmtExecutor.class); + FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class); + Mockito.when(sessionsManager.getConnectContext(Mockito.anyString())).thenReturn(ctx); + CallContext callContext = Mockito.mock(CallContext.class); + Mockito.when(callContext.peerIdentity()).thenReturn("token"); + + DorisFlightSqlProducer producer = new DorisFlightSqlProducer( + Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager); + try (MockedConstruction mocked = Mockito.mockConstruction( + FlightSqlConnectProcessor.class, (mock, context) -> { + Mockito.doAnswer(invocation -> { + ctx.setReturnResultFromLocal(false); + if (registerBeforeSeal) { + Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(deferred)); + } + ctx.sealAndCloseFlightSqlDeferredExecutors(); + if (!registerBeforeSeal) { + Assert.assertFalse(ctx.addFlightSqlDeferredExecutor(deferred)); + } + return null; + }).when(mock).handleQuery(Mockito.anyString()); + Mockito.when(mock.getArrowSchema()).thenReturn(new Schema(Collections.emptyList())); + })) { + CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 1").build(); + FlightDescriptor descriptor = FlightDescriptor.command(new byte[0]); + + try { + producer.getFlightInfoStatement(request, callContext, descriptor); + Assert.fail("teardown must prevent publishing a ticket for finalized query resources"); + } catch (Throwable expected) { + Assert.assertTrue(expected.getMessage().contains("torn down")); + } + + Mockito.verify(deferred, Mockito.times(registerBeforeSeal ? 1 : 0)).finalizeArrowFlightQuery(); + } finally { + producer.close(); + } + } } From f42e0d0f77aec1f09770580eb1e6182b64cab5b5 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Sat, 22 Aug 2026 09:51:35 +0800 Subject: [PATCH 12/38] [fix](connector) Close remaining lifecycle race windows Keep streaming task cleanup on the execution owner, atomically commit Arrow Flight result publication, and bound background Iceberg snapshot loads to their exact catalog generation. --- .../streaming/AbstractStreamingTask.java | 75 +++++++++++++------ .../insert/streaming/StreamingInsertJob.java | 1 - .../insert/streaming/StreamingInsertTask.java | 22 ++++-- .../job/scheduler/StreamingTaskScheduler.java | 2 + .../org/apache/doris/qe/ConnectContext.java | 5 +- .../arrowflight/DorisFlightSqlProducer.java | 22 ++++-- .../iceberg/IcebergTableCacheValueTest.java | 17 +++++ .../StreamingInsertTaskResourceTest.java | 57 ++++++++++++++ .../DorisFlightSqlProducerTest.java | 13 ++++ 9 files changed, 175 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java index 224fe7c5dbb610..39adbc53820e31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java @@ -59,6 +59,9 @@ public abstract class AbstractStreamingTask { protected Long finishTimeMs; @Getter private AtomicBoolean isCanceled = new AtomicBoolean(false); + private final Object executionCompletion = new Object(); + private boolean executionStarted; + private boolean executionFinished; public AbstractStreamingTask(long jobId, long taskId, UserIdentity userIdentity) { this.jobId = jobId; @@ -94,34 +97,58 @@ public long getRunningBackendId() { } public void execute() throws JobException { - while (retryCount <= MAX_RETRY) { - try { - before(); - run(); - onSuccess(); - return; - } catch (Exception e) { - if (TaskStatus.CANCELED.equals(status)) { - return; - } - this.errMsg = e.getMessage(); - retryCount++; - if (noRetry || retryCount > MAX_RETRY) { - log.error("Task execution failed, job id {}, task id {}, noRetry {}, retry {}.", - jobId, taskId, noRetry, retryCount, e); - onFail(e.getMessage()); + synchronized (executionCompletion) { + executionStarted = true; + } + try { + while (retryCount <= MAX_RETRY) { + try { + before(); + run(); + onSuccess(); return; - } - log.warn("execute streaming task error, job id is {}, task id is {}, retrying {}/{}: {}", - jobId, taskId, retryCount, MAX_RETRY, e.getMessage()); - } finally { - // The cancel logic will call the closeOrReleased Resources method by itself. - // If it is also called here, - // it may result in the inability to obtain relevant information when canceling the task - if (!TaskStatus.CANCELED.equals(status)) { + } catch (Exception e) { + if (TaskStatus.CANCELED.equals(status)) { + return; + } + this.errMsg = e.getMessage(); + retryCount++; + if (noRetry || retryCount > MAX_RETRY) { + log.error("Task execution failed, job id {}, task id {}, noRetry {}, retry {}.", + jobId, taskId, noRetry, retryCount, e); + onFail(e.getMessage()); + return; + } + log.warn("execute streaming task error, job id is {}, task id is {}, retrying {}/{}: {}", + jobId, taskId, retryCount, MAX_RETRY, e.getMessage()); + } finally { + // Only the scheduler worker that created this attempt's ConnectContext may tear it down. + // A cancelling thread waits for this handoff instead of racing before() and clearing fields + // while planning is still publishing them. closeOrReleaseResources(); } } + } finally { + synchronized (executionCompletion) { + executionFinished = true; + executionCompletion.notifyAll(); + } + } + } + + protected void awaitExecutionCompletion() { + boolean interrupted = false; + synchronized (executionCompletion) { + while (executionStarted && !executionFinished) { + try { + executionCompletion.wait(); + } catch (InterruptedException e) { + interrupted = true; + } + } + } + if (interrupted) { + Thread.currentThread().interrupt(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java index 185aa720c4aaef..d61af07b0f772c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java @@ -815,7 +815,6 @@ public void clearRunningStreamTask(JobStatus newJobStatus) { log.info("clear running streaming insert task for job {}, task {}, status {} ", getJobId(), runningStreamTask.getTaskId(), runningStreamTask.getStatus()); runningStreamTask.cancel(JobStatus.STOPPED.equals(newJobStatus) ? false : true); - runningStreamTask.closeOrReleaseResources(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java index 539dae8b63d090..3581199969e5c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.thrift.TCell; @@ -54,7 +55,7 @@ @Getter public class StreamingInsertTask extends AbstractStreamingTask { private String sql; - private StmtExecutor stmtExecutor; + private volatile StmtExecutor stmtExecutor; private InsertIntoTableCommand taskCommand; private String currentDb; private ConnectContext ctx; @@ -173,20 +174,31 @@ protected void onFail(String errMsg) throws JobException { @Override public void cancel(boolean needWaitCancelComplete) { super.cancel(needWaitCancelComplete); - if (null != stmtExecutor) { + StmtExecutor executor = stmtExecutor; + if (null != executor) { log.info("cancelling streaming insert task, job id is {}, task id is {}", getJobId(), getTaskId()); - stmtExecutor.cancel(new Status(TStatusCode.CANCELLED, "streaming insert task cancelled"), + executor.cancel(new Status(TStatusCode.CANCELLED, "streaming insert task cancelled"), needWaitCancelComplete); } + if (needWaitCancelComplete) { + awaitExecutionCompletion(); + } } @Override public synchronized void closeOrReleaseResources() { ConnectContext taskContext = ctx; try { - if (taskContext != null && taskContext.getStatementContext() != null) { - taskContext.getStatementContext().close(); + if (taskContext != null) { + if (taskContext.queryId() != null) { + // Planning can register query-finish callbacks before a coordinator exists. Always run the + // registry teardown so Hive read transactions do not survive a failed/cancelled attempt. + QeProcessorImpl.INSTANCE.unregisterQuery(taskContext.queryId()); + } + if (taskContext.getStatementContext() != null) { + taskContext.getStatementContext().close(); + } } } finally { stmtExecutor = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java index 91d5fb1a658737..97cde56305fd84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java @@ -140,6 +140,8 @@ private void scheduleOneTask(AbstractStreamingTask task) { task.getTaskId(), task.getJobId(), System.currentTimeMillis() - start); } catch (Exception e) { log.error("Failed to execute task, task id: {}, job id: {}", task.getTaskId(), task.getJobId(), e); + } finally { + Env.getCurrentEnv().getJobManager().getStreamingTaskManager().removeRunningTask(task); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 8f1482b2cdad05..4814e6aa70ae37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -1017,14 +1017,17 @@ public boolean beginFlightSqlResultPublication() { } } - public void endFlightSqlResultPublication() { + public boolean endFlightSqlResultPublication() { List toClose = null; + boolean published; synchronized (flightSqlDeferredExecutors) { + published = !flightSqlDeferredExecutorsSealed; if (--flightSqlResultPublishers == 0 && flightSqlDeferredExecutorsSealed) { toClose = drainFlightSqlDeferredExecutors(); } } finalizeFlightSqlDeferredExecutors(toClose); + return published; } public void closeFlightSqlDeferredExecutors() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index 5620593af71ae7..e2763d748ffd55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -216,9 +216,11 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con final ByteString handle = ByteString.copyFromUtf8(peerIdentity + ":" + queryId); TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder() .setStatementHandle(handle).build(); - return publishFlightInfo(connectContext, getFlightInfoForSchema(ticketStatement, descriptor, + FlightInfo flightInfo = getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel().getResult(queryId).getVectorSchemaRoot() - .getSchema())); + .getSchema()); + resultPublisher = false; + return publishFlightInfo(connectContext, flightInfo); } else { // A Flight Sql request can only contain one statement that returns result, // otherwise expected thrown exception during execution. @@ -232,10 +234,12 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con peerIdentity + ":" + DebugUtil.printId(connectContext.queryId())); TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder() .setStatementHandle(handle).build(); - return publishFlightInfo(connectContext, - getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel() + FlightInfo flightInfo = getFlightInfoForSchema(ticketStatement, descriptor, + connectContext.getFlightSqlChannel() .getResult(DebugUtil.printId(connectContext.queryId())).getVectorSchemaRoot() - .getSchema())); + .getSchema()); + resultPublisher = false; + return publishFlightInfo(connectContext, flightInfo); } } else { // Now only query stmt will pull results from BE. @@ -285,8 +289,10 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con endpoints.add(new FlightEndpoint(ticket, location)); } // TODO Set in BE callback after query end, Client will not callback. - return publishFlightInfo(connectContext, - new FlightInfo(flightSQLConnectProcessor.getArrowSchema(), descriptor, endpoints, -1, -1)); + FlightInfo flightInfo = new FlightInfo( + flightSQLConnectProcessor.getArrowSchema(), descriptor, endpoints, -1, -1); + resultPublisher = false; + return publishFlightInfo(connectContext, flightInfo); } } } catch (Throwable e) { @@ -312,7 +318,7 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con } private FlightInfo publishFlightInfo(ConnectContext connectContext, FlightInfo flightInfo) { - Preconditions.checkState(connectContext.canPublishFlightSqlResult(), + Preconditions.checkState(connectContext.endFlightSqlResultPublication(), "Arrow Flight SQL session was torn down before GetFlightInfo completed"); return flightInfo; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java index 10c74149b2b0c9..d04fab7ce53c38 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -31,6 +31,7 @@ import java.lang.reflect.Proxy; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; @@ -55,6 +56,22 @@ void leaseKeepsExecutorFromItsTableGeneration() { } } + @Test + void backgroundSnapshotCopyDropsRuntimeGenerationOwners() { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + try { + IcebergSnapshotCacheValue runtimeValue = new IcebergSnapshotCacheValue( + null, null, Optional.empty(), newProxy(Table.class), executor); + + IcebergSnapshotCacheValue detached = runtimeValue.metadataOnlyCopy(); + + Assertions.assertFalse(detached.getIcebergTable().isPresent()); + Assertions.assertNull(detached.getPlanningExecutor()); + } finally { + executor.shutdownNow(); + } + } + @Test void classifiesOnlyPerTableFileIOAsOwned() { FileIO tableIo = newProxy(FileIO.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java index f72e5f144055fb..f3330b567070c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TUniqueId; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -27,6 +28,10 @@ import java.lang.reflect.Field; import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; class StreamingInsertTaskResourceTest { @@ -40,6 +45,8 @@ void closeReleasesExactAttemptStatementContextAndWorkerContext() throws Exceptio StreamingInsertTask task = new StreamingInsertTask( 1L, 2L, "", null, "", null, Collections.emptyMap(), null, null); ConnectContext taskContext = new ConnectContext(); + TUniqueId queryId = new TUniqueId(10L, 20L); + taskContext.setQueryId(queryId); StatementContext statementContext = Mockito.mock(StatementContext.class); taskContext.setStatementContext(statementContext); taskContext.setThreadLocalInfo(); @@ -54,4 +61,54 @@ void closeReleasesExactAttemptStatementContextAndWorkerContext() throws Exceptio Assertions.assertNull(task.getCtx()); Assertions.assertNull(ConnectContext.get()); } + + @Test + void cancellationLeavesAttemptCleanupToExecutionOwner() throws Exception { + CountDownLatch planningStarted = new CountDownLatch(1); + CountDownLatch finishPlanning = new CountDownLatch(1); + AtomicInteger closeCalls = new AtomicInteger(); + AtomicBoolean cleanupRanOnWorker = new AtomicBoolean(); + Thread[] workerRef = new Thread[1]; + AbstractStreamingTask task = new AbstractStreamingTask(1L, 2L, null) { + @Override + public void before() throws Exception { + planningStarted.countDown(); + finishPlanning.await(10, TimeUnit.SECONDS); + } + + @Override + public void run() { + } + + @Override + public boolean onSuccess() { + return false; + } + + @Override + public void closeOrReleaseResources() { + closeCalls.incrementAndGet(); + cleanupRanOnWorker.set(Thread.currentThread() == workerRef[0]); + } + }; + Thread worker = new Thread(() -> { + workerRef[0] = Thread.currentThread(); + try { + task.execute(); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + worker.start(); + Assertions.assertTrue(planningStarted.await(10, TimeUnit.SECONDS)); + + task.cancel(false); + Assertions.assertEquals(0, closeCalls.get()); + finishPlanning.countDown(); + worker.join(TimeUnit.SECONDS.toMillis(10)); + + Assertions.assertFalse(worker.isAlive()); + Assertions.assertEquals(1, closeCalls.get()); + Assertions.assertTrue(cleanupRanOnWorker.get()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java index e557ca3c2194a1..de281ba8f9f888 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -220,6 +220,19 @@ public void testGetFlightInfoFailsWhenTeardownSealsBeforeQueryRegistration() thr assertTeardownPreventsFlightInfoPublication(false); } + @Test + public void testPublicationCompletionAtomicallyObservesTerminalSeal() { + ConnectContext ctx = new ConnectContext(); + StmtExecutor deferred = Mockito.mock(StmtExecutor.class); + Assert.assertTrue(ctx.beginFlightSqlResultPublication()); + Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(deferred)); + + ctx.sealAndCloseFlightSqlDeferredExecutors(); + Assert.assertFalse("a publication in flight before teardown must not commit after the terminal seal", + ctx.endFlightSqlResultPublication()); + Mockito.verify(deferred).finalizeArrowFlightQuery(); + } + private void assertTeardownPreventsFlightInfoPublication(boolean registerBeforeSeal) throws Exception { ConnectContext ctx = Mockito.spy(new ConnectContext()); Mockito.doReturn(Mockito.mock(FlightSqlChannel.class)).when(ctx).getFlightSqlChannel(); From 1f5269503d379759def9b613c2bfe0168c0aac24 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Sat, 22 Aug 2026 17:05:51 +0800 Subject: [PATCH 13/38] fix: close remaining query lifecycle races --- .../datasource/hive/HMSExternalCatalog.java | 12 +- .../datasource/hudi/HudiFsViewCacheValue.java | 6 + .../datasource/hudi/source/HudiScanNode.java | 121 ++++++++++++++++-- .../iceberg/IcebergExternalCatalog.java | 8 +- .../streaming/AbstractStreamingTask.java | 50 +++++--- .../insert/streaming/StreamingInsertJob.java | 21 ++- .../insert/streaming/StreamingInsertTask.java | 20 ++- .../doris/plsql/executor/DorisRowResult.java | 49 ++++++- .../plsql/executor/PlsqlQueryExecutor.java | 22 +++- .../sessions/FlightSqlConnectPoolMgr.java | 4 +- .../hudi/HudiFsViewCacheValueTest.java | 5 - .../hudi/source/HudiBatchFsViewOwnerTest.java | 49 +++++++ ...reamingInsertJobOffsetPersistenceTest.java | 32 +++++ .../StreamingInsertTaskResourceTest.java | 81 ++++++++++++ .../plsql/executor/DorisRowResultTest.java | 69 ++++++++++ .../sessions/FlightSqlConnectPoolMgrTest.java | 7 + 16 files changed, 506 insertions(+), 50 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index e343a02dfc7706..97ea317c383030 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -25,6 +25,7 @@ import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; @@ -280,10 +281,15 @@ public synchronized IcebergTableLoadContext beginIcebergTableLoad() { @Override public synchronized void resetToUninitialized(boolean invalidCache) { + ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + cacheMgr.runCatalogLifecycle(getId(), () -> resetCatalogRuntime(cacheMgr, invalidCache)); + } + + private void resetCatalogRuntime(ExternalMetaCacheMgr cacheMgr, boolean invalidCache) { runtimeGeneration.incrementAndGet(); - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + cacheMgr.removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); + cacheMgr.removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); + cacheMgr.removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); super.resetToUninitialized(invalidCache); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java index 3f3b4feead6085..664f5417ccfd11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java @@ -62,6 +62,12 @@ public synchronized Lease tryAcquire() { public synchronized void evict() { evicted = true; + // A value rejected before getFsView() returns still owns the loader's transferable reference. + // Consume it here so a sealed/generation-lost load can reach zero without a nonexistent caller. + if (loaderReferenceAvailable) { + loaderReferenceAvailable = false; + refCount--; + } maybeClose(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 80deaa295f6817..25d3ca9a21d683 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -91,6 +91,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; @@ -138,6 +139,7 @@ public class HudiScanNode extends HiveScanNode { private HudiFsViewCacheValue.Lease fsViewLease; private final AtomicBoolean fsViewReleased = new AtomicBoolean(false); private final Object batchFsViewResourceKey = new Object(); + private final Object listingFsViewResourceKey = new Object(); // The schema information involved in the current query process (including historical schema). protected ConcurrentHashMap currentQuerySchema = new ConcurrentHashMap<>(); @@ -532,29 +534,50 @@ private List planPartitionSplits(HivePartition partition) throws IOEx private void getPartitionsSplits(List partitions, List splits) { Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); - List> acceptedTasks = new ArrayList<>(partitions.size()); + ListingFsViewOwner createdOwner = new ListingFsViewOwner(fsViewLease); + ListingFsViewOwner owner = createdOwner; + ConnectContext connectContext = ConnectContext.get(); + StatementContext statementContext = connectContext == null ? null : connectContext.getStatementContext(); + if (statementContext != null) { + try { + owner = statementContext.getOrRegisterStatementResource(listingFsViewResourceKey, () -> createdOwner); + if (owner != createdOwner) { + throw new IllegalStateException("Hudi listing owner was registered twice"); + } + } catch (RuntimeException e) { + createdOwner.discardBeforeSubmission(); + throw e; + } + } + // The owner now releases the exact fs-view generation after every accepted task terminates. + if (!fsViewReleased.compareAndSet(false, true)) { + owner.discardBeforeSubmission(); + throw new IllegalStateException("Hudi filesystem-view lease has already been released"); + } AtomicReference throwable = new AtomicReference<>(); RuntimeException submissionFailure = null; long startTime = System.currentTimeMillis(); for (HivePartition partition : partitions) { + TerminalTask task = terminalTask(() -> { + try { + ensureHmsRuntimeGeneration(); + getPartitionSplits(partition, splits); + ensureHmsRuntimeGeneration(); + } catch (Throwable t) { + throwable.compareAndSet(null, t); + } + }, () -> { }); + owner.track(task); try { - acceptedTasks.add(CompletableFuture.runAsync(() -> { - try { - ensureHmsRuntimeGeneration(); - getPartitionSplits(partition, splits); - ensureHmsRuntimeGeneration(); - } catch (Throwable t) { - throwable.compareAndSet(null, t); - } - }, executor)); + executor.execute(task); } catch (RuntimeException e) { submissionFailure = e; + task.cancelBeforeStart(); break; } } - // CompletableFuture.allOf has no Phaser party limit and join is uninterruptible: every accepted task is - // terminal before the caller releases the filesystem-view lease, including submission rejection. - CompletableFuture.allOf(acceptedTasks.toArray(new CompletableFuture[0])).join(); + owner.submissionDone(); + owner.awaitCompletion(); if (submissionFailure != null) { throw submissionFailure; } @@ -834,6 +857,78 @@ public void close() { } } + @VisibleForTesting + static class ListingFsViewOwner implements Closeable { + private final HudiFsViewCacheValue.Lease lease; + private final AtomicInteger pendingTasks = new AtomicInteger(1); + private final AtomicBoolean submissionFinished = new AtomicBoolean(); + private final AtomicBoolean stopping = new AtomicBoolean(); + private final ConcurrentLinkedQueue tasks = new ConcurrentLinkedQueue<>(); + private final CompletableFuture tasksFinished = new CompletableFuture<>(); + private final CompletableFuture cancelled = new CompletableFuture<>(); + + ListingFsViewOwner(HudiFsViewCacheValue.Lease lease) { + this.lease = lease; + } + + void track(TerminalTask task) { + pendingTasks.incrementAndGet(); + task.setOwnerDone(() -> { + tasks.remove(task); + taskDone(); + }); + tasks.add(task); + if (stopping.get()) { + task.requestStop(); + } + } + + void submissionDone() { + if (submissionFinished.compareAndSet(false, true)) { + taskDone(); + } + } + + void discardBeforeSubmission() { + close(); + submissionDone(); + } + + private void taskDone() { + if (pendingTasks.decrementAndGet() == 0) { + try { + lease.close(); + tasksFinished.complete(null); + } catch (RuntimeException e) { + tasksFinished.completeExceptionally(e); + } + } + } + + void awaitCompletion() { + try { + CompletableFuture.anyOf(tasksFinished, cancelled).get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + close(); + throw new CancellationException("Hudi split listing was interrupted"); + } catch (java.util.concurrent.ExecutionException e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + if (cancelled.isDone() && !tasksFinished.isDone()) { + throw new CancellationException("Hudi split listing was cancelled"); + } + } + + @Override + public void close() { + if (stopping.compareAndSet(false, true)) { + tasks.forEach(TerminalTask::requestStop); + cancelled.complete(null); + } + } + } + @Override public boolean isBatchMode() { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 5260bb0bbee028..da6dabc7892f41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -26,6 +26,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; @@ -199,7 +200,12 @@ public synchronized void onClose() { @Override public synchronized void resetToUninitialized(boolean invalidCache) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + cacheMgr.runCatalogLifecycle(getId(), () -> resetCatalogRuntime(cacheMgr, invalidCache)); + } + + private void resetCatalogRuntime(ExternalMetaCacheMgr cacheMgr, boolean invalidCache) { + cacheMgr.removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); super.resetToUninitialized(invalidCache); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java index 39adbc53820e31..3ff579e28901fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java @@ -62,6 +62,7 @@ public abstract class AbstractStreamingTask { private final Object executionCompletion = new Object(); private boolean executionStarted; private boolean executionFinished; + private Thread executionOwner; public AbstractStreamingTask(long jobId, long taskId, UserIdentity userIdentity) { this.jobId = jobId; @@ -99,38 +100,52 @@ public long getRunningBackendId() { public void execute() throws JobException { synchronized (executionCompletion) { executionStarted = true; + executionOwner = Thread.currentThread(); } try { while (retryCount <= MAX_RETRY) { + Exception attemptFailure = null; try { before(); run(); - onSuccess(); - return; } catch (Exception e) { - if (TaskStatus.CANCELED.equals(status)) { - return; - } - this.errMsg = e.getMessage(); - retryCount++; - if (noRetry || retryCount > MAX_RETRY) { - log.error("Task execution failed, job id {}, task id {}, noRetry {}, retry {}.", - jobId, taskId, noRetry, retryCount, e); - onFail(e.getMessage()); - return; - } - log.warn("execute streaming task error, job id is {}, task id is {}, retrying {}/{}: {}", - jobId, taskId, retryCount, MAX_RETRY, e.getMessage()); + attemptFailure = e; } finally { // Only the scheduler worker that created this attempt's ConnectContext may tear it down. // A cancelling thread waits for this handoff instead of racing before() and clearing fields // while planning is still publishing them. - closeOrReleaseResources(); + try { + closeOrReleaseResources(); + } catch (RuntimeException cleanupFailure) { + if (attemptFailure == null) { + attemptFailure = cleanupFailure; + } else { + attemptFailure.addSuppressed(cleanupFailure); + } + } } + if (attemptFailure == null) { + onSuccess(); + return; + } + if (TaskStatus.CANCELED.equals(status)) { + return; + } + this.errMsg = attemptFailure.getMessage(); + retryCount++; + if (noRetry || retryCount > MAX_RETRY) { + log.error("Task execution failed, job id {}, task id {}, noRetry {}, retry {}.", + jobId, taskId, noRetry, retryCount, attemptFailure); + onFail(attemptFailure.getMessage()); + return; + } + log.warn("execute streaming task error, job id is {}, task id is {}, retrying {}/{}: {}", + jobId, taskId, retryCount, MAX_RETRY, attemptFailure.getMessage()); } } finally { synchronized (executionCompletion) { executionFinished = true; + executionOwner = null; executionCompletion.notifyAll(); } } @@ -139,6 +154,9 @@ public void execute() throws JobException { protected void awaitExecutionCompletion() { boolean interrupted = false; synchronized (executionCompletion) { + if (Thread.currentThread() == executionOwner) { + return; + } while (executionStarted && !executionFinished) { try { executionCompletion.wait(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java index d61af07b0f772c..4895355235f4c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java @@ -553,19 +553,33 @@ public void alterJob(AlterJobCommand alterJobCommand) throws AnalysisException, @Override public void updateJobStatus(JobStatus status) throws JobException { + AbstractStreamingTask taskToCancel = null; + boolean waitForTask = JobStatus.PAUSED.equals(status); lock.writeLock().lock(); try { - super.updateJobStatus(status); - if (JobStatus.PAUSED.equals(getJobStatus())) { - clearRunningStreamTask(status); + if ((JobStatus.PAUSED.equals(status) || JobStatus.STOPPED.equals(status)) + && status != getJobStatus()) { + taskToCancel = runningStreamTask; + runningStreamTask = null; } + super.updateJobStatus(status); if (isFinalStatus()) { Env.getCurrentGlobalTransactionMgr().getCallbackFactory().removeCallback(getJobId()); } log.info("Streaming insert job {} update status to {}", getJobId(), getJobStatus()); + } catch (RuntimeException | JobException e) { + if (taskToCancel != null) { + runningStreamTask = taskToCancel; + } + throw e; } finally { lock.writeLock().unlock(); } + if (taskToCancel != null) { + // The task owner can need this job's write lock while finishing transaction callbacks. + // Cancel and wait only after publishing the status and releasing the job lock. + taskToCancel.cancel(waitForTask); + } } public void resetFailureInfo(FailureReason reason) { @@ -815,6 +829,7 @@ public void clearRunningStreamTask(JobStatus newJobStatus) { log.info("clear running streaming insert task for job {}, task {}, status {} ", getJobId(), runningStreamTask.getTaskId(), runningStreamTask.getStatus()); runningStreamTask.cancel(JobStatus.STOPPED.equals(newJobStatus) ? false : true); + runningStreamTask = null; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java index 3581199969e5c2..e99e77ef581a2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java @@ -189,15 +189,28 @@ public void cancel(boolean needWaitCancelComplete) { @Override public synchronized void closeOrReleaseResources() { ConnectContext taskContext = ctx; + RuntimeException cleanupFailure = null; try { if (taskContext != null) { if (taskContext.queryId() != null) { // Planning can register query-finish callbacks before a coordinator exists. Always run the // registry teardown so Hive read transactions do not survive a failed/cancelled attempt. - QeProcessorImpl.INSTANCE.unregisterQuery(taskContext.queryId()); + try { + QeProcessorImpl.INSTANCE.unregisterQuery(taskContext.queryId()); + } catch (RuntimeException e) { + cleanupFailure = e; + } } if (taskContext.getStatementContext() != null) { - taskContext.getStatementContext().close(); + try { + taskContext.getStatementContext().close(); + } catch (RuntimeException e) { + if (cleanupFailure == null) { + cleanupFailure = e; + } else { + cleanupFailure.addSuppressed(e); + } + } } } } finally { @@ -210,6 +223,9 @@ public synchronized void closeOrReleaseResources() { ConnectContext.remove(); } } + if (cleanupFailure != null) { + throw cleanupFailure; + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java index e286485fd42e0b..6ab547708a1364 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java @@ -26,6 +26,8 @@ import org.apache.doris.qe.RowBatch; import org.apache.doris.statistics.util.InternalQueryBuffer; +import java.io.Closeable; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; @@ -47,19 +49,31 @@ public class DorisRowResult implements RowResult { private boolean eof; private Object[] current; + private Closeable statementResources; public DorisRowResult(Coordinator coord, List columnNames, List dorisTypes) { + this(coord, columnNames, dorisTypes, null); + } + + public DorisRowResult(Coordinator coord, List columnNames, List dorisTypes, + Closeable statementResources) { this.coord = coord; this.columnNames = columnNames; this.dorisTypes = dorisTypes; this.current = columnNames != null ? new Object[columnNames.size()] : null; this.isLazyLoading = false; this.eof = false; + this.statementResources = statementResources; } @Override public boolean next() { - if (eof || coord == null) { + if (eof) { + return false; + } + if (coord == null) { + eof = true; + close(); return false; } try { @@ -69,6 +83,7 @@ public boolean next() { index = 0; if (batch.isEos()) { eof = true; + close(); return false; } } else { @@ -76,6 +91,11 @@ public boolean next() { } isLazyLoading = true; } catch (Exception e) { + try { + close(); + } catch (RuntimeException closeFailure) { + e.addSuppressed(closeFailure); + } throw new QueryException(e); } return true; @@ -83,7 +103,32 @@ public boolean next() { @Override public void close() { - // TODO + RuntimeException failure = null; + if (coord != null) { + try { + coord.close(); + } catch (RuntimeException e) { + failure = e; + } finally { + coord = null; + } + } + if (statementResources != null) { + try { + statementResources.close(); + } catch (IOException | RuntimeException e) { + if (failure == null) { + failure = new RuntimeException("Failed to close PLSQL statement resources", e); + } else { + failure.addSuppressed(e); + } + } finally { + statementResources = null; + } + } + if (failure != null) { + throw failure; + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java index 37a8cf310a1d95..e4f5b2c6942c65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java @@ -29,6 +29,7 @@ import org.antlr.v4.runtime.ParserRuleContext; +import java.io.Closeable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -42,21 +43,34 @@ public QueryResult executeQuery(String sql, ParserRuleContext ctx) { // A cursor may correspond to a query, and if the user opens multiple cursors, need to save multiple // query states, so here each query constructs a ConnectProcessor and the ConnectContext shares some data. ConnectContext context = ConnectContext.get().cloneContext(); + Closeable statementResources = null; try (AutoCloseConnectContext autoCloseCtx = new AutoCloseConnectContext(context)) { autoCloseCtx.call(); context.setRunProcedure(true); ConnectProcessor processor = new MysqlConnectProcessor(context); processor.executeQuery(sql); StmtExecutor executor = context.getExecutor(); + statementResources = context.getStatementContext().detachStatementResources(); + DorisRowResult rowResult; if (executor.getParsedStmt().getResultExprs() != null) { - return new QueryResult(new DorisRowResult(executor.getCoord(), executor.getColumns(), - executor.getReturnTypes()), () -> metadata(executor), processor, null); + rowResult = new DorisRowResult(executor.getCoord(), executor.getColumns(), + executor.getReturnTypes(), statementResources); + statementResources = null; + return new QueryResult(rowResult, () -> metadata(executor), processor, null); } else { // If ResultExpr is empty, not need to return result in plsql.Stmt.statement() - return new QueryResult(new DorisRowResult(executor.getCoord(), executor.getColumns(), null), - null, processor, null); + rowResult = new DorisRowResult(executor.getCoord(), executor.getColumns(), null, statementResources); + statementResources = null; + return new QueryResult(rowResult, null, processor, null); } } catch (Exception e) { + if (statementResources != null) { + try { + statementResources.close(); + } catch (Exception closeFailure) { + e.addSuppressed(closeFailure); + } + } return new QueryResult(null, () -> new Metadata(Collections.emptyList()), null, e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java index 9315ee9ef5cda5..7bf35cb8eb13bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java @@ -57,6 +57,9 @@ public int registerConnection(ConnectContext ctx) { @Override public void unregisterConnection(ConnectContext ctx) { + // Reject new publications and wait for in-flight GetFlightInfo publication to decide its + // outcome before destroying either local Arrow results or deferred coordinators. + ctx.sealAndCloseFlightSqlDeferredExecutors(); // All Flight SQL session teardown paths (idle/query timeout, bearer token expiry, and // explicit CloseSession) reach here. Release channel-cached Arrow results before removing // the context from the pool. @@ -77,7 +80,6 @@ public void unregisterConnection(ConnectContext ctx) { // Finalize any Arrow Flight query whose coordinator was kept alive across the // GetFlightInfo -> DoGet phases (see #62259), releasing its resources (e.g. external-table // batch SplitSources and the query queue slot). - ctx.sealAndCloseFlightSqlDeferredExecutors(); ctx.closeTxn(); if (connectionMap.remove(ctx.getConnectionId()) != null) { numberConnection.decrementAndGet(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java index 5b280212b6d02e..f3a2be19541575 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java @@ -50,11 +50,6 @@ public void testEvictionBeforeLoaderReferenceHandoff() { value.evict(); - Mockito.verify(view, Mockito.never()).close(); - HudiFsViewCacheValue.Lease lease = value.tryAcquire(); - Assert.assertNotNull(lease); - Mockito.verify(view).sync(); - lease.close(); Mockito.verify(view).close(); Assert.assertNull(value.tryAcquire()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index 6f753b8a455c8b..466bc0b0f05cb8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -117,4 +118,52 @@ void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { executor.shutdownNow(); } } + + @Test + void synchronousListingCancellationReturnsBeforeBlockedTaskAndRetainsLease() throws Exception { + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask(() -> { + started.countDown(); + while (release.getCount() > 0) { + try { + release.await(3, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + // Model storage code that does not terminate when interrupted. + } + } + }, () -> { }); + owner.track(task); + owner.submissionDone(); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + executor.execute(task); + Assertions.assertTrue(started.await(3, TimeUnit.SECONDS)); + Future waiter = executor.submit(() -> + Assertions.assertThrows(CancellationException.class, owner::awaitCompletion)); + + owner.close(); + + waiter.get(3, TimeUnit.SECONDS); + Mockito.verify(lease, Mockito.never()).close(); + release.countDown(); + Mockito.verify(lease, Mockito.timeout(3000)).close(); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + + @Test + void synchronousListingDiscardBeforeSubmissionReleasesLease() { + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease); + + owner.discardBeforeSubmission(); + + Mockito.verify(lease).close(); + Assertions.assertDoesNotThrow(owner::awaitCompletion); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java index 0b13d0ec74586e..e215a450606f08 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java @@ -195,6 +195,20 @@ public void testReplayUpdatedRestoresStartTime() { Assert.assertEquals(1234L, job.getStartTimeMs()); } + @Test + public void testPauseCancelsTaskAfterReleasingJobWriteLock() throws Exception { + TestStreamingInsertJob job = newJob(new JdbcSourceOffsetProvider(), 1017L); + ReentrantReadWriteLock jobLock = Deencapsulation.getField(job, "lock"); + LockCheckingTask task = new LockCheckingTask(1017L, jobLock); + Deencapsulation.setField(job, "runningStreamTask", task); + + job.updateJobStatus(JobStatus.PAUSED); + + Assert.assertTrue(task.cancelCalled); + Assert.assertFalse(task.cancelObservedWriteLock); + Assert.assertNull(Deencapsulation.getField(job, "runningStreamTask")); + } + private static TestStreamingInsertJob newJob(JdbcSourceOffsetProvider provider, long taskId) { TestStreamingInsertJob job = new TestStreamingInsertJob(); Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); @@ -260,4 +274,22 @@ private static class NoopStreamingMultiTblTask extends StreamingMultiTblTask { public void successCallback(CommitOffsetRequest offsetRequest) throws JobException { } } + + private static class LockCheckingTask extends NoopStreamingMultiTblTask { + private final ReentrantReadWriteLock jobLock; + private boolean cancelCalled; + private boolean cancelObservedWriteLock; + + LockCheckingTask(long taskId, ReentrantReadWriteLock jobLock) { + super(taskId); + this.jobLock = jobLock; + } + + @Override + public void cancel(boolean needWaitCancelComplete) { + cancelCalled = true; + cancelObservedWriteLock = jobLock.isWriteLockedByCurrentThread(); + super.cancel(false); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java index f3330b567070c3..de2c2432b66eee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java @@ -111,4 +111,85 @@ public void closeOrReleaseResources() { Assertions.assertEquals(1, closeCalls.get()); Assertions.assertTrue(cleanupRanOnWorker.get()); } + + @Test + void terminalFailureCanCancelFromExecutionOwnerWithoutSelfWait() throws Exception { + AtomicBoolean failed = new AtomicBoolean(); + StreamingInsertTask task = new StreamingInsertTask( + 1L, 2L, "", null, "", null, Collections.emptyMap(), null, null) { + @Override + public void before() { + setStatus(org.apache.doris.job.common.TaskStatus.RUNNING); + noRetry = true; + } + + @Override + public void run() throws org.apache.doris.job.exception.JobException { + throw new org.apache.doris.job.exception.JobException("expected"); + } + + @Override + public synchronized void closeOrReleaseResources() { + } + + @Override + protected void onFail(String errMsg) { + setStatus(org.apache.doris.job.common.TaskStatus.FAILED); + cancel(true); + failed.set(true); + } + }; + Thread worker = new Thread(() -> { + try { + task.execute(); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + + worker.start(); + worker.join(TimeUnit.SECONDS.toMillis(10)); + + Assertions.assertFalse(worker.isAlive()); + Assertions.assertTrue(failed.get()); + } + + @Test + void cleanupFailureIsHandledByTaskFailureStateMachine() throws Exception { + AtomicBoolean failed = new AtomicBoolean(); + AtomicBoolean successCalled = new AtomicBoolean(); + AbstractStreamingTask task = new AbstractStreamingTask(1L, 2L, null) { + @Override + public void before() { + setStatus(org.apache.doris.job.common.TaskStatus.RUNNING); + noRetry = true; + } + + @Override + public void run() { + } + + @Override + public boolean onSuccess() { + successCalled.set(true); + return true; + } + + @Override + public void closeOrReleaseResources() { + throw new IllegalStateException("cleanup failed"); + } + + @Override + protected void onFail(String errMsg) { + Assertions.assertEquals("cleanup failed", errMsg); + failed.set(true); + } + }; + + task.execute(); + + Assertions.assertTrue(failed.get()); + Assertions.assertFalse(successCalled.get()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java b/fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java new file mode 100644 index 00000000000000..1efd6558f10c74 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.plsql.executor; + +import org.apache.doris.qe.Coordinator; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.Closeable; +import java.util.Collections; + +class DorisRowResultTest { + + @Test + void closeReleasesCoordinatorAndDetachedStatementResourcesOnce() throws Exception { + Coordinator coordinator = Mockito.mock(Coordinator.class); + Closeable statementResources = Mockito.mock(Closeable.class); + DorisRowResult result = new DorisRowResult( + coordinator, Collections.emptyList(), Collections.emptyList(), statementResources); + + result.close(); + result.close(); + + Mockito.verify(coordinator).close(); + Mockito.verify(statementResources).close(); + } + + @Test + void coordinatorFailureDoesNotSkipStatementResourceCleanup() throws Exception { + Coordinator coordinator = Mockito.mock(Coordinator.class); + Closeable statementResources = Mockito.mock(Closeable.class); + Mockito.doThrow(new IllegalStateException("coordinator close failed")).when(coordinator).close(); + DorisRowResult result = new DorisRowResult( + coordinator, Collections.emptyList(), Collections.emptyList(), statementResources); + + IllegalStateException failure = Assertions.assertThrows(IllegalStateException.class, result::close); + + Assertions.assertEquals("coordinator close failed", failure.getMessage()); + Mockito.verify(statementResources).close(); + } + + @Test + void noCoordinatorClosesDetachedStatementResourcesOnFirstFetch() throws Exception { + Closeable statementResources = Mockito.mock(Closeable.class); + DorisRowResult result = new DorisRowResult( + null, Collections.emptyList(), Collections.emptyList(), statementResources); + + Assertions.assertFalse(result.next()); + + Mockito.verify(statementResources).close(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java index 5b02e297a884f2..d0fcb5636dfcc8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java @@ -22,6 +22,7 @@ import org.junit.Assert; import org.junit.Test; +import org.mockito.InOrder; import org.mockito.Mockito; public class FlightSqlConnectPoolMgrTest { @@ -44,6 +45,9 @@ public void testUnregisterConnectionFinalizesDeferredExecutors() { // never registered in the pool (an abandoned connection is still cleaned up, not leaked). Mockito.verify(channel).close(); Mockito.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + InOrder teardownOrder = Mockito.inOrder(ctx, channel); + teardownOrder.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + teardownOrder.verify(channel).close(); } // Cleanup must run before the connection bookkeeping (closeTxn / map removal), so that a failure @@ -64,6 +68,9 @@ public void testUnregisterRegisteredConnectionFinalizesDeferredExecutors() { Mockito.verify(channel).close(); Mockito.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + InOrder teardownOrder = Mockito.inOrder(ctx, channel); + teardownOrder.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + teardownOrder.verify(channel).close(); Assert.assertNull(poolMgr.getConnectionMap().get(7)); } } From af899e6b8982b52dd58d48406ee213ba3b413ef8 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Sat, 22 Aug 2026 19:52:20 +0800 Subject: [PATCH 14/38] [fix](external) Close remaining metadata lifecycle races --- .../datasource/hive/HMSExternalCatalog.java | 30 +++++++++++++- .../datasource/hive/source/HiveScanNode.java | 10 +++++ .../datasource/hudi/source/HudiScanNode.java | 10 +++-- .../iceberg/IcebergExternalCatalog.java | 24 ++++++++++- .../iceberg/IcebergExternalTable.java | 11 ++--- .../streaming/AbstractStreamingTask.java | 37 +++++++++++++++-- .../insert/streaming/StreamingInsertJob.java | 23 ++++++++--- .../insert/streaming/StreamingInsertTask.java | 20 +++++++--- .../apache/doris/job/manager/JobManager.java | 2 +- .../org/apache/doris/qe/ConnectContext.java | 19 ++++++--- .../hudi/source/HudiBatchFsViewOwnerTest.java | 16 +++++--- .../iceberg/IcebergTableCacheValueTest.java | 17 ++++++++ ...reamingInsertJobOffsetPersistenceTest.java | 2 + .../StreamingInsertTaskResourceTest.java | 40 +++++++++++++++++++ .../apache/doris/qe/ConnectContextTest.java | 25 ++++++++++++ 15 files changed, 247 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 97ea317c383030..b87da756f57aec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -21,6 +21,7 @@ import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; @@ -36,6 +37,8 @@ import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractHiveProperties; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystemProvider; import org.apache.doris.fs.FileSystemProviderImpl; import org.apache.doris.fs.remote.dfs.DFSFileSystem; @@ -48,6 +51,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -276,7 +280,9 @@ public synchronized IcebergMetadataOps getIcebergMetadataOps() { public synchronized IcebergTableLoadContext beginIcebergTableLoad() { makeSureInitialized(); IcebergMetadataOps ops = getIcebergMetadataOps(); - return new IcebergTableLoadContext(ops, threadPoolWithPreAuth, icebergResourceTracker.beginLoad()); + return new IcebergTableLoadContext(ops, threadPoolWithPreAuth, executionAuthenticator, + catalogProperty.getMetastoreProperties(), + new HashMap<>(catalogProperty.getStoragePropertiesMap()), icebergResourceTracker.beginLoad()); } @Override @@ -296,12 +302,20 @@ private void resetCatalogRuntime(ExternalMetaCacheMgr cacheMgr, boolean invalidC public final class IcebergTableLoadContext implements AutoCloseable { private final IcebergMetadataOps ops; private final ThreadPoolExecutor executor; + private final ExecutionAuthenticator authenticator; + private final MetastoreProperties metastoreProperties; + private final Map storageProperties; private final IcebergCatalogResourceTracker.LoadGuard guard; private IcebergTableLoadContext(IcebergMetadataOps ops, ThreadPoolExecutor executor, + ExecutionAuthenticator authenticator, MetastoreProperties metastoreProperties, + Map storageProperties, IcebergCatalogResourceTracker.LoadGuard guard) { this.ops = ops; this.executor = executor; + this.authenticator = authenticator; + this.metastoreProperties = metastoreProperties; + this.storageProperties = storageProperties; this.guard = guard; } @@ -313,8 +327,20 @@ public ThreadPoolExecutor getExecutor() { return executor; } + public ExecutionAuthenticator getAuthenticator() { + return authenticator; + } + + public MetastoreProperties getMetastoreProperties() { + return metastoreProperties; + } + + public Map getStorageProperties() { + return storageProperties; + } + public Table loadTable(String dbName, String tableName) throws Exception { - return executionAuthenticator.execute(() -> ops.loadTable(dbName, tableName)); + return authenticator.execute(() -> ops.loadTable(dbName, tableName)); } public IcebergCatalogResourceTracker.ResourceLease promote() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index d9d15c095a60e8..190f3024c6e17f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -74,6 +74,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -146,6 +147,15 @@ protected void doInitialize() throws UserException { this.hiveTransaction = new HiveTransaction(DebugUtil.printId(ConnectContext.get().queryId()), ConnectContext.get().getQualifiedUser(), hmsTable, hmsTable.isFullAcidTable()); Env.getCurrentHiveTransactionMgr().register(hiveTransaction); + try { + StatementContext statementContext = ConnectContext.get().getStatementContext(); + String queryId = hiveTransaction.getQueryId(); + statementContext.getOrRegisterStatementResource("hive-transaction:" + queryId, + () -> (Closeable) () -> Env.getCurrentHiveTransactionMgr().deregister(queryId)); + } catch (RuntimeException | Error e) { + Env.getCurrentHiveTransactionMgr().deregister(hiveTransaction.getQueryId()); + throw e; + } skipCheckingAcidVersionFile = sessionVariable.skipCheckingAcidVersionFile; } ensureHmsRuntimeGeneration(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 25d3ca9a21d683..45cb91e01087ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -790,14 +790,16 @@ protected void done() { } private void recordBatchException(Throwable t) { - batchException.compareAndSet(null, new UserException(t.getMessage(), t)); + UserException failure = new UserException(t.getMessage(), t); + if (batchException.compareAndSet(null, failure)) { + // Consumers should observe the known failure immediately; sibling terminal accounting + // remains independent and still owns the filesystem-view lease until every task exits. + splitAssignment.setException(failure); + } } private void finishBatchSplit(BatchFsViewOwner batchOwner, long startTime) { try { - if (batchException.get() != null) { - splitAssignment.setException(batchException.get()); - } if (getSummaryProfile() != null) { getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index da6dabc7892f41..007c3b1a0ed9af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -33,6 +33,8 @@ import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractIcebergProperties; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.transaction.TransactionManagerFactory; import org.apache.iceberg.Table; @@ -40,6 +42,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; @@ -159,7 +162,8 @@ public Catalog getCatalog() { synchronized TableLoadContext beginTableLoad() { makeSureInitialized(); return new TableLoadContext((IcebergMetadataOps) metadataOps, executionAuthenticator, icebergCatalogType, - resourceTracker.beginLoad()); + catalogProperty.getMetastoreProperties(), + new HashMap<>(catalogProperty.getStoragePropertiesMap()), resourceTracker.beginLoad()); } public String getIcebergCatalogType() { @@ -226,13 +230,19 @@ final class TableLoadContext implements AutoCloseable { private final IcebergMetadataOps ops; private final ExecutionAuthenticator authenticator; private final String catalogType; + private final MetastoreProperties metastoreProperties; + private final Map storageProperties; private final IcebergCatalogResourceTracker.LoadGuard guard; private TableLoadContext(IcebergMetadataOps ops, ExecutionAuthenticator authenticator, String catalogType, + MetastoreProperties metastoreProperties, + Map storageProperties, IcebergCatalogResourceTracker.LoadGuard guard) { this.ops = ops; this.authenticator = authenticator; this.catalogType = catalogType; + this.metastoreProperties = metastoreProperties; + this.storageProperties = storageProperties; this.guard = guard; } @@ -248,6 +258,18 @@ String getCatalogType() { return catalogType; } + ExecutionAuthenticator getAuthenticator() { + return authenticator; + } + + MetastoreProperties getMetastoreProperties() { + return metastoreProperties; + } + + Map getStorageProperties() { + return storageProperties; + } + IcebergCatalogResourceTracker.ResourceLease promote() { return guard.promote(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 7adda3f51aecae..81d08123ec8315 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -431,8 +431,7 @@ public String location() { View icebergView = getIcebergView(); return icebergView.location(); } else { - Table icebergTable = getIcebergTable(); - return icebergTable.location(); + return IcebergUtils.withIcebergTable(this, Table::location); } } @@ -445,16 +444,14 @@ public Map properties() { View icebergView = getIcebergView(); return icebergView.properties(); } else { - Table icebergTable = getIcebergTable(); - return icebergTable.properties(); + return IcebergUtils.withIcebergTable(this, table -> new HashMap<>(table.properties())); } } @Override public boolean isPartitionedTable() { makeSureInitialized(); - Table table = getIcebergTable(); - return table.spec().isPartitioned(); + return IcebergUtils.withIcebergTable(this, table -> table.spec().isPartitioned()); } /** @@ -462,7 +459,7 @@ public boolean isPartitionedTable() { * @return SQL string representing ORDER BY clause, or empty string if no sort order */ public String getSortOrderSql() { - return getSortOrderSql(getIcebergTable()); + return IcebergUtils.withIcebergTable(this, this::getSortOrderSql); } /** Return the sort order SQL for an already resolved Iceberg metadata generation. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java index 3ff579e28901fd..c9a8210cb156d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java @@ -105,9 +105,11 @@ public void execute() throws JobException { try { while (retryCount <= MAX_RETRY) { Exception attemptFailure = null; + boolean executionSucceeded = false; try { before(); run(); + executionSucceeded = true; } catch (Exception e) { attemptFailure = e; } finally { @@ -124,8 +126,21 @@ public void execute() throws JobException { } } } + // A completed insert must never be replayed merely because teardown failed. Likewise, + // successor-publication failures belong to the job state machine, not to the insert retry loop. + if (executionSucceeded) { + if (attemptFailure != null) { + failCompletedAttempt(attemptFailure); + return; + } + try { + onSuccess(); + } catch (Exception completionFailure) { + failCompletedAttempt(completionFailure); + } + return; + } if (attemptFailure == null) { - onSuccess(); return; } if (TaskStatus.CANCELED.equals(status)) { @@ -148,18 +163,34 @@ public void execute() throws JobException { executionOwner = null; executionCompletion.notifyAll(); } + onExecutionFinished(); } } - protected void awaitExecutionCompletion() { + protected void onExecutionFinished() { + } + + private void failCompletedAttempt(Exception failure) throws JobException { + this.errMsg = failure.getMessage(); + log.error("Completed streaming task could not publish its terminal state, job id {}, task id {}.", + jobId, taskId, failure); + onFail(failure.getMessage()); + } + + protected void awaitExecutionCompletion(long timeoutMs) { boolean interrupted = false; synchronized (executionCompletion) { if (Thread.currentThread() == executionOwner) { return; } + long deadline = System.currentTimeMillis() + timeoutMs; while (executionStarted && !executionFinished) { + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + break; + } try { - executionCompletion.wait(); + executionCompletion.wait(remaining); } catch (InterruptedException e) { interrupted = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java index 4895355235f4c1..71d7c64f62dedf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java @@ -560,7 +560,6 @@ public void updateJobStatus(JobStatus status) throws JobException { if ((JobStatus.PAUSED.equals(status) || JobStatus.STOPPED.equals(status)) && status != getJobStatus()) { taskToCancel = runningStreamTask; - runningStreamTask = null; } super.updateJobStatus(status); if (isFinalStatus()) { @@ -575,7 +574,7 @@ public void updateJobStatus(JobStatus status) throws JobException { } finally { lock.writeLock().unlock(); } - if (taskToCancel != null) { + if (taskToCancel != null && waitForTask) { // The task owner can need this job's write lock while finishing transaction callbacks. // Cancel and wait only after publishing the status and releasing the job lock. taskToCancel.cancel(waitForTask); @@ -612,7 +611,9 @@ public void cancelAllTasks(boolean needWaitCancelComplete) throws JobException { // already counted by onStreamTaskFail(), so skip to avoid double-counting. boolean wasActive = TaskStatus.RUNNING.equals(runningStreamTask.getStatus()) || TaskStatus.PENDING.equals(runningStreamTask.getStatus()); - runningStreamTask.cancel(needWaitCancelComplete); + // Publish cancellation under the job lock, but never wait here: transaction callbacks + // need the same lock and keep using this exact task until the owner reaches terminality. + runningStreamTask.cancel(false); if (wasActive) { canceledTaskCount.incrementAndGet(); } @@ -760,12 +761,11 @@ protected void fetchMeta() throws JobException { || !InternalErrorCode.MANUAL_PAUSE_ERR.equals(this.getFailureReason().getCode())) { // When a job is manually paused, it does not need to be set again, // otherwise, it may be woken up by auto resume. - // Pause before setting the reason: updateJobStatus's writeLock orders this after any - // task-success callback that clears failureReason, so a success can't wipe the reason. - this.updateJobStatus(JobStatus.PAUSED); this.setFailureReason( new FailureReason(InternalErrorCode.GET_REMOTE_DATA_ERROR, "Failed to fetch meta, " + ex.getMessage())); + // Publish the non-resumable reason before PAUSED becomes scheduler-visible. + this.updateJobStatus(JobStatus.PAUSED); if (MetricRepo.isInit) { MetricRepo.COUNTER_STREAMING_JOB_GET_META_FAIL_COUNT.increase(1L); @@ -833,6 +833,17 @@ public void clearRunningStreamTask(JobStatus newJobStatus) { } } + public void clearRunningStreamTask(AbstractStreamingTask finishedTask) { + lock.writeLock().lock(); + try { + if (runningStreamTask == finishedTask) { + runningStreamTask = null; + } + } finally { + lock.writeLock().unlock(); + } + } + // Command entry for a manual status change: reset the failure/retry budget, and on manual pause // release the reader (keep slot). "Manual" is decided by the caller, never by reading failureReason. public void onManualStatusAltered(JobStatus newStatus, FailureReason reason) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java index e99e77ef581a2b..3b25bdcdc87b3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java @@ -54,6 +54,7 @@ @Log4j2 @Getter public class StreamingInsertTask extends AbstractStreamingTask { + private static final long CANCEL_WAIT_TIMEOUT_MS = 1000; private String sql; private volatile StmtExecutor stmtExecutor; private InsertIntoTableCommand taskCommand; @@ -150,11 +151,10 @@ public boolean onSuccess() throws JobException { if (getIsCanceled().get()) { return false; } - this.status = TaskStatus.SUCCESS; - this.finishTimeMs = System.currentTimeMillis(); if (!isCallable()) { return false; } + this.finishTimeMs = System.currentTimeMillis(); Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); if (null == job) { log.info("job is null, job id is {}", jobId); @@ -163,6 +163,7 @@ public boolean onSuccess() throws JobException { StreamingInsertJob streamingInsertJob = (StreamingInsertJob) job; streamingInsertJob.onStreamTaskSuccess(this); + this.status = TaskStatus.SUCCESS; return true; } @@ -178,11 +179,12 @@ public void cancel(boolean needWaitCancelComplete) { if (null != executor) { log.info("cancelling streaming insert task, job id is {}, task id is {}", getJobId(), getTaskId()); - executor.cancel(new Status(TStatusCode.CANCELLED, "streaming insert task cancelled"), - needWaitCancelComplete); + executor.cancel(new Status(TStatusCode.CANCELLED, "streaming insert task cancelled"), false); } if (needWaitCancelComplete) { - awaitExecutionCompletion(); + // Planning may still be blocked before stmtExecutor is published. Do not let PAUSE wait + // forever; the scheduler owner remains responsible for exact-once cleanup in execute(). + awaitExecutionCompletion(CANCEL_WAIT_TIMEOUT_MS); } } @@ -228,6 +230,14 @@ public synchronized void closeOrReleaseResources() { } } + @Override + protected void onExecutionFinished() { + Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); + if (job instanceof StreamingInsertJob) { + ((StreamingInsertJob) job).clearRunningStreamTask(this); + } + } + @Override public TRow getTvfInfo(String jobName) { TRow trow = super.getTvfInfo(jobName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java index 8c55856da57d7c..82f3020b64d67b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java @@ -280,10 +280,10 @@ public void alterJobStatus(String jobName, JobStatus jobStatus, FailureReason re if (a.getJobName().equals(jobName)) { try { checkSameStatus(a, jobStatus); - alterJobStatus(a.getJobId(), jobStatus); if (a instanceof StreamingInsertJob) { ((StreamingInsertJob) a).onManualStatusAltered(jobStatus, reason); } + alterJobStatus(a.getJobId(), jobStatus); } catch (JobException e) { throw new JobException("Alter job status error, jobName is %s, errorMsg is %s", jobName, e.getMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 4814e6aa70ae37..d6a513fc50ebe3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -1024,6 +1024,7 @@ public boolean endFlightSqlResultPublication() { published = !flightSqlDeferredExecutorsSealed; if (--flightSqlResultPublishers == 0 && flightSqlDeferredExecutorsSealed) { toClose = drainFlightSqlDeferredExecutors(); + flightSqlDeferredExecutors.notifyAll(); } } finalizeFlightSqlDeferredExecutors(toClose); @@ -1044,11 +1045,19 @@ private void closeFlightSqlDeferredExecutors(boolean seal) { synchronized (flightSqlDeferredExecutors) { if (seal) { flightSqlDeferredExecutorsSealed = true; - // An in-flight GetFlightInfo owns the coordinator until it either publishes its result or - // observes the seal and fails. Let its terminal path perform the drain so teardown cannot - // release the query resources while a successful ticket is still being constructed. - if (flightSqlResultPublishers != 0) { - return; + // The result channel is destroyed immediately after this method returns. Wait until every + // admitted publisher has either committed or observed the seal, so a losing local-result + // publisher cannot insert Arrow buffers after the channel's one-time invalidation. + boolean interrupted = false; + while (flightSqlResultPublishers != 0) { + try { + flightSqlDeferredExecutors.wait(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); } } toClose = drainFlightSqlDeferredExecutors(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index 466bc0b0f05cb8..dab6094d5e52eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -125,14 +125,19 @@ void synchronousListingCancellationReturnsBeforeBlockedTaskAndRetainsLease() thr HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease); CountDownLatch started = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); + CountDownLatch terminated = new CountDownLatch(1); HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask(() -> { started.countDown(); - while (release.getCount() > 0) { - try { - release.await(3, TimeUnit.SECONDS); - } catch (InterruptedException ignored) { - // Model storage code that does not terminate when interrupted. + try { + while (release.getCount() > 0) { + try { + release.await(3, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + // Model storage code that does not terminate when interrupted. + } } + } finally { + terminated.countDown(); } }, () -> { }); owner.track(task); @@ -149,6 +154,7 @@ void synchronousListingCancellationReturnsBeforeBlockedTaskAndRetainsLease() thr waiter.get(3, TimeUnit.SECONDS); Mockito.verify(lease, Mockito.never()).close(); release.countDown(); + Assertions.assertTrue(terminated.await(3, TimeUnit.SECONDS)); Mockito.verify(lease, Mockito.timeout(3000)).close(); } finally { release.countDown(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java index d04fab7ce53c38..4c306cf0bacf60 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -108,6 +108,23 @@ void evictionWaitsForActiveBorrower() { Assertions.assertEquals(1, cleanupCount.get()); } + @Test + void retainedLeaseKeepsGenerationAliveForAsyncOwner() { + AtomicInteger cleanupCount = new AtomicInteger(); + IcebergTableCacheValue value = newValue(cleanupCount); + IcebergTableCacheValue.Lease statementLease = value.tryAcquire(); + Assertions.assertNotNull(statementLease); + IcebergTableCacheValue.Lease asyncLease = statementLease.retain(); + value.releaseLoaderReference(); + value.releaseCacheReference(); + + statementLease.close(); + Assertions.assertEquals(0, cleanupCount.get()); + + asyncLease.close(); + Assertions.assertEquals(1, cleanupCount.get()); + } + @Test void statementCloseReleasesBorrowerAfterPlannerResources() { AtomicInteger cleanupCount = new AtomicInteger(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java index e215a450606f08..9d81c6e9d73749 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java @@ -206,6 +206,8 @@ public void testPauseCancelsTaskAfterReleasingJobWriteLock() throws Exception { Assert.assertTrue(task.cancelCalled); Assert.assertFalse(task.cancelObservedWriteLock); + Assert.assertSame(task, Deencapsulation.getField(job, "runningStreamTask")); + job.clearRunningStreamTask(task); Assert.assertNull(Deencapsulation.getField(job, "runningStreamTask")); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java index de2c2432b66eee..7af0356530a547 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java @@ -158,6 +158,7 @@ protected void onFail(String errMsg) { void cleanupFailureIsHandledByTaskFailureStateMachine() throws Exception { AtomicBoolean failed = new AtomicBoolean(); AtomicBoolean successCalled = new AtomicBoolean(); + AtomicInteger runCalls = new AtomicInteger(); AbstractStreamingTask task = new AbstractStreamingTask(1L, 2L, null) { @Override public void before() { @@ -167,6 +168,7 @@ public void before() { @Override public void run() { + runCalls.incrementAndGet(); } @Override @@ -191,5 +193,43 @@ protected void onFail(String errMsg) { Assertions.assertTrue(failed.get()); Assertions.assertFalse(successCalled.get()); + Assertions.assertEquals(1, runCalls.get()); + } + + @Test + void successCallbackFailureDoesNotReplayCompletedInsert() throws Exception { + AtomicInteger runCalls = new AtomicInteger(); + AtomicBoolean failed = new AtomicBoolean(); + AbstractStreamingTask task = new AbstractStreamingTask(1L, 2L, null) { + @Override + public void before() { + setStatus(org.apache.doris.job.common.TaskStatus.RUNNING); + } + + @Override + public void run() { + runCalls.incrementAndGet(); + } + + @Override + public boolean onSuccess() { + throw new IllegalStateException("success publication failed"); + } + + @Override + public void closeOrReleaseResources() { + } + + @Override + protected void onFail(String errMsg) { + Assertions.assertEquals("success publication failed", errMsg); + failed.set(true); + } + }; + + task.execute(); + + Assertions.assertEquals(1, runCalls.get()); + Assertions.assertTrue(failed.get()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index b0d789655abeb3..27d450190847d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -57,6 +57,11 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; public class ConnectContextTest { @@ -942,4 +947,24 @@ public void testSessionTeardownRejectsLateDeferredExecutorRegistration() { ctx.closeFlightSqlDeferredExecutors(); Mockito.verifyNoInteractions(late); } + + @Test + public void testSessionSealWaitsForAdmittedResultPublisher() throws Exception { + ConnectContext ctx = new ConnectContext(); + Assert.assertTrue(ctx.beginFlightSqlResultPublication()); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future teardown = executor.submit(ctx::sealAndCloseFlightSqlDeferredExecutors); + try { + teardown.get(100, TimeUnit.MILLISECONDS); + Assert.fail("teardown must not destroy the result channel while a publisher is admitted"); + } catch (TimeoutException expected) { + // expected + } + Assert.assertFalse(ctx.endFlightSqlResultPublication()); + teardown.get(10, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + } } From 23ac2374889c2ab989fe73f9efc3a64fc434ebcf Mon Sep 17 00:00:00 2001 From: 924060929 Date: Sun, 23 Aug 2026 07:50:16 +0800 Subject: [PATCH 15/38] [fix](external) Fence catalog alter and Flight teardown Fence legacy catalog property candidates from concurrent initialization and serialize Arrow Flight result publication with connection teardown. --- .../org/apache/doris/qe/ConnectContext.java | 17 ++++++++- .../arrowflight/DorisFlightSqlProducer.java | 11 +++--- .../sessions/FlightSqlConnectPoolMgr.java | 11 ++++-- .../apache/doris/qe/ConnectContextTest.java | 27 +++++++++---- .../DorisFlightSqlProducerTest.java | 38 ++++++++++++++++++- .../sessions/FlightSqlConnectPoolMgrTest.java | 16 ++++++-- 6 files changed, 96 insertions(+), 24 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index d6a513fc50ebe3..de8f38d89214fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -1009,10 +1009,10 @@ public boolean canPublishFlightSqlResult() { public boolean beginFlightSqlResultPublication() { synchronized (flightSqlDeferredExecutors) { - if (flightSqlDeferredExecutorsSealed) { + if (flightSqlDeferredExecutorsSealed || flightSqlResultPublishers != 0) { return false; } - flightSqlResultPublishers++; + flightSqlResultPublishers = 1; return true; } } @@ -1037,6 +1037,19 @@ public void closeFlightSqlDeferredExecutors() { /** Prevents a session teardown race from accepting an executor after the final drain. */ public void sealAndCloseFlightSqlDeferredExecutors() { + sealFlightSqlDeferredExecutors(); + awaitAndCloseFlightSqlDeferredExecutors(); + } + + /** Rejects new result/executor publications without waiting for an admitted publisher. */ + public void sealFlightSqlDeferredExecutors() { + synchronized (flightSqlDeferredExecutors) { + flightSqlDeferredExecutorsSealed = true; + } + } + + /** Waits for admitted publishers after their query has been canceled, then drains retained executors. */ + public void awaitAndCloseFlightSqlDeferredExecutors() { closeFlightSqlDeferredExecutors(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index e2763d748ffd55..c40e0d9d1db4fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -186,13 +186,12 @@ public void closePreparedStatement(final ActionClosePreparedStatementRequest req private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext connectContext, String query, final FlightDescriptor descriptor) { - boolean resultPublisher = false; + Preconditions.checkState(null != connectContext); + Preconditions.checkState(!query.isEmpty()); + boolean resultPublisher = connectContext.beginFlightSqlResultPublication(); + Preconditions.checkState(resultPublisher, + "Arrow Flight SQL session already has an active result publisher or is torn down"); try { - Preconditions.checkState(null != connectContext); - Preconditions.checkState(!query.isEmpty()); - resultPublisher = connectContext.beginFlightSqlResultPublication(); - Preconditions.checkState(resultPublisher, - "Arrow Flight SQL session is already torn down"); // Finalize the previous query's coordinator on this connection whose close was // deferred (Arrow Flight keeps it alive across GetFlightInfo -> DoGet so the BE can // fetch external-table splits during DoGet). By now the previous DoGet is done. #62259 diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java index 7bf35cb8eb13bd..e47096b0d28caf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java @@ -17,11 +17,13 @@ package org.apache.doris.service.arrowflight.sessions; +import org.apache.doris.common.Status; import org.apache.doris.common.util.TokenMasker; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContext.ConnectType; import org.apache.doris.qe.ConnectPoolMgr; import org.apache.doris.service.arrowflight.results.FlightSqlChannel; +import org.apache.doris.thrift.TStatusCode; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; @@ -57,9 +59,12 @@ public int registerConnection(ConnectContext ctx) { @Override public void unregisterConnection(ConnectContext ctx) { - // Reject new publications and wait for in-flight GetFlightInfo publication to decide its - // outcome before destroying either local Arrow results or deferred coordinators. - ctx.sealAndCloseFlightSqlDeferredExecutors(); + // Reject new publications first, then signal the active query before waiting for an admitted + // GetFlightInfo publisher. Waiting before cancellation can deadlock KILL CONNECTION behind the + // publisher whose query must be canceled in order to leave publication. + ctx.sealFlightSqlDeferredExecutors(); + ctx.cancelQuery(new Status(TStatusCode.CANCELLED, "arrow flight connection closed")); + ctx.awaitAndCloseFlightSqlDeferredExecutors(); // All Flight SQL session teardown paths (idle/query timeout, bearer token expiry, and // explicit CloseSession) reach here. Release channel-cached Arrow results before removing // the context from the pool. diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index 27d450190847d6..984fd8b9559a86 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -57,12 +57,13 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class ConnectContextTest { @Mocked @@ -952,15 +953,27 @@ public void testSessionTeardownRejectsLateDeferredExecutorRegistration() { public void testSessionSealWaitsForAdmittedResultPublisher() throws Exception { ConnectContext ctx = new ConnectContext(); Assert.assertTrue(ctx.beginFlightSqlResultPublication()); + Assert.assertFalse("one Flight SQL session cannot publish two queries concurrently", + ctx.beginFlightSqlResultPublication()); + ctx.sealFlightSqlDeferredExecutors(); + Assert.assertFalse(ctx.canPublishFlightSqlResult()); ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch teardownEntered = new CountDownLatch(1); + AtomicReference teardownThread = new AtomicReference<>(); try { - Future teardown = executor.submit(ctx::sealAndCloseFlightSqlDeferredExecutors); - try { - teardown.get(100, TimeUnit.MILLISECONDS); - Assert.fail("teardown must not destroy the result channel while a publisher is admitted"); - } catch (TimeoutException expected) { - // expected + Future teardown = executor.submit(() -> { + teardownThread.set(Thread.currentThread()); + teardownEntered.countDown(); + ctx.awaitAndCloseFlightSqlDeferredExecutors(); + }); + Assert.assertTrue(teardownEntered.await(10, TimeUnit.SECONDS)); + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (teardownThread.get().getState() != Thread.State.WAITING + && System.nanoTime() < deadlineNanos) { + Thread.yield(); } + Assert.assertEquals("teardown must be waiting for the admitted publisher", + Thread.State.WAITING, teardownThread.get().getState()); Assert.assertFalse(ctx.endFlightSqlResultPublication()); teardown.get(10, TimeUnit.SECONDS); } finally { diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java index de281ba8f9f888..456e7c4d55fb0b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -18,6 +18,7 @@ package org.apache.doris.service.arrowflight; import org.apache.doris.common.FeConstants; +import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.service.arrowflight.results.FlightSqlChannel; @@ -227,12 +228,45 @@ public void testPublicationCompletionAtomicallyObservesTerminalSeal() { Assert.assertTrue(ctx.beginFlightSqlResultPublication()); Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(deferred)); - ctx.sealAndCloseFlightSqlDeferredExecutors(); + ctx.sealFlightSqlDeferredExecutors(); Assert.assertFalse("a publication in flight before teardown must not commit after the terminal seal", ctx.endFlightSqlResultPublication()); Mockito.verify(deferred).finalizeArrowFlightQuery(); } + @Test + public void testRejectedConcurrentPublisherDoesNotTouchActiveQueryState() throws Exception { + ConnectContext ctx = new ConnectContext(); + StmtExecutor activeDeferred = Mockito.mock(StmtExecutor.class); + Assert.assertTrue(ctx.beginFlightSqlResultPublication()); + Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(activeDeferred)); + ctx.setCommand(MysqlCommand.COM_QUERY); + FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class); + Mockito.when(sessionsManager.getConnectContext(Mockito.anyString())).thenReturn(ctx); + CallContext callContext = Mockito.mock(CallContext.class); + Mockito.when(callContext.peerIdentity()).thenReturn("token"); + DorisFlightSqlProducer producer = new DorisFlightSqlProducer( + Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager); + + try { + CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 2").build(); + FlightDescriptor descriptor = FlightDescriptor.command(new byte[0]); + Throwable rejected = Assert.assertThrows(Throwable.class, + () -> producer.getFlightInfoStatement(request, callContext, descriptor)); + Assert.assertTrue(rejected.getMessage(), + rejected.getMessage().contains("active result publisher")); + + Mockito.verify(activeDeferred, Mockito.never()).finalizeArrowFlightQuery(); + Assert.assertEquals("a rejected publisher must not mark the active query idle", + MysqlCommand.COM_QUERY, ctx.getCommand()); + Assert.assertTrue(ctx.endFlightSqlResultPublication()); + ctx.closeFlightSqlDeferredExecutors(); + Mockito.verify(activeDeferred).finalizeArrowFlightQuery(); + } finally { + producer.close(); + } + } + private void assertTeardownPreventsFlightInfoPublication(boolean registerBeforeSeal) throws Exception { ConnectContext ctx = Mockito.spy(new ConnectContext()); Mockito.doReturn(Mockito.mock(FlightSqlChannel.class)).when(ctx).getFlightSqlChannel(); @@ -251,7 +285,7 @@ private void assertTeardownPreventsFlightInfoPublication(boolean registerBeforeS if (registerBeforeSeal) { Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(deferred)); } - ctx.sealAndCloseFlightSqlDeferredExecutors(); + ctx.sealFlightSqlDeferredExecutors(); if (!registerBeforeSeal) { Assert.assertFalse(ctx.addFlightSqlDeferredExecutor(deferred)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java index d0fcb5636dfcc8..1558e97a42f890 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java @@ -44,9 +44,13 @@ public void testUnregisterConnectionFinalizesDeferredExecutors() { // The deferred coordinators must be released on teardown even though this connection was // never registered in the pool (an abandoned connection is still cleaned up, not leaked). Mockito.verify(channel).close(); - Mockito.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + Mockito.verify(ctx).sealFlightSqlDeferredExecutors(); + Mockito.verify(ctx).cancelQuery(Mockito.any()); + Mockito.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); InOrder teardownOrder = Mockito.inOrder(ctx, channel); - teardownOrder.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + teardownOrder.verify(ctx).sealFlightSqlDeferredExecutors(); + teardownOrder.verify(ctx).cancelQuery(Mockito.any()); + teardownOrder.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); teardownOrder.verify(channel).close(); } @@ -67,9 +71,13 @@ public void testUnregisterRegisteredConnectionFinalizesDeferredExecutors() { poolMgr.unregisterConnection(ctx); Mockito.verify(channel).close(); - Mockito.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + Mockito.verify(ctx).sealFlightSqlDeferredExecutors(); + Mockito.verify(ctx).cancelQuery(Mockito.any()); + Mockito.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); InOrder teardownOrder = Mockito.inOrder(ctx, channel); - teardownOrder.verify(ctx).sealAndCloseFlightSqlDeferredExecutors(); + teardownOrder.verify(ctx).sealFlightSqlDeferredExecutors(); + teardownOrder.verify(ctx).cancelQuery(Mockito.any()); + teardownOrder.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); teardownOrder.verify(channel).close(); Assert.assertNull(poolMgr.getConnectionMap().get(7)); } From ca8a689810c3d0d86c27736b5b6aea633abeabcf Mon Sep 17 00:00:00 2001 From: 924060929 Date: Sun, 23 Aug 2026 16:44:10 +0800 Subject: [PATCH 16/38] [fix](external) fence streaming task lifecycle handoffs --- .../streaming/AbstractStreamingTask.java | 14 +- .../insert/streaming/StreamingInsertJob.java | 288 ++++++++++++++---- .../streaming/StreamingJobSchedulerTask.java | 26 +- .../streaming/StreamingMultiTblTask.java | 69 ++++- .../apache/doris/job/manager/JobManager.java | 13 +- .../org/apache/doris/qe/ConnectContext.java | 41 ++- .../java/org/apache/doris/qe/Coordinator.java | 68 ++++- .../org/apache/doris/qe/MasterOpExecutor.java | 16 + .../apache/doris/qe/NereidsCoordinator.java | 12 +- .../org/apache/doris/qe/StmtExecutor.java | 42 ++- .../runtime/MultiFragmentsPipelineTask.java | 89 +++--- .../qe/runtime/PipelineExecutionTask.java | 11 +- .../doris/rpc/BackendServiceClient.java | 4 +- .../apache/doris/rpc/BackendServiceProxy.java | 6 +- .../doris/service/FrontendServiceImpl.java | 19 +- .../iceberg/IcebergExternalTableTest.java | 2 +- .../StreamingInsertJobLateCallbackTest.java | 73 +++++ .../apache/doris/qe/ConnectContextTest.java | 25 ++ .../qe/StmtExecutorCancellationTest.java | 51 ++++ 19 files changed, 719 insertions(+), 150 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java index c9a8210cb156d3..aad6babac1df28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java @@ -201,6 +201,13 @@ protected void awaitExecutionCompletion(long timeoutMs) { } } + /** True when cancellation can hand off the job slot without overlapping the execution owner. */ + boolean canHandoffAfterCancellation() { + synchronized (executionCompletion) { + return !executionStarted || executionFinished; + } + } + protected void onFail(String errMsg) throws JobException { if (getIsCanceled().get()) { return; @@ -226,7 +233,8 @@ protected boolean isCallable() { return false; } - public void cancel(boolean needWaitCancelComplete) { + /** Publishes cancellation without performing task-specific RPCs or waits. */ + public void publishCancellation() { // Flip isCanceled even on terminal states so late BE callbacks short-circuit. if (getIsCanceled().getAndSet(true)) { return; @@ -239,6 +247,10 @@ public void cancel(boolean needWaitCancelComplete) { this.errMsg = "task cancelled"; } + public void cancel(boolean needWaitCancelComplete) { + publishCancellation(); + } + /** * show streaming insert task info detail */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java index 71d7c64f62dedf..2ef2e922510411 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java @@ -195,6 +195,7 @@ public class StreamingInsertJob extends AbstractJob taskContext) { - return CollectionUtils.isEmpty(getRunningTasks()) && !isFinalStatus(); + return CollectionUtils.isEmpty(getRunningTasks()) && runningStreamTask == null && !isFinalStatus(); } @Override @@ -655,18 +765,90 @@ public List createTasks(TaskType taskType, Map queryAllStreamTasks() { protected void fetchMeta() throws JobException { long start = System.currentTimeMillis(); + long expectedEpoch = statusEpoch; try { // when fe restart, offsetProvider.jobId may be null Map props = getProviderProps(); @@ -757,16 +940,8 @@ protected void fetchMeta() throws JobException { offsetProvider.fetchRemoteMeta(props); } catch (Exception ex) { log.warn("fetch remote meta failed, job id: {}", getJobId(), ex); - if (this.getFailureReason() == null - || !InternalErrorCode.MANUAL_PAUSE_ERR.equals(this.getFailureReason().getCode())) { - // When a job is manually paused, it does not need to be set again, - // otherwise, it may be woken up by auto resume. - this.setFailureReason( - new FailureReason(InternalErrorCode.GET_REMOTE_DATA_ERROR, - "Failed to fetch meta, " + ex.getMessage())); - // Publish the non-resumable reason before PAUSED becomes scheduler-visible. - this.updateJobStatus(JobStatus.PAUSED); - + if (pauseForInternalFailure(new FailureReason(InternalErrorCode.GET_REMOTE_DATA_ERROR, + "Failed to fetch meta, " + ex.getMessage()), expectedEpoch)) { if (MetricRepo.isInit) { MetricRepo.COUNTER_STREAMING_JOB_GET_META_FAIL_COUNT.increase(1L); } @@ -785,6 +960,7 @@ protected void fetchMeta() throws JobException { * Called by scheduler each tick (PENDING/RUNNING). Mirrors fetchMeta error handling. */ public void advanceSplitsIfNeed() throws JobException { + long expectedEpoch = statusEpoch; if (offsetProvider.noMoreSplits()) { return; } @@ -804,13 +980,8 @@ public void advanceSplitsIfNeed() throws JobException { } } catch (Exception ex) { log.warn("advance splits failed, job id: {}", getJobId(), ex); - if (this.getFailureReason() == null - || !InternalErrorCode.MANUAL_PAUSE_ERR.equals(this.getFailureReason().getCode())) { - this.setFailureReason(new FailureReason( - InternalErrorCode.GET_REMOTE_DATA_ERROR, - "Failed to advance splits, " + ex.getMessage())); - this.updateJobStatus(JobStatus.PAUSED); - } + pauseForInternalFailure(new FailureReason(InternalErrorCode.GET_REMOTE_DATA_ERROR, + "Failed to advance splits, " + ex.getMessage()), expectedEpoch); } } @@ -844,28 +1015,6 @@ public void clearRunningStreamTask(AbstractStreamingTask finishedTask) { } } - // Command entry for a manual status change: reset the failure/retry budget, and on manual pause - // release the reader (keep slot). "Manual" is decided by the caller, never by reading failureReason. - public void onManualStatusAltered(JobStatus newStatus, FailureReason reason) { - AbstractStreamingTask taskToRelease = null; - lock.writeLock().lock(); - try { - resetFailureInfo(reason); - if (JobStatus.PAUSED.equals(newStatus) && runningStreamTask != null) { - // Force resume to swap in a fresh reader, in case the release RPC races or fails. - this.needRebuildReader = true; - taskToRelease = runningStreamTask; - } - } finally { - lock.writeLock().unlock(); - } - // Release outside the write lock: the RPC may block on first brpc connect and this is - // best-effort (needRebuildReader already forces a fresh reader; a stale release is a no-op). - if (taskToRelease != null) { - taskToRelease.releaseRemoteReader(); - } - } - public boolean hasMoreDataToConsume() { return offsetProvider.hasMoreDataToConsume(); } @@ -886,7 +1035,17 @@ public void onTaskSuccess(StreamingJobSchedulerTask task) throws JobException { } public void onStreamTaskFail(AbstractStreamingTask task) throws JobException { + if (!lock.writeLock().isHeldByCurrentThread()) { + writeLock(); + } try { + if (runningStreamTask != task) { + log.info("Ignore stale failure callback for streaming job {}, task {}", getJobId(), task.getTaskId()); + return; + } + if (!(task instanceof StreamingMultiTblTask)) { + runningStreamTask = null; + } this.needRebuildReader = true; failedTaskCount.incrementAndGet(); Env.getCurrentEnv().getJobManager().getStreamingTaskManager().removeRunningTask(task); @@ -898,14 +1057,38 @@ public void onStreamTaskFail(AbstractStreamingTask task) throws JobException { if (MetricRepo.isInit) { MetricRepo.COUNTER_STREAMING_JOB_TASK_FAILED_COUNT.increase(1L); } + updateJobStatus(JobStatus.PAUSED); } finally { writeUnlock(); } - updateJobStatus(JobStatus.PAUSED); + if (task instanceof StreamingMultiTblTask + && ((StreamingMultiTblTask) task).releaseRemoteReaderAndWait()) { + clearRunningStreamTask(task); + } } public void onStreamTaskSuccess(AbstractStreamingTask task) throws JobException { + onStreamTaskSuccess(task, null); + } + + void onStreamTaskSuccess(AbstractStreamingTask task, Runnable beforeHandoff) throws JobException { + // TVF transaction callbacks transfer one write-lock hold from beforeCommitted() to this + // terminal callback. Multi-table callbacks arrive without that hold and acquire it here. + if (!lock.writeLock().isHeldByCurrentThread()) { + writeLock(); + } try { + if (runningStreamTask != task || !JobStatus.RUNNING.equals(getJobStatus()) + || task.getIsCanceled().get()) { + log.info("Ignore stale success callback for streaming job {}, task {}", getJobId(), task.getTaskId()); + return; + } + if (beforeHandoff != null) { + beforeHandoff.run(); + } + // The success callback is the exact terminal handoff. Clear the predecessor before creating + // its successor; the execution owner's later finally block uses identity and cannot clear the new task. + runningStreamTask = null; this.needRebuildReader = false; resetFailureInfo(null); succeedTaskCount.incrementAndGet(); @@ -926,7 +1109,6 @@ public void onStreamTaskSuccess(AbstractStreamingTask task) throws JobException return; } AbstractStreamingTask nextTask = createStreamingTask(); - this.runningStreamTask = nextTask; log.info("Streaming insert job {} create next streaming insert task {} after task {} success", getJobId(), nextTask.getTaskId(), task.getTaskId()); } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java index 030c3bb2b8bd6c..8087826dd5e7f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java @@ -55,33 +55,25 @@ public void run() throws JobException { } private void handlePendingState() throws JobException { + long expectedEpoch = streamingInsertJob.getStatusEpoch(); + if (streamingInsertJob.hasRunningStreamTask()) { + if (!streamingInsertJob.tryCompleteCanceledPredecessor()) { + return; + } + } if (Config.isCloudMode()) { try { streamingInsertJob.replayOnCloudMode(); } catch (JobException e) { - streamingInsertJob.setFailureReason( - new FailureReason(InternalErrorCode.INTERNAL_ERR, e.getMessage())); - streamingInsertJob.updateJobStatus(JobStatus.PAUSED); + streamingInsertJob.pauseForInternalFailure( + new FailureReason(InternalErrorCode.INTERNAL_ERR, e.getMessage()), expectedEpoch); return; } } streamingInsertJob.replayOffsetProviderIfNeed(); // Pre-advance one batch so the first task has splits to consume streamingInsertJob.advanceSplitsIfNeed(); - if (streamingInsertJob.getJobStatus() == JobStatus.PAUSED) { - // advanceSplits failed and paused the job; skip task dispatch this tick. - return; - } - if (streamingInsertJob.hasReachedEnd()) { - // Source already fully consumed (e.g. snapshot-only mode recovered after FE restart). - // Transition directly to FINISHED without creating a new task. - streamingInsertJob.updateJobStatus(JobStatus.FINISHED); - streamingInsertJob.logUpdateOperation(); - return; - } - streamingInsertJob.createStreamingTask(); - streamingInsertJob.setSampleStartTime(System.currentTimeMillis()); - streamingInsertJob.updateJobStatus(JobStatus.RUNNING); + streamingInsertJob.dispatchPendingTask(expectedEpoch); } private void handleRunningState() throws JobException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java index bf41d674afc9e1..cc5545ed7217bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java @@ -85,6 +85,7 @@ public class StreamingMultiTblTask extends AbstractStreamingTask { private long filteredRows = 0L; private long loadedRows = 0L; private volatile long runningBackendId; + private volatile boolean remoteReaderReleased; long lastScannedRows = -1; long lastProgressMs = 0; @@ -283,12 +284,19 @@ public void successCallback(CommitOffsetRequest offsetRequest) throws JobExcepti if (getIsCanceled().get()) { return; } + Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); + if (null == job) { + log.info("job is null, job id is {}", jobId); + return; + } + StreamingInsertJob streamingInsertJob = (StreamingInsertJob) job; + streamingInsertJob.onStreamTaskSuccess(this, () -> applySuccessCallback(offsetRequest)); + } + + private void applySuccessCallback(CommitOffsetRequest offsetRequest) { this.status = TaskStatus.SUCCESS; this.finishTimeMs = System.currentTimeMillis(); JdbcOffset runOffset = (JdbcOffset) this.runningOffset; - if (!isCallable()) { - return; - } // set end offset to running offset // binlogSplit : [{"splitId":"binlog-split"}] only 1 element // snapshotSplit:[{"splitId":"table-0"},...],...}] @@ -327,19 +335,11 @@ public void successCallback(CommitOffsetRequest offsetRequest) throws JobExcepti this.loadBytes = offsetRequest.getLoadBytes(); this.filteredRows = offsetRequest.getFilteredRows(); this.loadedRows = offsetRequest.getLoadedRows(); - Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); - if (null == job) { - log.info("job is null, job id is {}", jobId); - return; - } - StreamingInsertJob streamingInsertJob = (StreamingInsertJob) job; - streamingInsertJob.onStreamTaskSuccess(this); } @Override protected void onFail(String errMsg) throws JobException { - // Stop a possibly still-running reader now, so the PG slot frees before auto-resume re-acquires it. - releaseRemoteReader(); + // The job owns the acknowledged reader-release handoff before it allows auto resume. super.onFail(errMsg); } @@ -355,6 +355,18 @@ public void closeOrReleaseResources() { // No-op: the reader is async and reused; releasing here (per-iteration finally) would kill it. } + @Override + protected void onExecutionFinished() { + if (!getIsCanceled().get() || (runningBackendId > 0 && !remoteReaderReleased)) { + return; + } + try { + getStreamingJob().clearRunningStreamTask(this); + } catch (JobException e) { + log.info("Skip terminal handoff for removed streaming job {}, task {}", getJobId(), getTaskId()); + } + } + @Override public long getRunningBackendId() { return runningBackendId; @@ -390,6 +402,39 @@ public void releaseRemoteReader() { } } + /** Wait for the BE to acknowledge reader release before allowing a successor to reuse the source. */ + boolean releaseRemoteReaderAndWait() { + if (runningBackendId <= 0) { + return true; + } + Backend backend = Env.getCurrentSystemInfo().getBackend(runningBackendId); + if (backend == null) { + return false; + } + try { + JobBaseConfig releaseParams = new JobBaseConfig( + String.valueOf(getJobId()), dataSourceType.name(), sourceProperties, getFrontendAddress()); + InternalService.PRequestCdcClientRequest request = InternalService.PRequestCdcClientRequest.newBuilder() + .setApi("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/api/releaseReader/" + getTaskId()) + .setParams(new Gson().toJson(releaseParams)).build(); + TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort()); + PRequestCdcClientResult result = BackendServiceProxy.getInstance() + .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec) + .get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS); + ResponseBody response = objectMapper.readValue( + result.getResponse(), new TypeReference>() {}); + remoteReaderReleased = TStatusCode.findByValue(result.getStatus().getStatusCode()) == TStatusCode.OK + && response.getCode() == RestApiStatusCode.OK.code; + return remoteReaderReleased; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (Exception e) { + log.warn("Wait for reader release failed, job {} task {}", getJobId(), getTaskId(), e); + return false; + } + } + private String getFrontendAddress() { return Env.getCurrentEnv().getMasterHost() + ":" + Env.getCurrentEnv().getMasterHttpPort(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java index 82f3020b64d67b..e7a110f85022ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java @@ -281,9 +281,14 @@ public void alterJobStatus(String jobName, JobStatus jobStatus, FailureReason re try { checkSameStatus(a, jobStatus); if (a instanceof StreamingInsertJob) { - ((StreamingInsertJob) a).onManualStatusAltered(jobStatus, reason); + ((StreamingInsertJob) a).updateManualJobStatus(jobStatus, reason); + if (statusNeedsScheduling(jobStatus)) { + jobScheduler.cycleTimerJobScheduler(a); + } + a.logUpdateOperation(); + } else { + alterJobStatus(a.getJobId(), jobStatus); } - alterJobStatus(a.getJobId(), jobStatus); } catch (JobException e) { throw new JobException("Alter job status error, jobName is %s, errorMsg is %s", jobName, e.getMessage()); @@ -292,6 +297,10 @@ public void alterJobStatus(String jobName, JobStatus jobStatus, FailureReason re } } + private boolean statusNeedsScheduling(JobStatus status) { + return status.equals(JobStatus.RUNNING); + } + private void checkSameStatus(T a, JobStatus newStatus) throws JobException { if (newStatus.equals(a.getJobStatus())) { throw new JobException("Can't change job status to the same status"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index de8f38d89214fb..0a1dc88fa4d523 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -959,7 +959,46 @@ public void setDatabase(String db) { } public void setExecutor(StmtExecutor executor) { - this.executor = executor; + boolean cancelAfterPublication = false; + Status deferredCancelReason; + synchronized (executorPublicationLock) { + if (connectType == ConnectType.ARROW_FLIGHT_SQL) { + synchronized (flightSqlDeferredExecutors) { + this.executor = executor; + cancelAfterPublication = flightSqlDeferredExecutorsSealed; + } + } else { + this.executor = executor; + } + deferredCancelReason = pendingExecutorCancelReason; + pendingExecutorCancelReason = null; + } + if (executor != null) { + if (deferredCancelReason != null) { + executor.cancel(deferredCancelReason); + } + if (cancelAfterPublication) { + executor.cancel(new Status(TStatusCode.CANCELLED, "arrow flight connection closed"), false); + } + } + } + + private final Object executorPublicationLock = new Object(); + private Status pendingExecutorCancelReason; + + /** Preserve a forwarded-query cancel until proxyExecute publishes its StmtExecutor. */ + public void cancelQueryOnExecutorPublication(Status cancelReason) { + StmtExecutor executorRef; + synchronized (executorPublicationLock) { + executorRef = executor; + if (executorRef == null) { + if (pendingExecutorCancelReason == null) { + pendingExecutorCancelReason = cancelReason; + } + return; + } + } + executorRef.cancel(cancelReason); } public StmtExecutor getExecutor() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 13e14e6ff3eb48..11f2105d1fa997 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -143,6 +143,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutableTriple; @@ -784,6 +785,9 @@ private boolean shouldQueue() { // A call to Exec() must precede all other member function calls. @Override public void exec() throws Exception { + if (isQueryCancelled()) { + throw new UserException("Query was cancelled before execution"); + } // LoadTask does not have context, not controlled by queue now if (context != null) { if (Config.enable_workload_group) { @@ -916,7 +920,13 @@ protected void execInternal() throws Exception { protected void sendPipelineCtx() throws Exception { lock(); + boolean lockHeld = true; try { + // Linearize fragment dispatch with cancel(): cancel either publishes its status before this + // admission check, or waits for dispatch publication and then cancels the remote fragments. + if (queryStatus.isCancelled()) { + throw new UserException("Query was cancelled before fragment dispatch"); + } Multiset hostCounter = HashMultiset.create(); for (FragmentExecParams params : fragmentExecParamsMap.values()) { for (FInstanceExecParam fi : params.instanceExecParams) { @@ -1047,6 +1057,11 @@ protected void sendPipelineCtx() throws Exception { updateProfileIfPresent(profile -> profile.updateFragmentCompressedSize(compressedSize.get())); updateProfileIfPresent(profile -> profile.setFragmentSerializeTime()); + // All cancel-visible per-backend contexts are now published. Do not retain the coordinator lock + // while waiting for remote RPCs: each PipelineExecContexts linearizes its own send with cancel. + unlock(); + lockHeld = false; + // 4.2 send fragments rpc List>>> futures = Lists.newArrayList(); @@ -1080,7 +1095,9 @@ protected void sendPipelineCtx() throws Exception { updateProfileIfPresent(profile -> profile.setRpcPhase2Latency(rpcPhase2Latency)); } } finally { - unlock(); + if (lockHeld) { + unlock(); + } } } @@ -3162,6 +3179,9 @@ public static class PipelineExecContexts { ByteString serializedFragments = null; boolean hasCancelled = false; boolean cancelInProcess = false; + boolean cancelRequested = false; + ListenableFuture phaseOneFuture; + boolean deferredCancelScheduled = false; public PipelineExecContexts(TUniqueId queryId, Backend backend, TNetworkAddress brpcAddr, boolean twoPhaseExecution, @@ -3206,11 +3226,16 @@ public void unsetFields() { } } - public Future execRemoteFragmentsAsync(BackendServiceProxy proxy) + public synchronized Future execRemoteFragmentsAsync( + BackendServiceProxy proxy) throws TException { + if (cancelRequested) { + throw new TException("Query cancelled before fragment dispatch"); + } Preconditions.checkNotNull(serializedFragments); try { - return proxy.execPlanFragmentsAsync(brpcAddr, serializedFragments, twoPhaseExecution); + phaseOneFuture = proxy.execPlanFragmentsAsync(brpcAddr, serializedFragments, twoPhaseExecution); + return phaseOneFuture; } catch (RpcException e) { // DO NOT throw exception here, return a complete future with error code, // so that the following logic will cancel the fragment. @@ -3218,8 +3243,12 @@ public Future execRemoteFragmentsAsync( } } - public Future execPlanFragmentStartAsync(BackendServiceProxy proxy) + public synchronized Future execPlanFragmentStartAsync( + BackendServiceProxy proxy) throws TException { + if (cancelRequested) { + throw new TException("Query cancelled before fragment start"); + } try { PExecPlanFragmentStartRequest.Builder builder = PExecPlanFragmentStartRequest.newBuilder(); PUniqueId qid = PUniqueId.newBuilder().setHi(queryId.hi).setLo(queryId.lo).build(); @@ -3296,12 +3325,41 @@ public String debugInfo() { // Just send the cancel message to BE, not care about the result, because there is no retry // logic in upper logic. private synchronized void cancelQuery(Status cancelReason) { + cancelRequested = true; + scheduleCancelAfterPhaseOne(cancelReason); + cancelQueryInternal(cancelReason, false); + } + + private void scheduleCancelAfterPhaseOne(Status cancelReason) { + if (phaseOneFuture == null || deferredCancelScheduled) { + return; + } + deferredCancelScheduled = true; + Futures.addCallback(phaseOneFuture, new FutureCallback() { + @Override + public void onSuccess(PExecPlanFragmentResult result) { + replayCancelAfterPhaseOne(cancelReason); + } + + @Override + public void onFailure(Throwable t) { + LOG.debug("Phase-one fragment dispatch completed exceptionally before deferred cancel", t); + replayCancelAfterPhaseOne(cancelReason); + } + }, MoreExecutors.directExecutor()); + } + + private synchronized void replayCancelAfterPhaseOne(Status cancelReason) { + cancelQueryInternal(cancelReason, true); + } + + private void cancelQueryInternal(Status cancelReason, boolean force) { if (LOG.isDebugEnabled()) { LOG.debug("cancelRemoteFragments backend: {}, query={}, reason: {}", backend, DebugUtil.printId(queryId), cancelReason.toString()); } - if (this.hasCancelled || this.cancelInProcess) { + if (!force && (this.hasCancelled || this.cancelInProcess)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java index 067db673049c79..8556ac5967bc1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java @@ -35,6 +35,9 @@ public class MasterOpExecutor extends FEOpExecutor { private static final Logger LOG = LogManager.getLogger(MasterOpExecutor.class); private final int journalWaitTimeoutMs; + private final Object executionAdmissionLock = new Object(); + private boolean executionStarted; + private boolean cancellationRequested; public MasterOpExecutor(OriginStatement originStmt, ConnectContext ctx, RedirectStatus status, boolean isQuery) { super(new TNetworkAddress(ctx.getEnv().getMasterHost(), ctx.getEnv().getMasterRpcPort()), @@ -55,12 +58,25 @@ public MasterOpExecutor(ConnectContext ctx) { @Override public void execute() throws Exception { + synchronized (executionAdmissionLock) { + if (cancellationRequested) { + ctx.getState().setError("forward operation cancelled"); + return; + } + executionStarted = true; + } super.execute(); waitOnReplaying(); } @Override public void cancel() throws Exception { + synchronized (executionAdmissionLock) { + cancellationRequested = true; + if (!executionStarted) { + return; + } + } super.cancel(); waitOnReplaying(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index f8c7509f102678..4a1f02b4c6758f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -139,6 +139,9 @@ public NereidsCoordinator(Long jobId, TUniqueId queryId, DescriptorTable descTab @Override public void exec() throws Exception { + if (isQueryCancelled()) { + throw new UserException("Query was cancelled before execution"); + } enqueue(coordinatorContext.connectContext); processTopSink(coordinatorContext, coordinatorContext.topDistributedPlan); @@ -147,7 +150,14 @@ public void exec() throws Exception { Map workerToFragments = ThriftPlansBuilder.plansToThrift(coordinatorContext); - executionTask = PipelineExecutionTaskBuilder.build(coordinatorContext, workerToFragments); + executionTask = coordinatorContext.withLock(() -> { + if (coordinatorContext.readCloneStatus().isCancelled()) { + throw new UserException("Query was cancelled before fragment dispatch"); + } + // Publish under the cancel monitor. Per-backend task admission then linearizes each RPC with cancel, + // without holding the context monitor while waiting for a remote response. + return PipelineExecutionTaskBuilder.build(coordinatorContext, workerToFragments); + }); executionTask.execute(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 9abe85ea2f0f99..3f94995bd91af3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -144,7 +144,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.protobuf.ByteString; -import lombok.Setter; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -189,14 +188,17 @@ public class StmtExecutor { private List> changedSessionVarsForAudit; private ProfileType profileType = ProfileType.QUERY; - @Setter private volatile Coordinator coord = null; // Arrow Flight SQL: when true, this query's coordinator is kept alive past GetFlightInfo and // is finalized later by ConnectContext (see #62259), so the eager close in executeAndSendResult // is skipped. private volatile boolean deferredForArrowFlight = false; private Closeable deferredArrowFlightStatementResources; - private MasterOpExecutor masterOpExecutor = null; + private volatile MasterOpExecutor masterOpExecutor = null; + // Cancellation can arrive after executor publication but before execution resources exist. + // Retain it so execution admission and later coordinator publication cannot lose the signal. + private volatile Status pendingCancelReason = null; + private final Object executionAdmissionLock = new Object(); private RedirectStatus redirectStatus = null; private Planner planner; private boolean isProxy; @@ -505,6 +507,12 @@ public boolean isCached() { // query with a random sql public void execute() throws Exception { + synchronized (executionAdmissionLock) { + if (pendingCancelReason != null) { + context.getState().setError(pendingCancelReason.getErrorMsg()); + return; + } + } UUID uuid = UUID.randomUUID(); TUniqueId queryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); if (Config.enable_print_request_before_execution) { @@ -1212,6 +1220,11 @@ public boolean isProfileSafeStmt() { private void forwardToMaster() throws Exception { masterOpExecutor = new MasterOpExecutor(originStmt, context, redirectStatus, isQuery()); + if (pendingCancelReason != null) { + masterOpExecutor.cancel(); + context.getState().setError(pendingCancelReason.getErrorMsg()); + return; + } if (LOG.isDebugEnabled()) { LOG.debug("need to transfer to Master. stmt: {}", context.getStmtId()); } @@ -1246,6 +1259,9 @@ public void updateProfile(boolean isFinished) { // Because this is called by other thread public void cancel(Status cancelReason, boolean needWaitCancelComplete) { + synchronized (executionAdmissionLock) { + pendingCancelReason = cancelReason; + } if (masterOpExecutor != null) { try { masterOpExecutor.cancel(); @@ -1276,6 +1292,14 @@ public void cancel(Status cancelReason) { cancel(cancelReason, true); } + public void setCoord(Coordinator coordinator) { + this.coord = coordinator; + Status cancelReason = pendingCancelReason; + if (coordinator != null && cancelReason != null) { + coordinator.cancel(cancelReason); + } + } + private Optional getInsertOverwriteTableCommand() { if (parsedStmt instanceof LogicalPlanAdapter) { LogicalPlanAdapter logicalPlanAdapter = (LogicalPlanAdapter) parsedStmt; @@ -1445,15 +1469,15 @@ public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, context.getSessionVariable().getMaxMsgSizeOfResultReceiver()); context.getState().setIsQuery(true); } else if (planner instanceof NereidsPlanner && ((NereidsPlanner) planner).getDistributedPlans() != null) { - coord = new NereidsCoordinator(context, - (NereidsPlanner) planner, context.getStatsErrorEstimator()); + setCoord(new NereidsCoordinator(context, + (NereidsPlanner) planner, context.getStatsErrorEstimator())); profile.addExecutionProfile(coord.getExecutionProfile()); QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), new QueryInfo(context, originStmt.originStmt, coord)); coordBase = coord; } else { - coord = EnvFactory.getInstance().createCoordinator( - context, planner, context.getStatsErrorEstimator()); + setCoord(EnvFactory.getInstance().createCoordinator( + context, planner, context.getStatsErrorEstimator())); profile.addExecutionProfile(coord.getExecutionProfile()); QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), new QueryInfo(context, originStmt.originStmt, coord)); @@ -2153,8 +2177,8 @@ public List executeInternalQuery() { if (Config.enable_collect_internal_query_profile) { context.getSessionVariable().enableProfile = true; } - coord = EnvFactory.getInstance().createCoordinator(context, - planner, context.getStatsErrorEstimator()); + setCoord(EnvFactory.getInstance().createCoordinator(context, + planner, context.getStatsErrorEstimator())); profile.addExecutionProfile(coord.getExecutionProfile()); try { QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java index f5a77d79196435..e249ed82f57b05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java @@ -39,6 +39,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,7 +47,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -64,6 +64,9 @@ public class MultiFragmentsPipelineTask extends AbstractRuntimeTask phaseOneFuture; + private boolean deferredCancelScheduled; public MultiFragmentsPipelineTask( CoordinatorContext coordinatorContext, Backend backend, BackendServiceProxy backendClientProxy, @@ -78,15 +81,23 @@ public MultiFragmentsPipelineTask( ); this.hasCancelled = new AtomicBoolean(); this.cancelInProcess = new AtomicBoolean(); + this.cancelRequested = new AtomicBoolean(); } - public Future sendPhaseOneRpc(boolean twoPhaseExecution) { - return execRemoteFragmentsAsync( + public synchronized Future sendPhaseOneRpc(boolean twoPhaseExecution) { + if (cancelRequested.get()) { + return futureWithStatus(TStatusCode.CANCELLED, "Query cancelled before fragment dispatch"); + } + phaseOneFuture = execRemoteFragmentsAsync( backendClientProxy, serializeFragments, backend.getBrpcAddress(), twoPhaseExecution ); + return phaseOneFuture; } - public Future sendPhaseTwoRpc() { + public synchronized Future sendPhaseTwoRpc() { + if (cancelRequested.get()) { + return futureWithStatus(TStatusCode.CANCELLED, "Query cancelled before fragment start"); + } return execPlanFragmentStartAsync(backendClientProxy, backend.getBrpcAddress()); } @@ -102,13 +113,42 @@ public String toString() { } public synchronized void cancelExecute(Status cancelReason) { + cancelRequested.set(true); + scheduleCancelAfterPhaseOne(cancelReason); + cancelExecuteInternal(cancelReason, false); + } + + private void scheduleCancelAfterPhaseOne(Status cancelReason) { + if (phaseOneFuture == null || deferredCancelScheduled) { + return; + } + deferredCancelScheduled = true; + Futures.addCallback(phaseOneFuture, new FutureCallback() { + @Override + public void onSuccess(PExecPlanFragmentResult result) { + replayCancelAfterPhaseOne(cancelReason); + } + + @Override + public void onFailure(Throwable t) { + LOG.debug("Phase-one fragment dispatch completed exceptionally before deferred cancel", t); + replayCancelAfterPhaseOne(cancelReason); + } + }, MoreExecutors.directExecutor()); + } + + private synchronized void replayCancelAfterPhaseOne(Status cancelReason) { + cancelExecuteInternal(cancelReason, true); + } + + private void cancelExecuteInternal(Status cancelReason, boolean force) { TUniqueId queryId = coordinatorContext.queryId; if (LOG.isDebugEnabled()) { LOG.debug("cancelRemoteFragments backend: {}, query={}, reason: {}", backend, DebugUtil.printId(queryId), cancelReason.toString()); } - if (this.hasCancelled.get() || this.cancelInProcess.get()) { + if (!force && (this.hasCancelled.get() || this.cancelInProcess.get())) { LOG.info("Fragment has already been cancelled. Query {} backend: {}", DebugUtil.printId(queryId), backend); return; @@ -160,7 +200,7 @@ public Backend getBackend() { return backend; } - private Future execRemoteFragmentsAsync( + private ListenableFuture execRemoteFragmentsAsync( BackendServiceProxy proxy, ByteString serializedFragments, TNetworkAddress brpcAddr, boolean twoPhaseExecution) { Preconditions.checkNotNull(serializedFragments); @@ -191,35 +231,14 @@ public Future execPlanFragmentStartAsyn } } - private Future futureWithException(RpcException e) { - return new Future() { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public PExecPlanFragmentResult get() { - PExecPlanFragmentResult result = PExecPlanFragmentResult.newBuilder().setStatus( - Types.PStatus.newBuilder().addErrorMsgs(e.getMessage()) - .setStatusCode(TStatusCode.THRIFT_RPC_ERROR.getValue()).build()).build(); - return result; - } + private ListenableFuture futureWithException(RpcException e) { + return futureWithStatus(TStatusCode.THRIFT_RPC_ERROR, e.getMessage()); + } - @Override - public PExecPlanFragmentResult get(long timeout, TimeUnit unit) { - return get(); - } - }; + private ListenableFuture futureWithStatus(TStatusCode statusCode, String message) { + PExecPlanFragmentResult result = PExecPlanFragmentResult.newBuilder().setStatus( + Types.PStatus.newBuilder().addErrorMsgs(message) + .setStatusCode(statusCode.getValue()).build()).build(); + return Futures.immediateFuture(result); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java index 6a52e3a6d9f855..a374240d63b446 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java @@ -92,13 +92,10 @@ public PipelineExecutionTask( @Override public void execute() throws Exception { - coordinatorContext.withLock(() -> { - sendAndWaitPhaseOneRpc(); - if (coordinatorContext.twoPhaseExecution()) { - sendAndWaitPhaseTwoRpc(); - } - return null; - }); + sendAndWaitPhaseOneRpc(); + if (coordinatorContext.twoPhaseExecution()) { + sendAndWaitPhaseTwoRpc(); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java index 13489e1894fa4d..256d676182f3b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java @@ -65,13 +65,13 @@ public boolean isUsingLatestChannelConfig() { return channelConfigVersion == CHANNEL_PROVIDER.currentConfigVersion(); } - public Future execPlanFragmentAsync( + public ListenableFuture execPlanFragmentAsync( InternalService.PExecPlanFragmentRequest request) { return stub.withDeadlineAfter(execPlanTimeout, TimeUnit.MILLISECONDS) .execPlanFragment(request); } - public Future execPlanFragmentPrepareAsync( + public ListenableFuture execPlanFragmentPrepareAsync( InternalService.PExecPlanFragmentRequest request) { return stub.withDeadlineAfter(execPlanTimeout, TimeUnit.MILLISECONDS) .execPlanFragmentPrepare(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java index 15f35d3711721d..4682957a59c0d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java @@ -174,7 +174,7 @@ private BackendServiceClient getProxy(TNetworkAddress address) throws UnknownHos } } - public Future execPlanFragmentsAsync(TNetworkAddress address, + public ListenableFuture execPlanFragmentsAsync(TNetworkAddress address, TPipelineFragmentParamsList params, boolean twoPhaseExecution) throws TException, RpcException { InternalService.PExecPlanFragmentRequest.Builder builder = InternalService.PExecPlanFragmentRequest.newBuilder(); @@ -192,7 +192,7 @@ public Future execPlanFragmentsAsync(TN return execPlanFragmentsAsync(address, builder.build(), twoPhaseExecution); } - public Future execPlanFragmentsAsync(TNetworkAddress address, + public ListenableFuture execPlanFragmentsAsync(TNetworkAddress address, ByteString serializedFragments, boolean twoPhaseExecution) throws RpcException { InternalService.PExecPlanFragmentRequest.Builder builder = InternalService.PExecPlanFragmentRequest.newBuilder(); @@ -203,7 +203,7 @@ public Future execPlanFragmentsAsync(TN return execPlanFragmentsAsync(address, builder.build(), twoPhaseExecution); } - public Future execPlanFragmentsAsync(TNetworkAddress address, + public ListenableFuture execPlanFragmentsAsync(TNetworkAddress address, InternalService.PExecPlanFragmentRequest pRequest, boolean twoPhaseExecution) throws RpcException { MetricRepo.BE_COUNTER_QUERY_RPC_ALL.getOrAdd(address.hostname).increase(1L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index e6470e2074f264..5d28ee92ea1041 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -380,6 +380,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { private final Map proxyQueryIdToConnCtx = new ConcurrentHashMap<>(64); + private final Map pendingProxyQueryCancels = new ConcurrentHashMap<>(64); + private static final long PENDING_PROXY_CANCEL_TTL_MS = 5 * 60 * 1000L; private static TNetworkAddress getMasterAddress() { Env env = Env.getCurrentEnv(); @@ -1181,7 +1183,18 @@ public TMasterOpResult forward(TMasterOpRequest params) throws TException { TUniqueId queryId = params.getQueryId(); ConnectContext ctx = proxyQueryIdToConnCtx.get(queryId); if (ctx != null) { - ctx.cancelQuery(new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); + ctx.cancelQueryOnExecutorPublication( + new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); + } else { + long now = System.currentTimeMillis(); + pendingProxyQueryCancels.entrySet().removeIf( + entry -> now - entry.getValue() > PENDING_PROXY_CANCEL_TTL_MS); + pendingProxyQueryCancels.put(queryId, now); + ctx = proxyQueryIdToConnCtx.get(queryId); + if (ctx != null && pendingProxyQueryCancels.remove(queryId) != null) { + ctx.cancelQueryOnExecutorPublication( + new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); + } } final TMasterOpResult result = new TMasterOpResult(); result.setStatusCode(0); @@ -1214,6 +1227,10 @@ public TMasterOpResult forward(TMasterOpRequest params) throws TException { Runnable clearCallback = () -> {}; if (params.isSetQueryId()) { proxyQueryIdToConnCtx.put(params.getQueryId(), context); + if (pendingProxyQueryCancels.remove(params.getQueryId()) != null) { + context.cancelQueryOnExecutorPublication( + new Status(TStatusCode.CANCELLED, "cancel query before forward registration.")); + } clearCallback = () -> proxyQueryIdToConnCtx.remove(params.getQueryId()); } TMasterOpResult result = processor.proxyExecute(params); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java index 9e6a02854497f0..e5b339f11eb128 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java @@ -264,7 +264,7 @@ public void testGetComment() { IcebergExternalTable spy = createSpyTable(); Map properties = Maps.newHashMap(); properties.put("comment", "my-table-comment"); - Mockito.when(icebergTable.properties()).thenReturn(properties); + Mockito.doReturn(properties).when(spy).properties(); Assertions.assertEquals("my-table-comment", spy.getComment()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java index 66c0679d1b1178..7e4d85cb930ace 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.job.cdc.request.CommitOffsetRequest; +import org.apache.doris.job.common.FailureReason; import org.apache.doris.job.common.JobStatus; import org.apache.doris.job.common.TaskStatus; import org.apache.doris.job.offset.jdbc.JdbcSourceOffsetProvider; @@ -117,4 +118,76 @@ public void testCommitOffsetSkipsCanceledTask() throws Exception { Assert.assertEquals("task status must stay terminal — late callback ignored", TaskStatus.FAILED, task.getStatus()); } + + @Test + public void testInvalidManualTransitionDoesNotMutateFailureReason() throws Exception { + StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); + Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); + Deencapsulation.setField(job, "jobStatus", JobStatus.STOPPED); + FailureReason original = new FailureReason("terminal failure"); + Deencapsulation.setField(job, "failureReason", original); + + Assert.assertThrows(org.apache.doris.job.exception.JobException.class, + () -> job.updateManualJobStatus(JobStatus.RUNNING, null)); + Assert.assertSame(original, job.getFailureReason()); + } + + @Test + public void testPredecessorBlocksSuccessorSchedulingUntilTerminalHandoff() { + StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); + Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); + Deencapsulation.setField(job, "jobStatus", JobStatus.PENDING); + StreamingMultiTblTask predecessor = newTask(9002L, TaskStatus.CANCELED); + Deencapsulation.setField(job, "runningStreamTask", predecessor); + + Assert.assertFalse(job.isReadyForScheduling(new HashMap<>())); + job.clearRunningStreamTask(predecessor); + Assert.assertTrue(job.isReadyForScheduling(new HashMap<>())); + } + + @Test + public void testManualPauseResumeClearsMultiTaskAndReentersPending() throws Exception { + StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); + Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); + Deencapsulation.setField(job, "jobId", 9004L); + Deencapsulation.setField(job, "jobStatus", JobStatus.RUNNING); + StreamingMultiTblTask predecessor = newTask(9003L, TaskStatus.RUNNING); + Deencapsulation.setField(job, "runningStreamTask", predecessor); + + job.updateManualJobStatus(JobStatus.PAUSED, new FailureReason("manual pause")); + Assert.assertFalse(job.hasRunningStreamTask()); + job.updateManualJobStatus(JobStatus.RUNNING, null); + + Assert.assertEquals(JobStatus.PENDING, job.getJobStatus()); + Assert.assertTrue(job.isReadyForScheduling(new HashMap<>())); + } + + @Test + public void testManualStopPublishesCancellationAndClearsTask() throws Exception { + StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); + Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); + Deencapsulation.setField(job, "jobId", 9004L); + Deencapsulation.setField(job, "jobStatus", JobStatus.RUNNING); + StreamingMultiTblTask predecessor = newTask(9004L, TaskStatus.RUNNING); + Deencapsulation.setField(job, "runningStreamTask", predecessor); + + job.updateManualJobStatus(JobStatus.STOPPED, new FailureReason("manual stop")); + job.onStreamTaskSuccess(predecessor); + + Assert.assertEquals(JobStatus.STOPPED, job.getJobStatus()); + Assert.assertTrue(predecessor.getIsCanceled().get()); + Assert.assertFalse(job.hasRunningStreamTask()); + } + + @Test + public void testStalePendingTickCannotPublishAfterManualTransition() throws Exception { + StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); + Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); + Deencapsulation.setField(job, "jobStatus", JobStatus.PAUSED); + Deencapsulation.setField(job, "statusEpoch", 2L); + + Assert.assertFalse(job.dispatchPendingTask(1L)); + Assert.assertEquals(JobStatus.PAUSED, job.getJobStatus()); + Assert.assertFalse(job.hasRunningStreamTask()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index 984fd8b9559a86..f1116798083e08 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -30,6 +30,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.Pair; +import org.apache.doris.common.Status; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.MysqlCapability; @@ -39,6 +40,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.transaction.TransactionStatus; @@ -949,6 +951,29 @@ public void testSessionTeardownRejectsLateDeferredExecutorRegistration() { Mockito.verifyNoInteractions(late); } + @Test + public void testFlightSealReplaysCancelWhenExecutorPublishesLate() { + ConnectContext ctx = new ConnectContext(); + ctx.connectType = ConnectContext.ConnectType.ARROW_FLIGHT_SQL; + StmtExecutor lateExecutor = Mockito.mock(StmtExecutor.class); + + ctx.sealFlightSqlDeferredExecutors(); + ctx.setExecutor(lateExecutor); + + Mockito.verify(lateExecutor).cancel(Mockito.any(), Mockito.eq(false)); + } + + @Test + public void testForwardCancelReplayedWhenExecutorPublishesLate() { + ConnectContext ctx = new ConnectContext(); + StmtExecutor lateExecutor = Mockito.mock(StmtExecutor.class); + + ctx.cancelQueryOnExecutorPublication(new Status(TStatusCode.CANCELLED, "forward cancel")); + ctx.setExecutor(lateExecutor); + + Mockito.verify(lateExecutor).cancel(Mockito.any(Status.class)); + } + @Test public void testSessionSealWaitsForAdmittedResultPublisher() throws Exception { ConnectContext ctx = new ConnectContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java new file mode 100644 index 00000000000000..a93702945a3226 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.qe.QueryState.MysqlStateType; +import org.apache.doris.thrift.TUniqueId; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class StmtExecutorCancellationTest { + + @Test + public void testFlightLateExecutorCancellationPreventsExecutionAdmission() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.connectType = ConnectContext.ConnectType.ARROW_FLIGHT_SQL; + ctx.sealAndCloseFlightSqlDeferredExecutors(); + AtomicBoolean admitted = new AtomicBoolean(); + StmtExecutor executor = new StmtExecutor(ctx, Mockito.mock(StatementBase.class)) { + @Override + public void queryRetry(TUniqueId queryId) { + admitted.set(true); + } + }; + + ctx.setExecutor(executor); + executor.execute(); + + Assert.assertFalse(admitted.get()); + Assert.assertEquals(MysqlStateType.ERR, ctx.getState().getStateType()); + } +} From 8c4f469b4772157d06989c69458dede7c1830153 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Sun, 23 Aug 2026 22:18:33 +0800 Subject: [PATCH 17/38] [fix](external) narrow resource cleanup scope --- .../doris/datasource/SplitAssignment.java | 25 +- .../datasource/hive/source/HiveScanNode.java | 39 +-- .../iceberg/source/IcebergScanNode.java | 26 +- .../streaming/AbstractStreamingTask.java | 124 +------ .../insert/streaming/StreamingInsertJob.java | 307 +++--------------- .../insert/streaming/StreamingInsertTask.java | 68 +--- .../streaming/StreamingJobSchedulerTask.java | 26 +- .../streaming/StreamingMultiTblTask.java | 69 +--- .../apache/doris/job/manager/JobManager.java | 13 +- .../job/scheduler/StreamingTaskScheduler.java | 2 - .../java/org/apache/doris/mtmv/MTMVCache.java | 107 +++--- .../doris/nereids/StatementContext.java | 57 ---- .../trees/plans/commands/ExecuteCommand.java | 3 - .../doris/plsql/executor/DorisRowResult.java | 49 +-- .../plsql/executor/PlsqlQueryExecutor.java | 22 +- .../doris/qe/AutoCloseConnectContext.java | 17 +- .../org/apache/doris/qe/ConnectContext.java | 141 +------- .../org/apache/doris/qe/ConnectProcessor.java | 17 +- .../java/org/apache/doris/qe/Coordinator.java | 68 +--- .../org/apache/doris/qe/MasterOpExecutor.java | 16 - .../doris/qe/MysqlConnectProcessor.java | 2 +- .../apache/doris/qe/NereidsCoordinator.java | 12 +- .../org/apache/doris/qe/StmtExecutor.java | 104 +----- .../runtime/MultiFragmentsPipelineTask.java | 89 ++--- .../qe/runtime/PipelineExecutionTask.java | 11 +- .../doris/rpc/BackendServiceClient.java | 4 +- .../apache/doris/rpc/BackendServiceProxy.java | 6 +- .../doris/service/FrontendServiceImpl.java | 19 +- .../arrowflight/DorisFlightSqlProducer.java | 34 +- .../sessions/FlightSqlConnectPoolMgr.java | 9 +- .../doris/datasource/SplitAssignmentTest.java | 11 + .../hive/source/HiveScanNodeTest.java | 4 +- .../StreamingInsertJobLateCallbackTest.java | 73 ----- ...reamingInsertJobOffsetPersistenceTest.java | 34 -- .../StreamingInsertTaskResourceTest.java | 235 -------------- .../doris/nereids/StatementContextTest.java | 32 -- .../doris/nereids/mv/MTMVCacheTest.java | 12 - .../plans/commands/ExecuteCommandTest.java | 10 - .../plsql/executor/DorisRowResultTest.java | 69 ---- .../doris/qe/AutoCloseConnectContextTest.java | 52 --- .../apache/doris/qe/ConnectContextTest.java | 76 ----- .../qe/StmtExecutorCancellationTest.java | 51 --- .../org/apache/doris/qe/StmtExecutorTest.java | 42 --- .../DorisFlightSqlProducerTest.java | 101 ------ .../sessions/FlightSqlConnectPoolMgrTest.java | 19 +- 45 files changed, 296 insertions(+), 2011 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java index 5f79a006a7af11..d2bcd935e2294e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java @@ -191,11 +191,16 @@ public void finishSchedule() { } public void stop() { - if (isStop()) { - return; + List resources; + synchronized (closeableResources) { + if (isStop()) { + return; + } + isStopped.set(true); + resources = new ArrayList<>(closeableResources); + closeableResources.clear(); } - isStopped.set(true); - closeableResources.forEach((closeable) -> { + resources.forEach((closeable) -> { try { closeable.close(); } catch (Exception e) { @@ -214,6 +219,16 @@ public boolean isStop() { } public void addCloseable(Closeable resource) { - closeableResources.add(resource); + synchronized (closeableResources) { + if (!isStop()) { + closeableResources.add(resource); + return; + } + } + try { + resource.close(); + } catch (Exception e) { + LOG.warn("close resource registered after stop error:{}", e.getMessage(), e); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 190f3024c6e17f..1e45cbb3f7c2e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -74,7 +74,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -94,7 +93,6 @@ public class HiveScanNode extends FileQueryScanNode { private static final Logger LOG = LogManager.getLogger(HiveScanNode.class); protected final HMSExternalTable hmsTable; - private final long hmsRuntimeGeneration; private HiveTransaction hiveTransaction = null; // will only be set in Nereids, for lagency planner, it should be null @@ -127,7 +125,6 @@ public HiveScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, DirectoryLister directoryLister, ScanContext scanContext) { super(id, desc, planNodeName, statisticalType, scanContext, needCheckColumnPriv, sv); hmsTable = (HMSExternalTable) desc.getTable(); - hmsRuntimeGeneration = ((HMSExternalCatalog) hmsTable.getCatalog()).getRuntimeGeneration(); brokerName = hmsTable.getCatalog().bindBrokerName(); this.directoryLister = directoryLister; } @@ -139,7 +136,6 @@ public void setSelectedPartitions(SelectedPartitions selectedPartitions) { @Override protected void doInitialize() throws UserException { - ensureHmsRuntimeGeneration(); super.doInitialize(); if (hmsTable.isHiveTransactionalTable()) { @@ -147,24 +143,8 @@ protected void doInitialize() throws UserException { this.hiveTransaction = new HiveTransaction(DebugUtil.printId(ConnectContext.get().queryId()), ConnectContext.get().getQualifiedUser(), hmsTable, hmsTable.isFullAcidTable()); Env.getCurrentHiveTransactionMgr().register(hiveTransaction); - try { - StatementContext statementContext = ConnectContext.get().getStatementContext(); - String queryId = hiveTransaction.getQueryId(); - statementContext.getOrRegisterStatementResource("hive-transaction:" + queryId, - () -> (Closeable) () -> Env.getCurrentHiveTransactionMgr().deregister(queryId)); - } catch (RuntimeException | Error e) { - Env.getCurrentHiveTransactionMgr().deregister(hiveTransaction.getQueryId()); - throw e; - } skipCheckingAcidVersionFile = sessionVariable.skipCheckingAcidVersionFile; } - ensureHmsRuntimeGeneration(); - } - - private void ensureHmsRuntimeGeneration() { - if (((HMSExternalCatalog) hmsTable.getCatalog()).getRuntimeGeneration() != hmsRuntimeGeneration) { - throw new IllegalStateException("HMS catalog properties changed while planning the Hive scan; retry"); - } } static void markTransactionalHiveScanParams(TFileScanRangeParams scanParams) { @@ -176,7 +156,6 @@ static void markTransactionalHiveScanParams(TFileScanRangeParams scanParams) { } protected List getPartitions() throws AnalysisException { - ensureHmsRuntimeGeneration(); long startTime = System.currentTimeMillis(); List resPartitions = Lists.newArrayList(); try { @@ -215,7 +194,6 @@ protected List getPartitions() throws AnalysisException { getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); getSummaryProfile().setGetPartitionsFinishTime(); } - ensureHmsRuntimeGeneration(); return resPartitions; } catch (RuntimeException e) { if (getSummaryProfile() != null) { @@ -227,7 +205,6 @@ protected List getPartitions() throws AnalysisException { @Override public List getSplits(int numBackends) throws UserException { - ensureHmsRuntimeGeneration(); long start = System.currentTimeMillis(); try { if (!partitionInit) { @@ -247,7 +224,6 @@ public List getSplits(int numBackends) throws UserException { allFiles.size(), hmsTable.getDbName(), hmsTable.getName(), (System.currentTimeMillis() - start)); } - ensureHmsRuntimeGeneration(); return allFiles; } catch (Throwable t) { LOG.warn("get file split failed for table: {}", hmsTable.getName(), t); @@ -259,7 +235,6 @@ public List getSplits(int numBackends) throws UserException { @Override public void startSplit(int numBackends) { - ensureHmsRuntimeGeneration(); if (prunedPartitions.isEmpty()) { splitAssignment.finishSchedule(); return; @@ -279,7 +254,6 @@ public void startSplit(int numBackends) { splittersOnFlight.acquire(); CompletableFuture.runAsync(() -> { try { - ensureHmsRuntimeGeneration(); List allFiles = Lists.newArrayList(); getFileSplitByPartitions( cache, Collections.singletonList(partition), allFiles, bindBrokerName, @@ -288,7 +262,6 @@ public void startSplit(int numBackends) { numSplitsPerPartition.set(allFiles.size()); } if (splitAssignment.needMoreSplit()) { - ensureHmsRuntimeGeneration(); splitAssignment.addToQueue(allFiles); } } catch (Exception e) { @@ -365,7 +338,7 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List currentFileCaches = cache.getFilesByPartitions(partitions, true, partitions.size() > 1, directoryLister, hmsTable); HiveFileScanTaskCacheKey cacheKey = new HiveFileScanTaskCacheKey( - hmsTable.getCatalog().getId(), hmsTable.getId(), hmsRuntimeGeneration, partitions, + hmsTable.getCatalog().getId(), hmsTable.getId(), partitions, cache.getFileCacheInvalidationGeneration(hmsTable.getCatalog().getId()), currentFileCaches); try { fileCaches = getOrLoadExternalScanTasks(cacheKey, @@ -546,17 +519,14 @@ private static final class HiveFileScanTaskCacheKey implements ExternalScanTaskCacheKey { private final long catalogId; private final long tableId; - private final long hmsRuntimeGeneration; private final List partitions; private final long fileCacheInvalidationGeneration; private final List fileCacheValueGenerations; - private HiveFileScanTaskCacheKey(long catalogId, long tableId, long hmsRuntimeGeneration, - List partitions, + private HiveFileScanTaskCacheKey(long catalogId, long tableId, List partitions, long fileCacheInvalidationGeneration, List fileCaches) { this.catalogId = catalogId; this.tableId = tableId; - this.hmsRuntimeGeneration = hmsRuntimeGeneration; this.partitions = partitions.stream() .map(HivePartitionCacheKey::new) .collect(Collectors.toList()); @@ -577,7 +547,6 @@ public boolean equals(Object object) { HiveFileScanTaskCacheKey that = (HiveFileScanTaskCacheKey) object; return catalogId == that.catalogId && tableId == that.tableId - && hmsRuntimeGeneration == that.hmsRuntimeGeneration && fileCacheInvalidationGeneration == that.fileCacheInvalidationGeneration && fileCacheValueGenerations.equals(that.fileCacheValueGenerations) && partitions.equals(that.partitions); @@ -585,7 +554,7 @@ public boolean equals(Object object) { @Override public int hashCode() { - return Objects.hash(catalogId, tableId, hmsRuntimeGeneration, partitions, fileCacheInvalidationGeneration, + return Objects.hash(catalogId, tableId, partitions, fileCacheInvalidationGeneration, fileCacheValueGenerations); } } @@ -656,7 +625,6 @@ public TFileFormatType getFileFormatType() throws UserException { @Override protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { - ensureHmsRuntimeGeneration(); if (split instanceof HiveSplit) { HiveSplit hiveSplit = (HiveSplit) split; if (hiveSplit.isACID()) { @@ -719,7 +687,6 @@ protected List getDeleteFiles(TFileRangeDesc rangeDesc) { @Override protected Map getLocationProperties() { - ensureHmsRuntimeGeneration(); return hmsTable.getBackendStorageProperties(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d6921f9c9e2947..2a45a1b749f794 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -160,7 +160,6 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Supplier; @@ -175,7 +174,6 @@ public class IcebergScanNode extends FileQueryScanNode { private IcebergSource source; private Table icebergTable; - private ThreadPoolExecutor planningExecutor; private List pushdownIcebergPredicates = Lists.newArrayList(); // If tableLevelPushDownCount is true, means we can do count push down opt at table level. // which means all splits have no position/equality delete files, @@ -298,7 +296,6 @@ protected void doInitialize() throws UserException { getRelationSnapshot(); icebergTable = source.getIcebergTable(); icebergTable = useFrozenTableGeneration(icebergTable); - planningExecutor = getPlanningExecutor(); partitionMapInfos = new HashMap<>(); initializePartitionMetadata(); isPartitionedTable = icebergTable.spec().isPartitioned(); @@ -1757,7 +1754,7 @@ public TableScan createTableScan() throws UserException { this.pushdownIcebergPredicates.add(predicate.toString()); } - icebergTableScan = scan.planWith(planningExecutor); + icebergTableScan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); return icebergTableScan; } @@ -1802,25 +1799,6 @@ private Table useFrozenTableGeneration(Table currentTable) { return currentTable; } - private ThreadPoolExecutor getPlanningExecutor() { - Optional snapshot = getPinnedRelationSnapshot(); - if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { - ThreadPoolExecutor frozenExecutor = ((IcebergMvccSnapshot) snapshot.get()) - .getSnapshotCacheValue().getPlanningExecutor(); - if (frozenExecutor != null) { - return frozenExecutor; - } - } - TableIf targetTable = source.getTargetTable(); - if (targetTable instanceof IcebergSysExternalTable) { - targetTable = ((IcebergSysExternalTable) targetTable).getSourceTable(); - } - if (targetTable instanceof IcebergExternalTable) { - return IcebergUtils.getIcebergTableExecutor((IcebergExternalTable) targetTable); - } - return source.getCatalog().getThreadPoolWithPreAuth(); - } - @VisibleForTesting Schema getSystemTableProjectedSchema(List expressions, boolean caseSensitive) throws UserException { @@ -2723,7 +2701,7 @@ private List doGetPositionDeletesSystemTableSplits() throws UserException } long startTime = System.currentTimeMillis(); - scan = scan.planWith(planningExecutor); + scan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); BatchScan plannedScan = scan; try { positionDeleteTasks = getOrPlanPositionDeleteTasks(plannedScan, () -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java index aad6babac1df28..224fe7c5dbb610 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java @@ -59,10 +59,6 @@ public abstract class AbstractStreamingTask { protected Long finishTimeMs; @Getter private AtomicBoolean isCanceled = new AtomicBoolean(false); - private final Object executionCompletion = new Object(); - private boolean executionStarted; - private boolean executionFinished; - private Thread executionOwner; public AbstractStreamingTask(long jobId, long taskId, UserIdentity userIdentity) { this.jobId = jobId; @@ -98,114 +94,35 @@ public long getRunningBackendId() { } public void execute() throws JobException { - synchronized (executionCompletion) { - executionStarted = true; - executionOwner = Thread.currentThread(); - } - try { - while (retryCount <= MAX_RETRY) { - Exception attemptFailure = null; - boolean executionSucceeded = false; - try { - before(); - run(); - executionSucceeded = true; - } catch (Exception e) { - attemptFailure = e; - } finally { - // Only the scheduler worker that created this attempt's ConnectContext may tear it down. - // A cancelling thread waits for this handoff instead of racing before() and clearing fields - // while planning is still publishing them. - try { - closeOrReleaseResources(); - } catch (RuntimeException cleanupFailure) { - if (attemptFailure == null) { - attemptFailure = cleanupFailure; - } else { - attemptFailure.addSuppressed(cleanupFailure); - } - } - } - // A completed insert must never be replayed merely because teardown failed. Likewise, - // successor-publication failures belong to the job state machine, not to the insert retry loop. - if (executionSucceeded) { - if (attemptFailure != null) { - failCompletedAttempt(attemptFailure); - return; - } - try { - onSuccess(); - } catch (Exception completionFailure) { - failCompletedAttempt(completionFailure); - } - return; - } - if (attemptFailure == null) { - return; - } + while (retryCount <= MAX_RETRY) { + try { + before(); + run(); + onSuccess(); + return; + } catch (Exception e) { if (TaskStatus.CANCELED.equals(status)) { return; } - this.errMsg = attemptFailure.getMessage(); + this.errMsg = e.getMessage(); retryCount++; if (noRetry || retryCount > MAX_RETRY) { log.error("Task execution failed, job id {}, task id {}, noRetry {}, retry {}.", - jobId, taskId, noRetry, retryCount, attemptFailure); - onFail(attemptFailure.getMessage()); + jobId, taskId, noRetry, retryCount, e); + onFail(e.getMessage()); return; } log.warn("execute streaming task error, job id is {}, task id is {}, retrying {}/{}: {}", - jobId, taskId, retryCount, MAX_RETRY, attemptFailure.getMessage()); - } - } finally { - synchronized (executionCompletion) { - executionFinished = true; - executionOwner = null; - executionCompletion.notifyAll(); - } - onExecutionFinished(); - } - } - - protected void onExecutionFinished() { - } - - private void failCompletedAttempt(Exception failure) throws JobException { - this.errMsg = failure.getMessage(); - log.error("Completed streaming task could not publish its terminal state, job id {}, task id {}.", - jobId, taskId, failure); - onFail(failure.getMessage()); - } - - protected void awaitExecutionCompletion(long timeoutMs) { - boolean interrupted = false; - synchronized (executionCompletion) { - if (Thread.currentThread() == executionOwner) { - return; - } - long deadline = System.currentTimeMillis() + timeoutMs; - while (executionStarted && !executionFinished) { - long remaining = deadline - System.currentTimeMillis(); - if (remaining <= 0) { - break; - } - try { - executionCompletion.wait(remaining); - } catch (InterruptedException e) { - interrupted = true; + jobId, taskId, retryCount, MAX_RETRY, e.getMessage()); + } finally { + // The cancel logic will call the closeOrReleased Resources method by itself. + // If it is also called here, + // it may result in the inability to obtain relevant information when canceling the task + if (!TaskStatus.CANCELED.equals(status)) { + closeOrReleaseResources(); } } } - if (interrupted) { - Thread.currentThread().interrupt(); - } - } - - /** True when cancellation can hand off the job slot without overlapping the execution owner. */ - boolean canHandoffAfterCancellation() { - synchronized (executionCompletion) { - return !executionStarted || executionFinished; - } } protected void onFail(String errMsg) throws JobException { @@ -233,8 +150,7 @@ protected boolean isCallable() { return false; } - /** Publishes cancellation without performing task-specific RPCs or waits. */ - public void publishCancellation() { + public void cancel(boolean needWaitCancelComplete) { // Flip isCanceled even on terminal states so late BE callbacks short-circuit. if (getIsCanceled().getAndSet(true)) { return; @@ -247,10 +163,6 @@ public void publishCancellation() { this.errMsg = "task cancelled"; } - public void cancel(boolean needWaitCancelComplete) { - publishCancellation(); - } - /** * show streaming insert task info detail */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java index 2ef2e922510411..185aa720c4aaef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java @@ -195,7 +195,6 @@ public class StreamingInsertJob extends AbstractJob taskContext) { - return CollectionUtils.isEmpty(getRunningTasks()) && runningStreamTask == null && !isFinalStatus(); + return CollectionUtils.isEmpty(getRunningTasks()) && !isFinalStatus(); } @Override @@ -765,90 +640,18 @@ public List createTasks(TaskType taskType, Map queryAllStreamTasks() { protected void fetchMeta() throws JobException { long start = System.currentTimeMillis(); - long expectedEpoch = statusEpoch; try { // when fe restart, offsetProvider.jobId may be null Map props = getProviderProps(); @@ -940,8 +742,17 @@ protected void fetchMeta() throws JobException { offsetProvider.fetchRemoteMeta(props); } catch (Exception ex) { log.warn("fetch remote meta failed, job id: {}", getJobId(), ex); - if (pauseForInternalFailure(new FailureReason(InternalErrorCode.GET_REMOTE_DATA_ERROR, - "Failed to fetch meta, " + ex.getMessage()), expectedEpoch)) { + if (this.getFailureReason() == null + || !InternalErrorCode.MANUAL_PAUSE_ERR.equals(this.getFailureReason().getCode())) { + // When a job is manually paused, it does not need to be set again, + // otherwise, it may be woken up by auto resume. + // Pause before setting the reason: updateJobStatus's writeLock orders this after any + // task-success callback that clears failureReason, so a success can't wipe the reason. + this.updateJobStatus(JobStatus.PAUSED); + this.setFailureReason( + new FailureReason(InternalErrorCode.GET_REMOTE_DATA_ERROR, + "Failed to fetch meta, " + ex.getMessage())); + if (MetricRepo.isInit) { MetricRepo.COUNTER_STREAMING_JOB_GET_META_FAIL_COUNT.increase(1L); } @@ -960,7 +771,6 @@ protected void fetchMeta() throws JobException { * Called by scheduler each tick (PENDING/RUNNING). Mirrors fetchMeta error handling. */ public void advanceSplitsIfNeed() throws JobException { - long expectedEpoch = statusEpoch; if (offsetProvider.noMoreSplits()) { return; } @@ -980,8 +790,13 @@ public void advanceSplitsIfNeed() throws JobException { } } catch (Exception ex) { log.warn("advance splits failed, job id: {}", getJobId(), ex); - pauseForInternalFailure(new FailureReason(InternalErrorCode.GET_REMOTE_DATA_ERROR, - "Failed to advance splits, " + ex.getMessage()), expectedEpoch); + if (this.getFailureReason() == null + || !InternalErrorCode.MANUAL_PAUSE_ERR.equals(this.getFailureReason().getCode())) { + this.setFailureReason(new FailureReason( + InternalErrorCode.GET_REMOTE_DATA_ERROR, + "Failed to advance splits, " + ex.getMessage())); + this.updateJobStatus(JobStatus.PAUSED); + } } } @@ -1000,19 +815,30 @@ public void clearRunningStreamTask(JobStatus newJobStatus) { log.info("clear running streaming insert task for job {}, task {}, status {} ", getJobId(), runningStreamTask.getTaskId(), runningStreamTask.getStatus()); runningStreamTask.cancel(JobStatus.STOPPED.equals(newJobStatus) ? false : true); - runningStreamTask = null; + runningStreamTask.closeOrReleaseResources(); } } - public void clearRunningStreamTask(AbstractStreamingTask finishedTask) { + // Command entry for a manual status change: reset the failure/retry budget, and on manual pause + // release the reader (keep slot). "Manual" is decided by the caller, never by reading failureReason. + public void onManualStatusAltered(JobStatus newStatus, FailureReason reason) { + AbstractStreamingTask taskToRelease = null; lock.writeLock().lock(); try { - if (runningStreamTask == finishedTask) { - runningStreamTask = null; + resetFailureInfo(reason); + if (JobStatus.PAUSED.equals(newStatus) && runningStreamTask != null) { + // Force resume to swap in a fresh reader, in case the release RPC races or fails. + this.needRebuildReader = true; + taskToRelease = runningStreamTask; } } finally { lock.writeLock().unlock(); } + // Release outside the write lock: the RPC may block on first brpc connect and this is + // best-effort (needRebuildReader already forces a fresh reader; a stale release is a no-op). + if (taskToRelease != null) { + taskToRelease.releaseRemoteReader(); + } } public boolean hasMoreDataToConsume() { @@ -1035,17 +861,7 @@ public void onTaskSuccess(StreamingJobSchedulerTask task) throws JobException { } public void onStreamTaskFail(AbstractStreamingTask task) throws JobException { - if (!lock.writeLock().isHeldByCurrentThread()) { - writeLock(); - } try { - if (runningStreamTask != task) { - log.info("Ignore stale failure callback for streaming job {}, task {}", getJobId(), task.getTaskId()); - return; - } - if (!(task instanceof StreamingMultiTblTask)) { - runningStreamTask = null; - } this.needRebuildReader = true; failedTaskCount.incrementAndGet(); Env.getCurrentEnv().getJobManager().getStreamingTaskManager().removeRunningTask(task); @@ -1057,38 +873,14 @@ public void onStreamTaskFail(AbstractStreamingTask task) throws JobException { if (MetricRepo.isInit) { MetricRepo.COUNTER_STREAMING_JOB_TASK_FAILED_COUNT.increase(1L); } - updateJobStatus(JobStatus.PAUSED); } finally { writeUnlock(); } - if (task instanceof StreamingMultiTblTask - && ((StreamingMultiTblTask) task).releaseRemoteReaderAndWait()) { - clearRunningStreamTask(task); - } + updateJobStatus(JobStatus.PAUSED); } public void onStreamTaskSuccess(AbstractStreamingTask task) throws JobException { - onStreamTaskSuccess(task, null); - } - - void onStreamTaskSuccess(AbstractStreamingTask task, Runnable beforeHandoff) throws JobException { - // TVF transaction callbacks transfer one write-lock hold from beforeCommitted() to this - // terminal callback. Multi-table callbacks arrive without that hold and acquire it here. - if (!lock.writeLock().isHeldByCurrentThread()) { - writeLock(); - } try { - if (runningStreamTask != task || !JobStatus.RUNNING.equals(getJobStatus()) - || task.getIsCanceled().get()) { - log.info("Ignore stale success callback for streaming job {}, task {}", getJobId(), task.getTaskId()); - return; - } - if (beforeHandoff != null) { - beforeHandoff.run(); - } - // The success callback is the exact terminal handoff. Clear the predecessor before creating - // its successor; the execution owner's later finally block uses identity and cannot clear the new task. - runningStreamTask = null; this.needRebuildReader = false; resetFailureInfo(null); succeedTaskCount.incrementAndGet(); @@ -1109,6 +901,7 @@ void onStreamTaskSuccess(AbstractStreamingTask task, Runnable beforeHandoff) thr return; } AbstractStreamingTask nextTask = createStreamingTask(); + this.runningStreamTask = nextTask; log.info("Streaming insert job {} create next streaming insert task {} after task {} success", getJobId(), nextTask.getTaskId(), task.getTaskId()); } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java index 3b25bdcdc87b3f..df23f10724049f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTask.java @@ -34,7 +34,6 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.thrift.TCell; @@ -54,9 +53,8 @@ @Log4j2 @Getter public class StreamingInsertTask extends AbstractStreamingTask { - private static final long CANCEL_WAIT_TIMEOUT_MS = 1000; private String sql; - private volatile StmtExecutor stmtExecutor; + private StmtExecutor stmtExecutor; private InsertIntoTableCommand taskCommand; private String currentDb; private ConnectContext ctx; @@ -151,10 +149,11 @@ public boolean onSuccess() throws JobException { if (getIsCanceled().get()) { return false; } + this.status = TaskStatus.SUCCESS; + this.finishTimeMs = System.currentTimeMillis(); if (!isCallable()) { return false; } - this.finishTimeMs = System.currentTimeMillis(); Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); if (null == job) { log.info("job is null, job id is {}", jobId); @@ -163,7 +162,6 @@ public boolean onSuccess() throws JobException { StreamingInsertJob streamingInsertJob = (StreamingInsertJob) job; streamingInsertJob.onStreamTaskSuccess(this); - this.status = TaskStatus.SUCCESS; return true; } @@ -175,66 +173,24 @@ protected void onFail(String errMsg) throws JobException { @Override public void cancel(boolean needWaitCancelComplete) { super.cancel(needWaitCancelComplete); - StmtExecutor executor = stmtExecutor; - if (null != executor) { + if (null != stmtExecutor) { log.info("cancelling streaming insert task, job id is {}, task id is {}", getJobId(), getTaskId()); - executor.cancel(new Status(TStatusCode.CANCELLED, "streaming insert task cancelled"), false); - } - if (needWaitCancelComplete) { - // Planning may still be blocked before stmtExecutor is published. Do not let PAUSE wait - // forever; the scheduler owner remains responsible for exact-once cleanup in execute(). - awaitExecutionCompletion(CANCEL_WAIT_TIMEOUT_MS); + stmtExecutor.cancel(new Status(TStatusCode.CANCELLED, "streaming insert task cancelled"), + needWaitCancelComplete); } } @Override - public synchronized void closeOrReleaseResources() { - ConnectContext taskContext = ctx; - RuntimeException cleanupFailure = null; - try { - if (taskContext != null) { - if (taskContext.queryId() != null) { - // Planning can register query-finish callbacks before a coordinator exists. Always run the - // registry teardown so Hive read transactions do not survive a failed/cancelled attempt. - try { - QeProcessorImpl.INSTANCE.unregisterQuery(taskContext.queryId()); - } catch (RuntimeException e) { - cleanupFailure = e; - } - } - if (taskContext.getStatementContext() != null) { - try { - taskContext.getStatementContext().close(); - } catch (RuntimeException e) { - if (cleanupFailure == null) { - cleanupFailure = e; - } else { - cleanupFailure.addSuppressed(e); - } - } - } - } - } finally { + public void closeOrReleaseResources() { + if (null != stmtExecutor) { stmtExecutor = null; - taskCommand = null; - ctx = null; - // before() installs this attempt's context on the scheduler worker. Remove only that exact - // context: cancellation may invoke cleanup from a different thread while the worker is unwinding. - if (ConnectContext.get() == taskContext) { - ConnectContext.remove(); - } } - if (cleanupFailure != null) { - throw cleanupFailure; + if (null != taskCommand) { + taskCommand = null; } - } - - @Override - protected void onExecutionFinished() { - Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); - if (job instanceof StreamingInsertJob) { - ((StreamingInsertJob) job).clearRunningStreamTask(this); + if (null != ctx) { + ctx = null; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java index 8087826dd5e7f2..030c3bb2b8bd6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java @@ -55,25 +55,33 @@ public void run() throws JobException { } private void handlePendingState() throws JobException { - long expectedEpoch = streamingInsertJob.getStatusEpoch(); - if (streamingInsertJob.hasRunningStreamTask()) { - if (!streamingInsertJob.tryCompleteCanceledPredecessor()) { - return; - } - } if (Config.isCloudMode()) { try { streamingInsertJob.replayOnCloudMode(); } catch (JobException e) { - streamingInsertJob.pauseForInternalFailure( - new FailureReason(InternalErrorCode.INTERNAL_ERR, e.getMessage()), expectedEpoch); + streamingInsertJob.setFailureReason( + new FailureReason(InternalErrorCode.INTERNAL_ERR, e.getMessage())); + streamingInsertJob.updateJobStatus(JobStatus.PAUSED); return; } } streamingInsertJob.replayOffsetProviderIfNeed(); // Pre-advance one batch so the first task has splits to consume streamingInsertJob.advanceSplitsIfNeed(); - streamingInsertJob.dispatchPendingTask(expectedEpoch); + if (streamingInsertJob.getJobStatus() == JobStatus.PAUSED) { + // advanceSplits failed and paused the job; skip task dispatch this tick. + return; + } + if (streamingInsertJob.hasReachedEnd()) { + // Source already fully consumed (e.g. snapshot-only mode recovered after FE restart). + // Transition directly to FINISHED without creating a new task. + streamingInsertJob.updateJobStatus(JobStatus.FINISHED); + streamingInsertJob.logUpdateOperation(); + return; + } + streamingInsertJob.createStreamingTask(); + streamingInsertJob.setSampleStartTime(System.currentTimeMillis()); + streamingInsertJob.updateJobStatus(JobStatus.RUNNING); } private void handleRunningState() throws JobException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java index cc5545ed7217bf..bf41d674afc9e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java @@ -85,7 +85,6 @@ public class StreamingMultiTblTask extends AbstractStreamingTask { private long filteredRows = 0L; private long loadedRows = 0L; private volatile long runningBackendId; - private volatile boolean remoteReaderReleased; long lastScannedRows = -1; long lastProgressMs = 0; @@ -284,19 +283,12 @@ public void successCallback(CommitOffsetRequest offsetRequest) throws JobExcepti if (getIsCanceled().get()) { return; } - Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); - if (null == job) { - log.info("job is null, job id is {}", jobId); - return; - } - StreamingInsertJob streamingInsertJob = (StreamingInsertJob) job; - streamingInsertJob.onStreamTaskSuccess(this, () -> applySuccessCallback(offsetRequest)); - } - - private void applySuccessCallback(CommitOffsetRequest offsetRequest) { this.status = TaskStatus.SUCCESS; this.finishTimeMs = System.currentTimeMillis(); JdbcOffset runOffset = (JdbcOffset) this.runningOffset; + if (!isCallable()) { + return; + } // set end offset to running offset // binlogSplit : [{"splitId":"binlog-split"}] only 1 element // snapshotSplit:[{"splitId":"table-0"},...],...}] @@ -335,11 +327,19 @@ private void applySuccessCallback(CommitOffsetRequest offsetRequest) { this.loadBytes = offsetRequest.getLoadBytes(); this.filteredRows = offsetRequest.getFilteredRows(); this.loadedRows = offsetRequest.getLoadedRows(); + Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); + if (null == job) { + log.info("job is null, job id is {}", jobId); + return; + } + StreamingInsertJob streamingInsertJob = (StreamingInsertJob) job; + streamingInsertJob.onStreamTaskSuccess(this); } @Override protected void onFail(String errMsg) throws JobException { - // The job owns the acknowledged reader-release handoff before it allows auto resume. + // Stop a possibly still-running reader now, so the PG slot frees before auto-resume re-acquires it. + releaseRemoteReader(); super.onFail(errMsg); } @@ -355,18 +355,6 @@ public void closeOrReleaseResources() { // No-op: the reader is async and reused; releasing here (per-iteration finally) would kill it. } - @Override - protected void onExecutionFinished() { - if (!getIsCanceled().get() || (runningBackendId > 0 && !remoteReaderReleased)) { - return; - } - try { - getStreamingJob().clearRunningStreamTask(this); - } catch (JobException e) { - log.info("Skip terminal handoff for removed streaming job {}, task {}", getJobId(), getTaskId()); - } - } - @Override public long getRunningBackendId() { return runningBackendId; @@ -402,39 +390,6 @@ public void releaseRemoteReader() { } } - /** Wait for the BE to acknowledge reader release before allowing a successor to reuse the source. */ - boolean releaseRemoteReaderAndWait() { - if (runningBackendId <= 0) { - return true; - } - Backend backend = Env.getCurrentSystemInfo().getBackend(runningBackendId); - if (backend == null) { - return false; - } - try { - JobBaseConfig releaseParams = new JobBaseConfig( - String.valueOf(getJobId()), dataSourceType.name(), sourceProperties, getFrontendAddress()); - InternalService.PRequestCdcClientRequest request = InternalService.PRequestCdcClientRequest.newBuilder() - .setApi("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/api/releaseReader/" + getTaskId()) - .setParams(new Gson().toJson(releaseParams)).build(); - TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort()); - PRequestCdcClientResult result = BackendServiceProxy.getInstance() - .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec) - .get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS); - ResponseBody response = objectMapper.readValue( - result.getResponse(), new TypeReference>() {}); - remoteReaderReleased = TStatusCode.findByValue(result.getStatus().getStatusCode()) == TStatusCode.OK - && response.getCode() == RestApiStatusCode.OK.code; - return remoteReaderReleased; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } catch (Exception e) { - log.warn("Wait for reader release failed, job {} task {}", getJobId(), getTaskId(), e); - return false; - } - } - private String getFrontendAddress() { return Env.getCurrentEnv().getMasterHost() + ":" + Env.getCurrentEnv().getMasterHttpPort(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java index e7a110f85022ef..8c55856da57d7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java @@ -280,14 +280,9 @@ public void alterJobStatus(String jobName, JobStatus jobStatus, FailureReason re if (a.getJobName().equals(jobName)) { try { checkSameStatus(a, jobStatus); + alterJobStatus(a.getJobId(), jobStatus); if (a instanceof StreamingInsertJob) { - ((StreamingInsertJob) a).updateManualJobStatus(jobStatus, reason); - if (statusNeedsScheduling(jobStatus)) { - jobScheduler.cycleTimerJobScheduler(a); - } - a.logUpdateOperation(); - } else { - alterJobStatus(a.getJobId(), jobStatus); + ((StreamingInsertJob) a).onManualStatusAltered(jobStatus, reason); } } catch (JobException e) { throw new JobException("Alter job status error, jobName is %s, errorMsg is %s", @@ -297,10 +292,6 @@ public void alterJobStatus(String jobName, JobStatus jobStatus, FailureReason re } } - private boolean statusNeedsScheduling(JobStatus status) { - return status.equals(JobStatus.RUNNING); - } - private void checkSameStatus(T a, JobStatus newStatus) throws JobException { if (newStatus.equals(a.getJobStatus())) { throw new JobException("Can't change job status to the same status"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java index 97cde56305fd84..91d5fb1a658737 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java @@ -140,8 +140,6 @@ private void scheduleOneTask(AbstractStreamingTask task) { task.getTaskId(), task.getJobId(), System.currentTimeMillis() - start); } catch (Exception e) { log.error("Failed to execute task, task id: {}, job id: {}", task.getTaskId(), task.getJobId(), e); - } finally { - Env.getCurrentEnv().getJobManager().getStreamingTaskManager().removeRunningTask(task); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index d9c022504d68bb..391a9546500e2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -104,64 +104,59 @@ public static MTMVCache from(String defSql, boolean needCost, boolean needLock, ConnectContext currentContext, boolean addSessionVarGuard) throws AnalysisException { - StatementContext originalStatementContext = createCacheContext.getStatementContext(); - try (StatementContext mvSqlStatementContext = new StatementContext(createCacheContext, - new OriginStatement(defSql, 0))) { - if (!needLock) { - mvSqlStatementContext.setNeedLockTables(false); + StatementContext mvSqlStatementContext = new StatementContext(createCacheContext, + new OriginStatement(defSql, 0)); + if (!needLock) { + mvSqlStatementContext.setNeedLockTables(false); + } + if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) { + mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext); + } + createCacheContext.getStatementContext().setForceRecordTmpPlan(true); + mvSqlStatementContext.setForceRecordTmpPlan(true); + boolean originalRewriteFlag = createCacheContext.getSessionVariable().enableMaterializedViewRewrite; + createCacheContext.getSessionVariable().enableMaterializedViewRewrite = false; + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(defSql); + NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); + try { + // Can not convert to table sink, because use the same column from different table when self join + // the out slot is wrong + if (needCost) { + // Only in mv rewrite, we need plan with eliminated cost which is used for mv chosen + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + } else { + // No need cost for performance + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); } - createCacheContext.setStatementContext(mvSqlStatementContext); - createCacheContext.getStatementContext().setForceRecordTmpPlan(true); - mvSqlStatementContext.setForceRecordTmpPlan(true); - boolean originalRewriteFlag = createCacheContext.getSessionVariable().enableMaterializedViewRewrite; - createCacheContext.getSessionVariable().enableMaterializedViewRewrite = false; - try { - LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(defSql); - NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); - // Can not convert to table sink, because use the same column from different table when self join - // the out slot is wrong - if (needCost) { - // Only in mv rewrite, we need plan with eliminated cost which is used for mv chosen - planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); - } else { - // No need cost for performance - planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); - } - CascadesContext cascadesContext = planner.getCascadesContext(); - Plan rewritePlan = cascadesContext.getRewritePlan(); + CascadesContext cascadesContext = planner.getCascadesContext(); + Plan rewritePlan = cascadesContext.getRewritePlan(); - // Only add SessionVarGuardExpr if requested - Optional exprRewriter = addSessionVarGuard - ? Optional.of(new SessionVarGuardRewriter( - ConnectContextUtil.getAffectQueryResultInPlanVariables(createCacheContext), - cascadesContext)) - : Optional.empty(); - Plan addGuardRewritePlan = exprRewriter - .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, rewritePlan)) - .orElse(rewritePlan); - Pair finalPlanStructInfoPair = constructPlanAndStructInfo( - addGuardRewritePlan, cascadesContext); - List> tmpPlanUsedForRewrite = new ArrayList<>(); - for (Plan plan : cascadesContext.getStatementContext().getTmpPlanForMvRewrite()) { - Plan addGuardplan = exprRewriter - .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, plan)) - .orElse(plan); - tmpPlanUsedForRewrite.add(constructPlanAndStructInfo(addGuardplan, cascadesContext)); - } - MTMVCache cache = new MTMVCache(finalPlanStructInfoPair, addGuardRewritePlan, needCost - ? cascadesContext.getMemo().getRoot().getStatistics() : null, tmpPlanUsedForRewrite); - return cache; - } finally { - // Runtime-bearing MVCC snapshots are planning scratch state, not part of the returned logical plans. - // Drop them on both success and failure before the temporary statement releases its runtime leases. - mvSqlStatementContext.resetMvccSnapshots(); - createCacheContext.getStatementContext().setForceRecordTmpPlan(false); - mvSqlStatementContext.setForceRecordTmpPlan(false); - createCacheContext.getSessionVariable().enableMaterializedViewRewrite = originalRewriteFlag; - createCacheContext.setStatementContext(originalStatementContext); - if (currentContext != null) { - currentContext.setThreadLocalInfo(); - } + // Only add SessionVarGuardExpr if requested + Optional exprRewriter = addSessionVarGuard + ? Optional.of(new SessionVarGuardRewriter( + ConnectContextUtil.getAffectQueryResultInPlanVariables(createCacheContext), + cascadesContext)) + : Optional.empty(); + Plan addGuardRewritePlan = exprRewriter + .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, rewritePlan)) + .orElse(rewritePlan); + Pair finalPlanStructInfoPair = constructPlanAndStructInfo( + addGuardRewritePlan, cascadesContext); + List> tmpPlanUsedForRewrite = new ArrayList<>(); + for (Plan plan : cascadesContext.getStatementContext().getTmpPlanForMvRewrite()) { + Plan addGuardplan = exprRewriter + .map(rewriter -> SessionVarGuardRewriter.rewritePlanTree(rewriter, plan)) + .orElse(plan); + tmpPlanUsedForRewrite.add(constructPlanAndStructInfo(addGuardplan, cascadesContext)); + } + return new MTMVCache(finalPlanStructInfoPair, addGuardRewritePlan, needCost + ? cascadesContext.getMemo().getRoot().getStatistics() : null, tmpPlanUsedForRewrite); + } finally { + createCacheContext.getStatementContext().setForceRecordTmpPlan(false); + mvSqlStatementContext.setForceRecordTmpPlan(false); + createCacheContext.getSessionVariable().enableMaterializedViewRewrite = originalRewriteFlag; + if (currentContext != null) { + currentContext.setThreadLocalInfo(); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index a1c814a2507e4b..040b1265ea9f98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -960,29 +960,6 @@ public synchronized T getOrRegisterStatementResource( return resource; } - /** Start a new execution-scoped resource generation for a retained prepared statement context. */ - public synchronized void beginStatementResourceGeneration() { - if (!statementResources.isEmpty()) { - throw new IllegalStateException("Previous statement resources are still active"); - } - statementResourcesClosed = false; - } - - /** - * Transfers the current statement resources to a later completion boundary. The returned handle owns - * exactly this generation and is idempotent, while {@link #close()} will no longer release the resources. - */ - public synchronized Closeable detachStatementResources() { - if (statementResourcesClosed || statementResources.isEmpty()) { - statementResourcesClosed = true; - return () -> { }; - } - statementResourcesClosed = true; - List resources = new ArrayList<>(statementResources.values()); - statementResources.clear(); - return new DetachedStatementResources(resources); - } - private synchronized void releaseStatementResources() { if (statementResourcesClosed) { return; @@ -1006,40 +983,6 @@ private synchronized void releaseStatementResources() { } } - private static class DetachedStatementResources implements Closeable { - private final List resources; - private boolean closed; - - private DetachedStatementResources(List resources) { - this.resources = resources; - } - - @Override - public synchronized void close() { - if (closed) { - return; - } - closed = true; - Throwable throwable = null; - for (int i = resources.size() - 1; i >= 0; i--) { - try { - resources.get(i).close(); - } catch (Throwable t) { - if (throwable == null) { - throwable = t; - } else { - throwable.addSuppressed(t); - } - } - } - resources.clear(); - if (throwable != null) { - Throwables.throwIfInstanceOf(throwable, RuntimeException.class); - throw new IllegalStateException("Release detached statement resource failed", throwable); - } - } - } - // CHECKSTYLE OFF @Override protected void finalize() throws Throwable { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index 7ddeb5b6a677c6..6d0fc555e7824f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -73,9 +73,6 @@ public R accept(PlanVisitor visitor, C context) { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { StatementContext statementContext = ctx.getStatementContext(); - // PREPARE retains this StatementContext, but ConnectProcessor closes the resources from each - // COM_STMT_EXECUTE. Reopen an empty generation before the next execution starts planning. - statementContext.beginStatementResourceGeneration(); statementContext.setPrepareStage(false); statementContext.setIsInsert(false); statementContext.resetMvccSnapshots(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java index 6ab547708a1364..e286485fd42e0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java @@ -26,8 +26,6 @@ import org.apache.doris.qe.RowBatch; import org.apache.doris.statistics.util.InternalQueryBuffer; -import java.io.Closeable; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; @@ -49,31 +47,19 @@ public class DorisRowResult implements RowResult { private boolean eof; private Object[] current; - private Closeable statementResources; public DorisRowResult(Coordinator coord, List columnNames, List dorisTypes) { - this(coord, columnNames, dorisTypes, null); - } - - public DorisRowResult(Coordinator coord, List columnNames, List dorisTypes, - Closeable statementResources) { this.coord = coord; this.columnNames = columnNames; this.dorisTypes = dorisTypes; this.current = columnNames != null ? new Object[columnNames.size()] : null; this.isLazyLoading = false; this.eof = false; - this.statementResources = statementResources; } @Override public boolean next() { - if (eof) { - return false; - } - if (coord == null) { - eof = true; - close(); + if (eof || coord == null) { return false; } try { @@ -83,7 +69,6 @@ public boolean next() { index = 0; if (batch.isEos()) { eof = true; - close(); return false; } } else { @@ -91,11 +76,6 @@ public boolean next() { } isLazyLoading = true; } catch (Exception e) { - try { - close(); - } catch (RuntimeException closeFailure) { - e.addSuppressed(closeFailure); - } throw new QueryException(e); } return true; @@ -103,32 +83,7 @@ public boolean next() { @Override public void close() { - RuntimeException failure = null; - if (coord != null) { - try { - coord.close(); - } catch (RuntimeException e) { - failure = e; - } finally { - coord = null; - } - } - if (statementResources != null) { - try { - statementResources.close(); - } catch (IOException | RuntimeException e) { - if (failure == null) { - failure = new RuntimeException("Failed to close PLSQL statement resources", e); - } else { - failure.addSuppressed(e); - } - } finally { - statementResources = null; - } - } - if (failure != null) { - throw failure; - } + // TODO } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java index e4f5b2c6942c65..37a8cf310a1d95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java @@ -29,7 +29,6 @@ import org.antlr.v4.runtime.ParserRuleContext; -import java.io.Closeable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -43,34 +42,21 @@ public QueryResult executeQuery(String sql, ParserRuleContext ctx) { // A cursor may correspond to a query, and if the user opens multiple cursors, need to save multiple // query states, so here each query constructs a ConnectProcessor and the ConnectContext shares some data. ConnectContext context = ConnectContext.get().cloneContext(); - Closeable statementResources = null; try (AutoCloseConnectContext autoCloseCtx = new AutoCloseConnectContext(context)) { autoCloseCtx.call(); context.setRunProcedure(true); ConnectProcessor processor = new MysqlConnectProcessor(context); processor.executeQuery(sql); StmtExecutor executor = context.getExecutor(); - statementResources = context.getStatementContext().detachStatementResources(); - DorisRowResult rowResult; if (executor.getParsedStmt().getResultExprs() != null) { - rowResult = new DorisRowResult(executor.getCoord(), executor.getColumns(), - executor.getReturnTypes(), statementResources); - statementResources = null; - return new QueryResult(rowResult, () -> metadata(executor), processor, null); + return new QueryResult(new DorisRowResult(executor.getCoord(), executor.getColumns(), + executor.getReturnTypes()), () -> metadata(executor), processor, null); } else { // If ResultExpr is empty, not need to return result in plsql.Stmt.statement() - rowResult = new DorisRowResult(executor.getCoord(), executor.getColumns(), null, statementResources); - statementResources = null; - return new QueryResult(rowResult, null, processor, null); + return new QueryResult(new DorisRowResult(executor.getCoord(), executor.getColumns(), null), + null, processor, null); } } catch (Exception e) { - if (statementResources != null) { - try { - statementResources.close(); - } catch (Exception closeFailure) { - e.addSuppressed(closeFailure); - } - } return new QueryResult(null, () -> new Metadata(Collections.emptyList()), null, e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java index 3fc06a23220993..0c400950c58052 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseConnectContext.java @@ -36,19 +36,10 @@ public void call() { @Override public void close() { - try { - if (connectContext.getStatementContext() != null) { - connectContext.getStatementContext().close(); - } - } finally { - try { - connectContext.clear(); - } finally { - ConnectContext.remove(); - if (previousContext != null) { - previousContext.setThreadLocalInfo(); - } - } + connectContext.clear(); + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 0a1dc88fa4d523..1ad07cae935c46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -959,46 +959,7 @@ public void setDatabase(String db) { } public void setExecutor(StmtExecutor executor) { - boolean cancelAfterPublication = false; - Status deferredCancelReason; - synchronized (executorPublicationLock) { - if (connectType == ConnectType.ARROW_FLIGHT_SQL) { - synchronized (flightSqlDeferredExecutors) { - this.executor = executor; - cancelAfterPublication = flightSqlDeferredExecutorsSealed; - } - } else { - this.executor = executor; - } - deferredCancelReason = pendingExecutorCancelReason; - pendingExecutorCancelReason = null; - } - if (executor != null) { - if (deferredCancelReason != null) { - executor.cancel(deferredCancelReason); - } - if (cancelAfterPublication) { - executor.cancel(new Status(TStatusCode.CANCELLED, "arrow flight connection closed"), false); - } - } - } - - private final Object executorPublicationLock = new Object(); - private Status pendingExecutorCancelReason; - - /** Preserve a forwarded-query cancel until proxyExecute publishes its StmtExecutor. */ - public void cancelQueryOnExecutorPublication(Status cancelReason) { - StmtExecutor executorRef; - synchronized (executorPublicationLock) { - executorRef = executor; - if (executorRef == null) { - if (pendingExecutorCancelReason == null) { - pendingExecutorCancelReason = cancelReason; - } - return; - } - } - executorRef.cancel(cancelReason); + this.executor = executor; } public StmtExecutor getExecutor() { @@ -1026,109 +987,21 @@ public PlSqlOperation getPlSqlOperation() { // with "Split source X is released". These executors are finalized when the next query starts // on this connection, or when the connection is torn down. See #62259. private final List flightSqlDeferredExecutors = new ArrayList<>(); - private boolean flightSqlDeferredExecutorsSealed; - private int flightSqlResultPublishers; - public boolean addFlightSqlDeferredExecutor(StmtExecutor executor) { + public void addFlightSqlDeferredExecutor(StmtExecutor executor) { synchronized (flightSqlDeferredExecutors) { - if (flightSqlDeferredExecutorsSealed) { - return false; - } flightSqlDeferredExecutors.add(executor); - return true; - } - } - - /** Linearizes GetFlightInfo publication with the terminal session seal. */ - public boolean canPublishFlightSqlResult() { - synchronized (flightSqlDeferredExecutors) { - return !flightSqlDeferredExecutorsSealed; - } - } - - public boolean beginFlightSqlResultPublication() { - synchronized (flightSqlDeferredExecutors) { - if (flightSqlDeferredExecutorsSealed || flightSqlResultPublishers != 0) { - return false; - } - flightSqlResultPublishers = 1; - return true; - } - } - - public boolean endFlightSqlResultPublication() { - List toClose = null; - boolean published; - synchronized (flightSqlDeferredExecutors) { - published = !flightSqlDeferredExecutorsSealed; - if (--flightSqlResultPublishers == 0 && flightSqlDeferredExecutorsSealed) { - toClose = drainFlightSqlDeferredExecutors(); - flightSqlDeferredExecutors.notifyAll(); - } } - finalizeFlightSqlDeferredExecutors(toClose); - return published; } public void closeFlightSqlDeferredExecutors() { - closeFlightSqlDeferredExecutors(false); - } - - /** Prevents a session teardown race from accepting an executor after the final drain. */ - public void sealAndCloseFlightSqlDeferredExecutors() { - sealFlightSqlDeferredExecutors(); - awaitAndCloseFlightSqlDeferredExecutors(); - } - - /** Rejects new result/executor publications without waiting for an admitted publisher. */ - public void sealFlightSqlDeferredExecutors() { - synchronized (flightSqlDeferredExecutors) { - flightSqlDeferredExecutorsSealed = true; - } - } - - /** Waits for admitted publishers after their query has been canceled, then drains retained executors. */ - public void awaitAndCloseFlightSqlDeferredExecutors() { - closeFlightSqlDeferredExecutors(true); - } - - private void closeFlightSqlDeferredExecutors(boolean seal) { - List toClose = null; + List toClose; synchronized (flightSqlDeferredExecutors) { - if (seal) { - flightSqlDeferredExecutorsSealed = true; - // The result channel is destroyed immediately after this method returns. Wait until every - // admitted publisher has either committed or observed the seal, so a losing local-result - // publisher cannot insert Arrow buffers after the channel's one-time invalidation. - boolean interrupted = false; - while (flightSqlResultPublishers != 0) { - try { - flightSqlDeferredExecutors.wait(); - } catch (InterruptedException e) { - interrupted = true; - } - } - if (interrupted) { - Thread.currentThread().interrupt(); - } + if (flightSqlDeferredExecutors.isEmpty()) { + return; } - toClose = drainFlightSqlDeferredExecutors(); - } - finalizeFlightSqlDeferredExecutors(toClose); - } - - private List drainFlightSqlDeferredExecutors() { - if (flightSqlDeferredExecutors.isEmpty()) { - return null; - } - List toClose = new ArrayList<>(flightSqlDeferredExecutors); - flightSqlDeferredExecutors.clear(); - return toClose; - } - - private void finalizeFlightSqlDeferredExecutors(List toClose) { - if (toClose == null) { - return; + toClose = new ArrayList<>(flightSqlDeferredExecutors); + flightSqlDeferredExecutors.clear(); } for (StmtExecutor deferredExecutor : toClose) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 49caaf0a2eb4e6..90e0f6e8a62eef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -757,10 +757,6 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException } throw new RuntimeException("Prepare failed when proxy execute"); } - // Forwarded PREPARE and EXECUTE share the retained StatementContext, but they are distinct - // resource generations. Release any catalog/table leases acquired while analyzing PREPARE - // before ExecuteCommand opens the execution generation. - ctx.getStatementContext().detachStatementResources().close(); handleExecute(preparedStatementContext.command, Long.parseLong(preparedStmtId), preparedStatementContext, ByteBuffer.wrap(request.getPrepareExecuteBuffer()).order(ByteOrder.LITTLE_ENDIAN), queryId); @@ -777,17 +773,8 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException LOG.warn("Process one query failed because unknown reason: ", e); ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, "Unexpected exception: " + e.getMessage()); } - try { - return buildProxyResult(request, executor); - } finally { - if (ctx.getStatementContext() != null) { - ctx.getStatementContext().close(); - } - } - } - - private TMasterOpResult buildProxyResult(TMasterOpRequest request, StmtExecutor executor) { - // No matter whether execution succeeds or fails, return the result and current journal ID to the follower. + // no matter the master execute success or fail, the master must transfer the result to follower + // and tell the follower the current journalID. TMasterOpResult result = new TMasterOpResult(); if (ctx.queryId() != null // If none master FE not set query id or query id was reset in StmtExecutor diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 11f2105d1fa997..13e14e6ff3eb48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -143,7 +143,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutableTriple; @@ -785,9 +784,6 @@ private boolean shouldQueue() { // A call to Exec() must precede all other member function calls. @Override public void exec() throws Exception { - if (isQueryCancelled()) { - throw new UserException("Query was cancelled before execution"); - } // LoadTask does not have context, not controlled by queue now if (context != null) { if (Config.enable_workload_group) { @@ -920,13 +916,7 @@ protected void execInternal() throws Exception { protected void sendPipelineCtx() throws Exception { lock(); - boolean lockHeld = true; try { - // Linearize fragment dispatch with cancel(): cancel either publishes its status before this - // admission check, or waits for dispatch publication and then cancels the remote fragments. - if (queryStatus.isCancelled()) { - throw new UserException("Query was cancelled before fragment dispatch"); - } Multiset hostCounter = HashMultiset.create(); for (FragmentExecParams params : fragmentExecParamsMap.values()) { for (FInstanceExecParam fi : params.instanceExecParams) { @@ -1057,11 +1047,6 @@ protected void sendPipelineCtx() throws Exception { updateProfileIfPresent(profile -> profile.updateFragmentCompressedSize(compressedSize.get())); updateProfileIfPresent(profile -> profile.setFragmentSerializeTime()); - // All cancel-visible per-backend contexts are now published. Do not retain the coordinator lock - // while waiting for remote RPCs: each PipelineExecContexts linearizes its own send with cancel. - unlock(); - lockHeld = false; - // 4.2 send fragments rpc List>>> futures = Lists.newArrayList(); @@ -1095,9 +1080,7 @@ protected void sendPipelineCtx() throws Exception { updateProfileIfPresent(profile -> profile.setRpcPhase2Latency(rpcPhase2Latency)); } } finally { - if (lockHeld) { - unlock(); - } + unlock(); } } @@ -3179,9 +3162,6 @@ public static class PipelineExecContexts { ByteString serializedFragments = null; boolean hasCancelled = false; boolean cancelInProcess = false; - boolean cancelRequested = false; - ListenableFuture phaseOneFuture; - boolean deferredCancelScheduled = false; public PipelineExecContexts(TUniqueId queryId, Backend backend, TNetworkAddress brpcAddr, boolean twoPhaseExecution, @@ -3226,16 +3206,11 @@ public void unsetFields() { } } - public synchronized Future execRemoteFragmentsAsync( - BackendServiceProxy proxy) + public Future execRemoteFragmentsAsync(BackendServiceProxy proxy) throws TException { - if (cancelRequested) { - throw new TException("Query cancelled before fragment dispatch"); - } Preconditions.checkNotNull(serializedFragments); try { - phaseOneFuture = proxy.execPlanFragmentsAsync(brpcAddr, serializedFragments, twoPhaseExecution); - return phaseOneFuture; + return proxy.execPlanFragmentsAsync(brpcAddr, serializedFragments, twoPhaseExecution); } catch (RpcException e) { // DO NOT throw exception here, return a complete future with error code, // so that the following logic will cancel the fragment. @@ -3243,12 +3218,8 @@ public synchronized Future execRemoteFr } } - public synchronized Future execPlanFragmentStartAsync( - BackendServiceProxy proxy) + public Future execPlanFragmentStartAsync(BackendServiceProxy proxy) throws TException { - if (cancelRequested) { - throw new TException("Query cancelled before fragment start"); - } try { PExecPlanFragmentStartRequest.Builder builder = PExecPlanFragmentStartRequest.newBuilder(); PUniqueId qid = PUniqueId.newBuilder().setHi(queryId.hi).setLo(queryId.lo).build(); @@ -3325,41 +3296,12 @@ public String debugInfo() { // Just send the cancel message to BE, not care about the result, because there is no retry // logic in upper logic. private synchronized void cancelQuery(Status cancelReason) { - cancelRequested = true; - scheduleCancelAfterPhaseOne(cancelReason); - cancelQueryInternal(cancelReason, false); - } - - private void scheduleCancelAfterPhaseOne(Status cancelReason) { - if (phaseOneFuture == null || deferredCancelScheduled) { - return; - } - deferredCancelScheduled = true; - Futures.addCallback(phaseOneFuture, new FutureCallback() { - @Override - public void onSuccess(PExecPlanFragmentResult result) { - replayCancelAfterPhaseOne(cancelReason); - } - - @Override - public void onFailure(Throwable t) { - LOG.debug("Phase-one fragment dispatch completed exceptionally before deferred cancel", t); - replayCancelAfterPhaseOne(cancelReason); - } - }, MoreExecutors.directExecutor()); - } - - private synchronized void replayCancelAfterPhaseOne(Status cancelReason) { - cancelQueryInternal(cancelReason, true); - } - - private void cancelQueryInternal(Status cancelReason, boolean force) { if (LOG.isDebugEnabled()) { LOG.debug("cancelRemoteFragments backend: {}, query={}, reason: {}", backend, DebugUtil.printId(queryId), cancelReason.toString()); } - if (!force && (this.hasCancelled || this.cancelInProcess)) { + if (this.hasCancelled || this.cancelInProcess) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java index 8556ac5967bc1a..067db673049c79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java @@ -35,9 +35,6 @@ public class MasterOpExecutor extends FEOpExecutor { private static final Logger LOG = LogManager.getLogger(MasterOpExecutor.class); private final int journalWaitTimeoutMs; - private final Object executionAdmissionLock = new Object(); - private boolean executionStarted; - private boolean cancellationRequested; public MasterOpExecutor(OriginStatement originStmt, ConnectContext ctx, RedirectStatus status, boolean isQuery) { super(new TNetworkAddress(ctx.getEnv().getMasterHost(), ctx.getEnv().getMasterRpcPort()), @@ -58,25 +55,12 @@ public MasterOpExecutor(ConnectContext ctx) { @Override public void execute() throws Exception { - synchronized (executionAdmissionLock) { - if (cancellationRequested) { - ctx.getState().setError("forward operation cancelled"); - return; - } - executionStarted = true; - } super.execute(); waitOnReplaying(); } @Override public void cancel() throws Exception { - synchronized (executionAdmissionLock) { - cancellationRequested = true; - if (!executionStarted) { - return; - } - } super.cancel(); waitOnReplaying(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index a5285afc6c0d71..5bda3026fa6e49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -198,7 +198,7 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare AuditLogHelper.updateMetrics(ctx); } } finally { - prepCtx.statementContext.close(); + prepCtx.statementContext.clearExternalScanTasks(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index 4a1f02b4c6758f..f8c7509f102678 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -139,9 +139,6 @@ public NereidsCoordinator(Long jobId, TUniqueId queryId, DescriptorTable descTab @Override public void exec() throws Exception { - if (isQueryCancelled()) { - throw new UserException("Query was cancelled before execution"); - } enqueue(coordinatorContext.connectContext); processTopSink(coordinatorContext, coordinatorContext.topDistributedPlan); @@ -150,14 +147,7 @@ public void exec() throws Exception { Map workerToFragments = ThriftPlansBuilder.plansToThrift(coordinatorContext); - executionTask = coordinatorContext.withLock(() -> { - if (coordinatorContext.readCloneStatus().isCancelled()) { - throw new UserException("Query was cancelled before fragment dispatch"); - } - // Publish under the cancel monitor. Per-backend task admission then linearizes each RPC with cancel, - // without holding the context monitor while waiting for a remote response. - return PipelineExecutionTaskBuilder.build(coordinatorContext, workerToFragments); - }); + executionTask = PipelineExecutionTaskBuilder.build(coordinatorContext, workerToFragments); executionTask.execute(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 3f94995bd91af3..5c6004fcd10286 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -144,13 +144,13 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.protobuf.ByteString; +import lombok.Setter; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TSerializer; -import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -188,17 +188,13 @@ public class StmtExecutor { private List> changedSessionVarsForAudit; private ProfileType profileType = ProfileType.QUERY; + @Setter private volatile Coordinator coord = null; // Arrow Flight SQL: when true, this query's coordinator is kept alive past GetFlightInfo and // is finalized later by ConnectContext (see #62259), so the eager close in executeAndSendResult // is skipped. private volatile boolean deferredForArrowFlight = false; - private Closeable deferredArrowFlightStatementResources; - private volatile MasterOpExecutor masterOpExecutor = null; - // Cancellation can arrive after executor publication but before execution resources exist. - // Retain it so execution admission and later coordinator publication cannot lose the signal. - private volatile Status pendingCancelReason = null; - private final Object executionAdmissionLock = new Object(); + private MasterOpExecutor masterOpExecutor = null; private RedirectStatus redirectStatus = null; private Planner planner; private boolean isProxy; @@ -507,12 +503,6 @@ public boolean isCached() { // query with a random sql public void execute() throws Exception { - synchronized (executionAdmissionLock) { - if (pendingCancelReason != null) { - context.getState().setError(pendingCancelReason.getErrorMsg()); - return; - } - } UUID uuid = UUID.randomUUID(); TUniqueId queryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); if (Config.enable_print_request_before_execution) { @@ -982,71 +972,16 @@ public boolean isDeferredForArrowFlight() { return deferredForArrowFlight; } - void deferArrowFlightQuery() { - Closeable resources = statementContext.detachStatementResources(); - deferredArrowFlightStatementResources = resources; - deferredForArrowFlight = true; - boolean registered; - try { - registered = context.addFlightSqlDeferredExecutor(this); - } catch (RuntimeException | Error t) { - deferredForArrowFlight = false; - deferredArrowFlightStatementResources = null; - try { - resources.close(); - } catch (Throwable closeFailure) { - t.addSuppressed(closeFailure); - } - throw t; - } - if (!registered) { - // Session teardown sealed and drained the registry between detachment and registration. Finalize - // directly: no later owner can reach this executor, and the deferred flag keeps the statement's - // ordinary finally block from closing the same coordinator a second time. - finalizeArrowFlightQuery(); - } - } - // Finalize an Arrow Flight query whose coordinator was kept alive across the // GetFlightInfo -> DoGet phases: close the coordinator (releasing external-table batch // SplitSources and the query queue slot) and then unregister the query. See #62259. public void finalizeArrowFlightQuery() { - Throwable failure = null; try { if (coord != null) { coord.close(); } - } catch (Throwable t) { - failure = t; - } - try { - if (deferredArrowFlightStatementResources != null) { - deferredArrowFlightStatementResources.close(); - } - } catch (Throwable t) { - if (failure == null) { - failure = t; - } else { - failure.addSuppressed(t); - } - } - try { + } finally { finalizeQuery(); - } catch (Throwable t) { - if (failure == null) { - failure = t; - } else { - failure.addSuppressed(t); - } - } - if (failure != null) { - if (failure instanceof RuntimeException) { - throw (RuntimeException) failure; - } - if (failure instanceof Error) { - throw (Error) failure; - } - throw new IllegalStateException("Failed to finalize Arrow Flight query", failure); } } @@ -1220,11 +1155,6 @@ public boolean isProfileSafeStmt() { private void forwardToMaster() throws Exception { masterOpExecutor = new MasterOpExecutor(originStmt, context, redirectStatus, isQuery()); - if (pendingCancelReason != null) { - masterOpExecutor.cancel(); - context.getState().setError(pendingCancelReason.getErrorMsg()); - return; - } if (LOG.isDebugEnabled()) { LOG.debug("need to transfer to Master. stmt: {}", context.getStmtId()); } @@ -1259,9 +1189,6 @@ public void updateProfile(boolean isFinished) { // Because this is called by other thread public void cancel(Status cancelReason, boolean needWaitCancelComplete) { - synchronized (executionAdmissionLock) { - pendingCancelReason = cancelReason; - } if (masterOpExecutor != null) { try { masterOpExecutor.cancel(); @@ -1292,14 +1219,6 @@ public void cancel(Status cancelReason) { cancel(cancelReason, true); } - public void setCoord(Coordinator coordinator) { - this.coord = coordinator; - Status cancelReason = pendingCancelReason; - if (coordinator != null && cancelReason != null) { - coordinator.cancel(cancelReason); - } - } - private Optional getInsertOverwriteTableCommand() { if (parsedStmt instanceof LogicalPlanAdapter) { LogicalPlanAdapter logicalPlanAdapter = (LogicalPlanAdapter) parsedStmt; @@ -1469,15 +1388,15 @@ public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, context.getSessionVariable().getMaxMsgSizeOfResultReceiver()); context.getState().setIsQuery(true); } else if (planner instanceof NereidsPlanner && ((NereidsPlanner) planner).getDistributedPlans() != null) { - setCoord(new NereidsCoordinator(context, - (NereidsPlanner) planner, context.getStatsErrorEstimator())); + coord = new NereidsCoordinator(context, + (NereidsPlanner) planner, context.getStatsErrorEstimator()); profile.addExecutionProfile(coord.getExecutionProfile()); QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), new QueryInfo(context, originStmt.originStmt, coord)); coordBase = coord; } else { - setCoord(EnvFactory.getInstance().createCoordinator( - context, planner, context.getStatsErrorEstimator())); + coord = EnvFactory.getInstance().createCoordinator( + context, planner, context.getStatsErrorEstimator()); profile.addExecutionProfile(coord.getExecutionProfile()); QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), new QueryInfo(context, originStmt.originStmt, coord)); @@ -1515,7 +1434,8 @@ public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, // at the end of GetFlightInfo. Point queries use a different coordBase (not // deferred). See #62259. if (coordBase == coord) { - deferArrowFlightQuery(); + deferredForArrowFlight = true; + context.addFlightSqlDeferredExecutor(this); } return; } @@ -2177,8 +2097,8 @@ public List executeInternalQuery() { if (Config.enable_collect_internal_query_profile) { context.getSessionVariable().enableProfile = true; } - setCoord(EnvFactory.getInstance().createCoordinator(context, - planner, context.getStatsErrorEstimator())); + coord = EnvFactory.getInstance().createCoordinator(context, + planner, context.getStatsErrorEstimator()); profile.addExecutionProfile(coord.getExecutionProfile()); try { QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java index e249ed82f57b05..f5a77d79196435 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java @@ -39,7 +39,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,6 +46,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -64,9 +64,6 @@ public class MultiFragmentsPipelineTask extends AbstractRuntimeTask phaseOneFuture; - private boolean deferredCancelScheduled; public MultiFragmentsPipelineTask( CoordinatorContext coordinatorContext, Backend backend, BackendServiceProxy backendClientProxy, @@ -81,23 +78,15 @@ public MultiFragmentsPipelineTask( ); this.hasCancelled = new AtomicBoolean(); this.cancelInProcess = new AtomicBoolean(); - this.cancelRequested = new AtomicBoolean(); } - public synchronized Future sendPhaseOneRpc(boolean twoPhaseExecution) { - if (cancelRequested.get()) { - return futureWithStatus(TStatusCode.CANCELLED, "Query cancelled before fragment dispatch"); - } - phaseOneFuture = execRemoteFragmentsAsync( + public Future sendPhaseOneRpc(boolean twoPhaseExecution) { + return execRemoteFragmentsAsync( backendClientProxy, serializeFragments, backend.getBrpcAddress(), twoPhaseExecution ); - return phaseOneFuture; } - public synchronized Future sendPhaseTwoRpc() { - if (cancelRequested.get()) { - return futureWithStatus(TStatusCode.CANCELLED, "Query cancelled before fragment start"); - } + public Future sendPhaseTwoRpc() { return execPlanFragmentStartAsync(backendClientProxy, backend.getBrpcAddress()); } @@ -113,42 +102,13 @@ public String toString() { } public synchronized void cancelExecute(Status cancelReason) { - cancelRequested.set(true); - scheduleCancelAfterPhaseOne(cancelReason); - cancelExecuteInternal(cancelReason, false); - } - - private void scheduleCancelAfterPhaseOne(Status cancelReason) { - if (phaseOneFuture == null || deferredCancelScheduled) { - return; - } - deferredCancelScheduled = true; - Futures.addCallback(phaseOneFuture, new FutureCallback() { - @Override - public void onSuccess(PExecPlanFragmentResult result) { - replayCancelAfterPhaseOne(cancelReason); - } - - @Override - public void onFailure(Throwable t) { - LOG.debug("Phase-one fragment dispatch completed exceptionally before deferred cancel", t); - replayCancelAfterPhaseOne(cancelReason); - } - }, MoreExecutors.directExecutor()); - } - - private synchronized void replayCancelAfterPhaseOne(Status cancelReason) { - cancelExecuteInternal(cancelReason, true); - } - - private void cancelExecuteInternal(Status cancelReason, boolean force) { TUniqueId queryId = coordinatorContext.queryId; if (LOG.isDebugEnabled()) { LOG.debug("cancelRemoteFragments backend: {}, query={}, reason: {}", backend, DebugUtil.printId(queryId), cancelReason.toString()); } - if (!force && (this.hasCancelled.get() || this.cancelInProcess.get())) { + if (this.hasCancelled.get() || this.cancelInProcess.get()) { LOG.info("Fragment has already been cancelled. Query {} backend: {}", DebugUtil.printId(queryId), backend); return; @@ -200,7 +160,7 @@ public Backend getBackend() { return backend; } - private ListenableFuture execRemoteFragmentsAsync( + private Future execRemoteFragmentsAsync( BackendServiceProxy proxy, ByteString serializedFragments, TNetworkAddress brpcAddr, boolean twoPhaseExecution) { Preconditions.checkNotNull(serializedFragments); @@ -231,14 +191,35 @@ public Future execPlanFragmentStartAsyn } } - private ListenableFuture futureWithException(RpcException e) { - return futureWithStatus(TStatusCode.THRIFT_RPC_ERROR, e.getMessage()); - } + private Future futureWithException(RpcException e) { + return new Future() { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } - private ListenableFuture futureWithStatus(TStatusCode statusCode, String message) { - PExecPlanFragmentResult result = PExecPlanFragmentResult.newBuilder().setStatus( - Types.PStatus.newBuilder().addErrorMsgs(message) - .setStatusCode(statusCode.getValue()).build()).build(); - return Futures.immediateFuture(result); + @Override + public PExecPlanFragmentResult get() { + PExecPlanFragmentResult result = PExecPlanFragmentResult.newBuilder().setStatus( + Types.PStatus.newBuilder().addErrorMsgs(e.getMessage()) + .setStatusCode(TStatusCode.THRIFT_RPC_ERROR.getValue()).build()).build(); + return result; + } + + @Override + public PExecPlanFragmentResult get(long timeout, TimeUnit unit) { + return get(); + } + }; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java index a374240d63b446..6a52e3a6d9f855 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java @@ -92,10 +92,13 @@ public PipelineExecutionTask( @Override public void execute() throws Exception { - sendAndWaitPhaseOneRpc(); - if (coordinatorContext.twoPhaseExecution()) { - sendAndWaitPhaseTwoRpc(); - } + coordinatorContext.withLock(() -> { + sendAndWaitPhaseOneRpc(); + if (coordinatorContext.twoPhaseExecution()) { + sendAndWaitPhaseTwoRpc(); + } + return null; + }); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java index 256d676182f3b1..13489e1894fa4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java @@ -65,13 +65,13 @@ public boolean isUsingLatestChannelConfig() { return channelConfigVersion == CHANNEL_PROVIDER.currentConfigVersion(); } - public ListenableFuture execPlanFragmentAsync( + public Future execPlanFragmentAsync( InternalService.PExecPlanFragmentRequest request) { return stub.withDeadlineAfter(execPlanTimeout, TimeUnit.MILLISECONDS) .execPlanFragment(request); } - public ListenableFuture execPlanFragmentPrepareAsync( + public Future execPlanFragmentPrepareAsync( InternalService.PExecPlanFragmentRequest request) { return stub.withDeadlineAfter(execPlanTimeout, TimeUnit.MILLISECONDS) .execPlanFragmentPrepare(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java index 4682957a59c0d8..15f35d3711721d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java @@ -174,7 +174,7 @@ private BackendServiceClient getProxy(TNetworkAddress address) throws UnknownHos } } - public ListenableFuture execPlanFragmentsAsync(TNetworkAddress address, + public Future execPlanFragmentsAsync(TNetworkAddress address, TPipelineFragmentParamsList params, boolean twoPhaseExecution) throws TException, RpcException { InternalService.PExecPlanFragmentRequest.Builder builder = InternalService.PExecPlanFragmentRequest.newBuilder(); @@ -192,7 +192,7 @@ public ListenableFuture execPlanFragmen return execPlanFragmentsAsync(address, builder.build(), twoPhaseExecution); } - public ListenableFuture execPlanFragmentsAsync(TNetworkAddress address, + public Future execPlanFragmentsAsync(TNetworkAddress address, ByteString serializedFragments, boolean twoPhaseExecution) throws RpcException { InternalService.PExecPlanFragmentRequest.Builder builder = InternalService.PExecPlanFragmentRequest.newBuilder(); @@ -203,7 +203,7 @@ public ListenableFuture execPlanFragmen return execPlanFragmentsAsync(address, builder.build(), twoPhaseExecution); } - public ListenableFuture execPlanFragmentsAsync(TNetworkAddress address, + public Future execPlanFragmentsAsync(TNetworkAddress address, InternalService.PExecPlanFragmentRequest pRequest, boolean twoPhaseExecution) throws RpcException { MetricRepo.BE_COUNTER_QUERY_RPC_ALL.getOrAdd(address.hostname).increase(1L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 5d28ee92ea1041..e6470e2074f264 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -380,8 +380,6 @@ public class FrontendServiceImpl implements FrontendService.Iface { private final Map proxyQueryIdToConnCtx = new ConcurrentHashMap<>(64); - private final Map pendingProxyQueryCancels = new ConcurrentHashMap<>(64); - private static final long PENDING_PROXY_CANCEL_TTL_MS = 5 * 60 * 1000L; private static TNetworkAddress getMasterAddress() { Env env = Env.getCurrentEnv(); @@ -1183,18 +1181,7 @@ public TMasterOpResult forward(TMasterOpRequest params) throws TException { TUniqueId queryId = params.getQueryId(); ConnectContext ctx = proxyQueryIdToConnCtx.get(queryId); if (ctx != null) { - ctx.cancelQueryOnExecutorPublication( - new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); - } else { - long now = System.currentTimeMillis(); - pendingProxyQueryCancels.entrySet().removeIf( - entry -> now - entry.getValue() > PENDING_PROXY_CANCEL_TTL_MS); - pendingProxyQueryCancels.put(queryId, now); - ctx = proxyQueryIdToConnCtx.get(queryId); - if (ctx != null && pendingProxyQueryCancels.remove(queryId) != null) { - ctx.cancelQueryOnExecutorPublication( - new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); - } + ctx.cancelQuery(new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); } final TMasterOpResult result = new TMasterOpResult(); result.setStatusCode(0); @@ -1227,10 +1214,6 @@ public TMasterOpResult forward(TMasterOpRequest params) throws TException { Runnable clearCallback = () -> {}; if (params.isSetQueryId()) { proxyQueryIdToConnCtx.put(params.getQueryId(), context); - if (pendingProxyQueryCancels.remove(params.getQueryId()) != null) { - context.cancelQueryOnExecutorPublication( - new Status(TStatusCode.CANCELLED, "cancel query before forward registration.")); - } clearCallback = () -> proxyQueryIdToConnCtx.remove(params.getQueryId()); } TMasterOpResult result = processor.proxyExecute(params); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index c40e0d9d1db4fe..e64690a54cb10f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -186,12 +186,9 @@ public void closePreparedStatement(final ActionClosePreparedStatementRequest req private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext connectContext, String query, final FlightDescriptor descriptor) { - Preconditions.checkState(null != connectContext); - Preconditions.checkState(!query.isEmpty()); - boolean resultPublisher = connectContext.beginFlightSqlResultPublication(); - Preconditions.checkState(resultPublisher, - "Arrow Flight SQL session already has an active result publisher or is torn down"); try { + Preconditions.checkState(null != connectContext); + Preconditions.checkState(!query.isEmpty()); // Finalize the previous query's coordinator on this connection whose close was // deferred (Arrow Flight keeps it alive across GetFlightInfo -> DoGet so the BE can // fetch external-table splits during DoGet). By now the previous DoGet is done. #62259 @@ -215,11 +212,9 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con final ByteString handle = ByteString.copyFromUtf8(peerIdentity + ":" + queryId); TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder() .setStatementHandle(handle).build(); - FlightInfo flightInfo = getFlightInfoForSchema(ticketStatement, descriptor, + return getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel().getResult(queryId).getVectorSchemaRoot() .getSchema()); - resultPublisher = false; - return publishFlightInfo(connectContext, flightInfo); } else { // A Flight Sql request can only contain one statement that returns result, // otherwise expected thrown exception during execution. @@ -233,12 +228,9 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con peerIdentity + ":" + DebugUtil.printId(connectContext.queryId())); TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder() .setStatementHandle(handle).build(); - FlightInfo flightInfo = getFlightInfoForSchema(ticketStatement, descriptor, - connectContext.getFlightSqlChannel() - .getResult(DebugUtil.printId(connectContext.queryId())).getVectorSchemaRoot() - .getSchema()); - resultPublisher = false; - return publishFlightInfo(connectContext, flightInfo); + return getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel() + .getResult(DebugUtil.printId(connectContext.queryId())).getVectorSchemaRoot() + .getSchema()); } } else { // Now only query stmt will pull results from BE. @@ -288,10 +280,7 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con endpoints.add(new FlightEndpoint(ticket, location)); } // TODO Set in BE callback after query end, Client will not callback. - FlightInfo flightInfo = new FlightInfo( - flightSQLConnectProcessor.getArrowSchema(), descriptor, endpoints, -1, -1); - resultPublisher = false; - return publishFlightInfo(connectContext, flightInfo); + return new FlightInfo(flightSQLConnectProcessor.getArrowSchema(), descriptor, endpoints, -1, -1); } } } catch (Throwable e) { @@ -309,19 +298,10 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con LOG.error(errMsg, e); throw CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException(); } finally { - if (resultPublisher) { - connectContext.endFlightSqlResultPublication(); - } connectContext.setCommand(MysqlCommand.COM_SLEEP); } } - private FlightInfo publishFlightInfo(ConnectContext connectContext, FlightInfo flightInfo) { - Preconditions.checkState(connectContext.endFlightSqlResultPublication(), - "Arrow Flight SQL session was torn down before GetFlightInfo completed"); - return flightInfo; - } - @Override public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, final CallContext context, final FlightDescriptor descriptor) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java index e47096b0d28caf..c8854507e00114 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java @@ -17,13 +17,11 @@ package org.apache.doris.service.arrowflight.sessions; -import org.apache.doris.common.Status; import org.apache.doris.common.util.TokenMasker; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContext.ConnectType; import org.apache.doris.qe.ConnectPoolMgr; import org.apache.doris.service.arrowflight.results.FlightSqlChannel; -import org.apache.doris.thrift.TStatusCode; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; @@ -59,12 +57,6 @@ public int registerConnection(ConnectContext ctx) { @Override public void unregisterConnection(ConnectContext ctx) { - // Reject new publications first, then signal the active query before waiting for an admitted - // GetFlightInfo publisher. Waiting before cancellation can deadlock KILL CONNECTION behind the - // publisher whose query must be canceled in order to leave publication. - ctx.sealFlightSqlDeferredExecutors(); - ctx.cancelQuery(new Status(TStatusCode.CANCELLED, "arrow flight connection closed")); - ctx.awaitAndCloseFlightSqlDeferredExecutors(); // All Flight SQL session teardown paths (idle/query timeout, bearer token expiry, and // explicit CloseSession) reach here. Release channel-cached Arrow results before removing // the context from the pool. @@ -85,6 +77,7 @@ public void unregisterConnection(ConnectContext ctx) { // Finalize any Arrow Flight query whose coordinator was kept alive across the // GetFlightInfo -> DoGet phases (see #62259), releasing its resources (e.g. external-table // batch SplitSources and the query queue slot). + ctx.closeFlightSqlDeferredExecutors(); ctx.closeTxn(); if (connectionMap.remove(ctx.getConnectionId()) != null) { numberConnection.decrementAndGet(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java index 28b2c604fdfc30..3cc30ea61ac824 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java @@ -41,6 +41,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; public class SplitAssignmentTest { @@ -81,6 +82,16 @@ void setUp() { ); } + @Test + void testCloseableRegisteredAfterStopIsClosedImmediately() { + splitAssignment.stop(); + AtomicBoolean closed = new AtomicBoolean(); + + splitAssignment.addCloseable(() -> closed.set(true)); + + Assertions.assertTrue(closed.get()); + } + // ==================== init() method tests ==================== @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java index d5dfb2d44cabc0..a194e8f67c6080 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java @@ -556,9 +556,9 @@ private Object newHiveFileScanTaskCacheKey( HivePartition partition, HiveExternalMetaCache.FileCacheValue fileCacheValue) throws Exception { Class keyClass = Class.forName(HiveScanNode.class.getName() + "$HiveFileScanTaskCacheKey"); Constructor constructor = keyClass.getDeclaredConstructor( - long.class, long.class, long.class, List.class, long.class, List.class); + long.class, long.class, List.class, long.class, List.class); constructor.setAccessible(true); - return constructor.newInstance(1L, 2L, 3L, Collections.singletonList(partition), 0L, + return constructor.newInstance(1L, 2L, Collections.singletonList(partition), 0L, Collections.singletonList(fileCacheValue)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java index 7e4d85cb930ace..66c0679d1b1178 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobLateCallbackTest.java @@ -19,7 +19,6 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.job.cdc.request.CommitOffsetRequest; -import org.apache.doris.job.common.FailureReason; import org.apache.doris.job.common.JobStatus; import org.apache.doris.job.common.TaskStatus; import org.apache.doris.job.offset.jdbc.JdbcSourceOffsetProvider; @@ -118,76 +117,4 @@ public void testCommitOffsetSkipsCanceledTask() throws Exception { Assert.assertEquals("task status must stay terminal — late callback ignored", TaskStatus.FAILED, task.getStatus()); } - - @Test - public void testInvalidManualTransitionDoesNotMutateFailureReason() throws Exception { - StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); - Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); - Deencapsulation.setField(job, "jobStatus", JobStatus.STOPPED); - FailureReason original = new FailureReason("terminal failure"); - Deencapsulation.setField(job, "failureReason", original); - - Assert.assertThrows(org.apache.doris.job.exception.JobException.class, - () -> job.updateManualJobStatus(JobStatus.RUNNING, null)); - Assert.assertSame(original, job.getFailureReason()); - } - - @Test - public void testPredecessorBlocksSuccessorSchedulingUntilTerminalHandoff() { - StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); - Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); - Deencapsulation.setField(job, "jobStatus", JobStatus.PENDING); - StreamingMultiTblTask predecessor = newTask(9002L, TaskStatus.CANCELED); - Deencapsulation.setField(job, "runningStreamTask", predecessor); - - Assert.assertFalse(job.isReadyForScheduling(new HashMap<>())); - job.clearRunningStreamTask(predecessor); - Assert.assertTrue(job.isReadyForScheduling(new HashMap<>())); - } - - @Test - public void testManualPauseResumeClearsMultiTaskAndReentersPending() throws Exception { - StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); - Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); - Deencapsulation.setField(job, "jobId", 9004L); - Deencapsulation.setField(job, "jobStatus", JobStatus.RUNNING); - StreamingMultiTblTask predecessor = newTask(9003L, TaskStatus.RUNNING); - Deencapsulation.setField(job, "runningStreamTask", predecessor); - - job.updateManualJobStatus(JobStatus.PAUSED, new FailureReason("manual pause")); - Assert.assertFalse(job.hasRunningStreamTask()); - job.updateManualJobStatus(JobStatus.RUNNING, null); - - Assert.assertEquals(JobStatus.PENDING, job.getJobStatus()); - Assert.assertTrue(job.isReadyForScheduling(new HashMap<>())); - } - - @Test - public void testManualStopPublishesCancellationAndClearsTask() throws Exception { - StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); - Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); - Deencapsulation.setField(job, "jobId", 9004L); - Deencapsulation.setField(job, "jobStatus", JobStatus.RUNNING); - StreamingMultiTblTask predecessor = newTask(9004L, TaskStatus.RUNNING); - Deencapsulation.setField(job, "runningStreamTask", predecessor); - - job.updateManualJobStatus(JobStatus.STOPPED, new FailureReason("manual stop")); - job.onStreamTaskSuccess(predecessor); - - Assert.assertEquals(JobStatus.STOPPED, job.getJobStatus()); - Assert.assertTrue(predecessor.getIsCanceled().get()); - Assert.assertFalse(job.hasRunningStreamTask()); - } - - @Test - public void testStalePendingTickCannotPublishAfterManualTransition() throws Exception { - StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class); - Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); - Deencapsulation.setField(job, "jobStatus", JobStatus.PAUSED); - Deencapsulation.setField(job, "statusEpoch", 2L); - - Assert.assertFalse(job.dispatchPendingTask(1L)); - Assert.assertEquals(JobStatus.PAUSED, job.getJobStatus()); - Assert.assertFalse(job.hasRunningStreamTask()); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java index 9d81c6e9d73749..0b13d0ec74586e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java @@ -195,22 +195,6 @@ public void testReplayUpdatedRestoresStartTime() { Assert.assertEquals(1234L, job.getStartTimeMs()); } - @Test - public void testPauseCancelsTaskAfterReleasingJobWriteLock() throws Exception { - TestStreamingInsertJob job = newJob(new JdbcSourceOffsetProvider(), 1017L); - ReentrantReadWriteLock jobLock = Deencapsulation.getField(job, "lock"); - LockCheckingTask task = new LockCheckingTask(1017L, jobLock); - Deencapsulation.setField(job, "runningStreamTask", task); - - job.updateJobStatus(JobStatus.PAUSED); - - Assert.assertTrue(task.cancelCalled); - Assert.assertFalse(task.cancelObservedWriteLock); - Assert.assertSame(task, Deencapsulation.getField(job, "runningStreamTask")); - job.clearRunningStreamTask(task); - Assert.assertNull(Deencapsulation.getField(job, "runningStreamTask")); - } - private static TestStreamingInsertJob newJob(JdbcSourceOffsetProvider provider, long taskId) { TestStreamingInsertJob job = new TestStreamingInsertJob(); Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); @@ -276,22 +260,4 @@ private static class NoopStreamingMultiTblTask extends StreamingMultiTblTask { public void successCallback(CommitOffsetRequest offsetRequest) throws JobException { } } - - private static class LockCheckingTask extends NoopStreamingMultiTblTask { - private final ReentrantReadWriteLock jobLock; - private boolean cancelCalled; - private boolean cancelObservedWriteLock; - - LockCheckingTask(long taskId, ReentrantReadWriteLock jobLock) { - super(taskId); - this.jobLock = jobLock; - } - - @Override - public void cancel(boolean needWaitCancelComplete) { - cancelCalled = true; - cancelObservedWriteLock = jobLock.isWriteLockedByCurrentThread(); - super.cancel(false); - } - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java deleted file mode 100644 index 7af0356530a547..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskResourceTest.java +++ /dev/null @@ -1,235 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.job.extensions.insert.streaming; - -import org.apache.doris.nereids.StatementContext; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TUniqueId; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.lang.reflect.Field; -import java.util.Collections; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -class StreamingInsertTaskResourceTest { - - @AfterEach - void tearDown() { - ConnectContext.remove(); - } - - @Test - void closeReleasesExactAttemptStatementContextAndWorkerContext() throws Exception { - StreamingInsertTask task = new StreamingInsertTask( - 1L, 2L, "", null, "", null, Collections.emptyMap(), null, null); - ConnectContext taskContext = new ConnectContext(); - TUniqueId queryId = new TUniqueId(10L, 20L); - taskContext.setQueryId(queryId); - StatementContext statementContext = Mockito.mock(StatementContext.class); - taskContext.setStatementContext(statementContext); - taskContext.setThreadLocalInfo(); - Field contextField = StreamingInsertTask.class.getDeclaredField("ctx"); - contextField.setAccessible(true); - contextField.set(task, taskContext); - - task.closeOrReleaseResources(); - task.closeOrReleaseResources(); - - Mockito.verify(statementContext).close(); - Assertions.assertNull(task.getCtx()); - Assertions.assertNull(ConnectContext.get()); - } - - @Test - void cancellationLeavesAttemptCleanupToExecutionOwner() throws Exception { - CountDownLatch planningStarted = new CountDownLatch(1); - CountDownLatch finishPlanning = new CountDownLatch(1); - AtomicInteger closeCalls = new AtomicInteger(); - AtomicBoolean cleanupRanOnWorker = new AtomicBoolean(); - Thread[] workerRef = new Thread[1]; - AbstractStreamingTask task = new AbstractStreamingTask(1L, 2L, null) { - @Override - public void before() throws Exception { - planningStarted.countDown(); - finishPlanning.await(10, TimeUnit.SECONDS); - } - - @Override - public void run() { - } - - @Override - public boolean onSuccess() { - return false; - } - - @Override - public void closeOrReleaseResources() { - closeCalls.incrementAndGet(); - cleanupRanOnWorker.set(Thread.currentThread() == workerRef[0]); - } - }; - Thread worker = new Thread(() -> { - workerRef[0] = Thread.currentThread(); - try { - task.execute(); - } catch (Exception e) { - throw new AssertionError(e); - } - }); - worker.start(); - Assertions.assertTrue(planningStarted.await(10, TimeUnit.SECONDS)); - - task.cancel(false); - Assertions.assertEquals(0, closeCalls.get()); - finishPlanning.countDown(); - worker.join(TimeUnit.SECONDS.toMillis(10)); - - Assertions.assertFalse(worker.isAlive()); - Assertions.assertEquals(1, closeCalls.get()); - Assertions.assertTrue(cleanupRanOnWorker.get()); - } - - @Test - void terminalFailureCanCancelFromExecutionOwnerWithoutSelfWait() throws Exception { - AtomicBoolean failed = new AtomicBoolean(); - StreamingInsertTask task = new StreamingInsertTask( - 1L, 2L, "", null, "", null, Collections.emptyMap(), null, null) { - @Override - public void before() { - setStatus(org.apache.doris.job.common.TaskStatus.RUNNING); - noRetry = true; - } - - @Override - public void run() throws org.apache.doris.job.exception.JobException { - throw new org.apache.doris.job.exception.JobException("expected"); - } - - @Override - public synchronized void closeOrReleaseResources() { - } - - @Override - protected void onFail(String errMsg) { - setStatus(org.apache.doris.job.common.TaskStatus.FAILED); - cancel(true); - failed.set(true); - } - }; - Thread worker = new Thread(() -> { - try { - task.execute(); - } catch (Exception e) { - throw new AssertionError(e); - } - }); - - worker.start(); - worker.join(TimeUnit.SECONDS.toMillis(10)); - - Assertions.assertFalse(worker.isAlive()); - Assertions.assertTrue(failed.get()); - } - - @Test - void cleanupFailureIsHandledByTaskFailureStateMachine() throws Exception { - AtomicBoolean failed = new AtomicBoolean(); - AtomicBoolean successCalled = new AtomicBoolean(); - AtomicInteger runCalls = new AtomicInteger(); - AbstractStreamingTask task = new AbstractStreamingTask(1L, 2L, null) { - @Override - public void before() { - setStatus(org.apache.doris.job.common.TaskStatus.RUNNING); - noRetry = true; - } - - @Override - public void run() { - runCalls.incrementAndGet(); - } - - @Override - public boolean onSuccess() { - successCalled.set(true); - return true; - } - - @Override - public void closeOrReleaseResources() { - throw new IllegalStateException("cleanup failed"); - } - - @Override - protected void onFail(String errMsg) { - Assertions.assertEquals("cleanup failed", errMsg); - failed.set(true); - } - }; - - task.execute(); - - Assertions.assertTrue(failed.get()); - Assertions.assertFalse(successCalled.get()); - Assertions.assertEquals(1, runCalls.get()); - } - - @Test - void successCallbackFailureDoesNotReplayCompletedInsert() throws Exception { - AtomicInteger runCalls = new AtomicInteger(); - AtomicBoolean failed = new AtomicBoolean(); - AbstractStreamingTask task = new AbstractStreamingTask(1L, 2L, null) { - @Override - public void before() { - setStatus(org.apache.doris.job.common.TaskStatus.RUNNING); - } - - @Override - public void run() { - runCalls.incrementAndGet(); - } - - @Override - public boolean onSuccess() { - throw new IllegalStateException("success publication failed"); - } - - @Override - public void closeOrReleaseResources() { - } - - @Override - protected void onFail(String errMsg) { - Assertions.assertEquals("success publication failed", errMsg); - failed.set(true); - } - }; - - task.execute(); - - Assertions.assertEquals(1, runCalls.get()); - Assertions.assertTrue(failed.get()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 1de4a2dcd90c6f..9892d4685a5b97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -83,38 +83,6 @@ public void testStatementResourceOutlivesPlannerResources() { () -> statementContext.getOrRegisterStatementResource("late", () -> () -> { })); } - @Test - public void testPreparedStatementStartsFreshResourceGenerationForEveryExecute() { - StatementContext statementContext = new StatementContext(); - AtomicInteger firstClosed = new AtomicInteger(); - AtomicInteger secondClosed = new AtomicInteger(); - - statementContext.getOrRegisterStatementResource("table", () -> firstClosed::incrementAndGet); - statementContext.close(); - org.junit.jupiter.api.Assertions.assertEquals(1, firstClosed.get()); - - statementContext.beginStatementResourceGeneration(); - statementContext.getOrRegisterStatementResource("table", () -> secondClosed::incrementAndGet); - statementContext.close(); - org.junit.jupiter.api.Assertions.assertEquals(1, firstClosed.get()); - org.junit.jupiter.api.Assertions.assertEquals(1, secondClosed.get()); - } - - @Test - public void testDetachedStatementResourcesOutliveStatementContext() throws Exception { - StatementContext statementContext = new StatementContext(); - AtomicInteger closed = new AtomicInteger(); - statementContext.getOrRegisterStatementResource("arrow-flight", () -> closed::incrementAndGet); - - Closeable detached = statementContext.detachStatementResources(); - statementContext.close(); - org.junit.jupiter.api.Assertions.assertEquals(0, closed.get()); - - detached.close(); - detached.close(); - org.junit.jupiter.api.Assertions.assertEquals(1, closed.get()); - } - @Test public void testPreloadExternalTablesBeforeLock() { ConnectContext connectContext = Mockito.mock(ConnectContext.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java index 9aca076d7898fa..5cb57665d59d84 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MTMVCacheTest.java @@ -48,18 +48,6 @@ */ public class MTMVCacheTest extends SqlTestBase { - @Test - void testFailureRestoresCallerStatementContextAndSessionFlag() { - org.apache.doris.nereids.StatementContext original = connectContext.getStatementContext(); - connectContext.getSessionVariable().enableMaterializedViewRewrite = true; - - Assertions.assertThrows(Exception.class, () -> MTMVCache.from( - "select from", connectContext, true, false, connectContext, false)); - - Assertions.assertSame(original, connectContext.getStatementContext()); - Assertions.assertTrue(connectContext.getSessionVariable().enableMaterializedViewRewrite); - } - @Test void testMTMVCacheIsCorrect() throws Exception { connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java index 8cb06f8cfec182..ca0d8306f535c9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java @@ -72,19 +72,9 @@ public void testResolvedScanOptionsAreResetForEveryExecute() throws Exception { new ExecuteCommand("stmt", prepareCommand, statementContext).run(connectContext, executor); Assertions.assertEquals("2", resolveNextSnapshot(scanParams, snapshotId)); - AtomicInteger firstResourceClosed = new AtomicInteger(); - statementContext.getOrRegisterStatementResource("iceberg-table", - () -> firstResourceClosed::incrementAndGet); - statementContext.close(); - Assertions.assertEquals(1, firstResourceClosed.get()); new ExecuteCommand("stmt", prepareCommand, statementContext).run(connectContext, executor); Assertions.assertEquals("3", resolveNextSnapshot(scanParams, snapshotId)); - AtomicInteger secondResourceClosed = new AtomicInteger(); - statementContext.getOrRegisterStatementResource("iceberg-table", - () -> secondResourceClosed::incrementAndGet); - statementContext.close(); - Assertions.assertEquals(1, secondResourceClosed.get()); Mockito.verify(executor, Mockito.times(2)).execute(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java b/fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java deleted file mode 100644 index 1efd6558f10c74..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/plsql/executor/DorisRowResultTest.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.plsql.executor; - -import org.apache.doris.qe.Coordinator; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.Closeable; -import java.util.Collections; - -class DorisRowResultTest { - - @Test - void closeReleasesCoordinatorAndDetachedStatementResourcesOnce() throws Exception { - Coordinator coordinator = Mockito.mock(Coordinator.class); - Closeable statementResources = Mockito.mock(Closeable.class); - DorisRowResult result = new DorisRowResult( - coordinator, Collections.emptyList(), Collections.emptyList(), statementResources); - - result.close(); - result.close(); - - Mockito.verify(coordinator).close(); - Mockito.verify(statementResources).close(); - } - - @Test - void coordinatorFailureDoesNotSkipStatementResourceCleanup() throws Exception { - Coordinator coordinator = Mockito.mock(Coordinator.class); - Closeable statementResources = Mockito.mock(Closeable.class); - Mockito.doThrow(new IllegalStateException("coordinator close failed")).when(coordinator).close(); - DorisRowResult result = new DorisRowResult( - coordinator, Collections.emptyList(), Collections.emptyList(), statementResources); - - IllegalStateException failure = Assertions.assertThrows(IllegalStateException.class, result::close); - - Assertions.assertEquals("coordinator close failed", failure.getMessage()); - Mockito.verify(statementResources).close(); - } - - @Test - void noCoordinatorClosesDetachedStatementResourcesOnFirstFetch() throws Exception { - Closeable statementResources = Mockito.mock(Closeable.class); - DorisRowResult result = new DorisRowResult( - null, Collections.emptyList(), Collections.emptyList(), statementResources); - - Assertions.assertFalse(result.next()); - - Mockito.verify(statementResources).close(); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java deleted file mode 100644 index 367ecbe0d65d38..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/AutoCloseConnectContextTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.qe; - -import org.apache.doris.nereids.StatementContext; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.atomic.AtomicBoolean; - -class AutoCloseConnectContextTest { - - @AfterEach - void tearDown() { - ConnectContext.remove(); - } - - @Test - void closeReleasesStatementResourcesAndRestoresPreviousContext() { - ConnectContext previous = new ConnectContext(); - previous.setThreadLocalInfo(); - ConnectContext current = new ConnectContext(); - StatementContext statementContext = new StatementContext(); - current.setStatementContext(statementContext); - AtomicBoolean closed = new AtomicBoolean(); - statementContext.getOrRegisterStatementResource("resource", () -> () -> closed.set(true)); - - try (AutoCloseConnectContext ignored = new AutoCloseConnectContext(current)) { - Assertions.assertSame(current, ConnectContext.get()); - } - - Assertions.assertTrue(closed.get()); - Assertions.assertSame(previous, ConnectContext.get()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index f1116798083e08..b626f6acf8a92e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -30,7 +30,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.Pair; -import org.apache.doris.common.Status; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.MysqlCapability; @@ -40,7 +39,6 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.system.Backend; -import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.transaction.TransactionStatus; @@ -59,13 +57,7 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; public class ConnectContextTest { @Mocked @@ -937,72 +929,4 @@ public void testCloseFlightSqlDeferredExecutorsFinalizesRemainingWhenOneFails() Mockito.verify(failing, Mockito.times(1)).finalizeArrowFlightQuery(); Mockito.verify(healthy, Mockito.times(1)).finalizeArrowFlightQuery(); } - - @Test - public void testSessionTeardownRejectsLateDeferredExecutorRegistration() { - ConnectContext ctx = new ConnectContext(); - StmtExecutor late = Mockito.mock(StmtExecutor.class); - - ctx.sealAndCloseFlightSqlDeferredExecutors(); - - Assert.assertFalse("an executor detached after the final teardown drain must not become unreachable", - ctx.addFlightSqlDeferredExecutor(late)); - ctx.closeFlightSqlDeferredExecutors(); - Mockito.verifyNoInteractions(late); - } - - @Test - public void testFlightSealReplaysCancelWhenExecutorPublishesLate() { - ConnectContext ctx = new ConnectContext(); - ctx.connectType = ConnectContext.ConnectType.ARROW_FLIGHT_SQL; - StmtExecutor lateExecutor = Mockito.mock(StmtExecutor.class); - - ctx.sealFlightSqlDeferredExecutors(); - ctx.setExecutor(lateExecutor); - - Mockito.verify(lateExecutor).cancel(Mockito.any(), Mockito.eq(false)); - } - - @Test - public void testForwardCancelReplayedWhenExecutorPublishesLate() { - ConnectContext ctx = new ConnectContext(); - StmtExecutor lateExecutor = Mockito.mock(StmtExecutor.class); - - ctx.cancelQueryOnExecutorPublication(new Status(TStatusCode.CANCELLED, "forward cancel")); - ctx.setExecutor(lateExecutor); - - Mockito.verify(lateExecutor).cancel(Mockito.any(Status.class)); - } - - @Test - public void testSessionSealWaitsForAdmittedResultPublisher() throws Exception { - ConnectContext ctx = new ConnectContext(); - Assert.assertTrue(ctx.beginFlightSqlResultPublication()); - Assert.assertFalse("one Flight SQL session cannot publish two queries concurrently", - ctx.beginFlightSqlResultPublication()); - ctx.sealFlightSqlDeferredExecutors(); - Assert.assertFalse(ctx.canPublishFlightSqlResult()); - ExecutorService executor = Executors.newSingleThreadExecutor(); - CountDownLatch teardownEntered = new CountDownLatch(1); - AtomicReference teardownThread = new AtomicReference<>(); - try { - Future teardown = executor.submit(() -> { - teardownThread.set(Thread.currentThread()); - teardownEntered.countDown(); - ctx.awaitAndCloseFlightSqlDeferredExecutors(); - }); - Assert.assertTrue(teardownEntered.await(10, TimeUnit.SECONDS)); - long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); - while (teardownThread.get().getState() != Thread.State.WAITING - && System.nanoTime() < deadlineNanos) { - Thread.yield(); - } - Assert.assertEquals("teardown must be waiting for the admitted publisher", - Thread.State.WAITING, teardownThread.get().getState()); - Assert.assertFalse(ctx.endFlightSqlResultPublication()); - teardown.get(10, TimeUnit.SECONDS); - } finally { - executor.shutdownNow(); - } - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java deleted file mode 100644 index a93702945a3226..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorCancellationTest.java +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.qe; - -import org.apache.doris.analysis.StatementBase; -import org.apache.doris.qe.QueryState.MysqlStateType; -import org.apache.doris.thrift.TUniqueId; - -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -import java.util.concurrent.atomic.AtomicBoolean; - -public class StmtExecutorCancellationTest { - - @Test - public void testFlightLateExecutorCancellationPreventsExecutionAdmission() throws Exception { - ConnectContext ctx = new ConnectContext(); - ctx.connectType = ConnectContext.ConnectType.ARROW_FLIGHT_SQL; - ctx.sealAndCloseFlightSqlDeferredExecutors(); - AtomicBoolean admitted = new AtomicBoolean(); - StmtExecutor executor = new StmtExecutor(ctx, Mockito.mock(StatementBase.class)) { - @Override - public void queryRetry(TUniqueId queryId) { - admitted.set(true); - } - }; - - ctx.setExecutor(executor); - executor.execute(); - - Assert.assertFalse(admitted.get()); - Assert.assertEquals(MysqlStateType.ERR, ctx.getState().getStateType()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 57aba92754e2f1..607cea3b40a302 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -24,7 +24,6 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlSerializer; -import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.ResultFileSink; @@ -108,47 +107,6 @@ public void testFinalizeArrowFlightQueryUnregistersQueryEvenIfCoordCloseThrows() Assert.assertNull(QeProcessorImpl.INSTANCE.getCoordinator(queryId)); } - @Test - public void testArrowFlightDefersStatementResourcesUntilDoGetCompletion() { - StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); - StatementContext statementContext = connectContext.getStatementContext(); - AtomicInteger resourceCloseCount = new AtomicInteger(); - statementContext.getOrRegisterStatementResource("hudi-batch-owner", - () -> resourceCloseCount::incrementAndGet); - Coordinator coord = Mockito.mock(Coordinator.class); - Mockito.when(coord.getQueryOptions()).thenReturn(new TQueryOptions()); - stmtExecutor.setCoord(coord); - - stmtExecutor.deferArrowFlightQuery(); - statementContext.close(); - Assert.assertEquals(0, resourceCloseCount.get()); - - stmtExecutor.finalizeArrowFlightQuery(); - Assert.assertEquals(1, resourceCloseCount.get()); - Mockito.verify(coord).close(); - } - - @Test - public void testArrowFlightFinalizesImmediatelyWhenSessionTeardownSealedRegistration() { - StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); - StatementContext statementContext = connectContext.getStatementContext(); - AtomicInteger resourceCloseCount = new AtomicInteger(); - statementContext.getOrRegisterStatementResource("hudi-batch-owner", - () -> resourceCloseCount::incrementAndGet); - Coordinator coord = Mockito.mock(Coordinator.class); - Mockito.when(coord.getQueryOptions()).thenReturn(new TQueryOptions()); - stmtExecutor.setCoord(coord); - - connectContext.sealAndCloseFlightSqlDeferredExecutors(); - stmtExecutor.deferArrowFlightQuery(); - - Assert.assertEquals(1, resourceCloseCount.get()); - Mockito.verify(coord).close(); - statementContext.close(); - Assert.assertEquals("the ordinary statement cleanup must not double-close detached resources", - 1, resourceCloseCount.get()); - } - @Test public void testKill() throws Exception { StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java index 456e7c4d55fb0b..eede12688517be 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -18,7 +18,6 @@ package org.apache.doris.service.arrowflight; import org.apache.doris.common.FeConstants; -import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.service.arrowflight.results.FlightSqlChannel; @@ -32,7 +31,6 @@ import org.apache.arrow.flight.Result; import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest; import org.apache.arrow.flight.sql.impl.FlightSql.CommandStatementQuery; -import org.apache.arrow.vector.types.pojo.Schema; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -40,7 +38,6 @@ import org.mockito.MockedConstruction; import org.mockito.Mockito; -import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -210,102 +207,4 @@ public void testGetFlightInfoFinalizesDeferredExecutorWhenSchemaFetchFails() thr producer.close(); } } - - @Test - public void testGetFlightInfoFailsWhenTeardownDrainsRegisteredQueryBeforePublication() throws Exception { - assertTeardownPreventsFlightInfoPublication(true); - } - - @Test - public void testGetFlightInfoFailsWhenTeardownSealsBeforeQueryRegistration() throws Exception { - assertTeardownPreventsFlightInfoPublication(false); - } - - @Test - public void testPublicationCompletionAtomicallyObservesTerminalSeal() { - ConnectContext ctx = new ConnectContext(); - StmtExecutor deferred = Mockito.mock(StmtExecutor.class); - Assert.assertTrue(ctx.beginFlightSqlResultPublication()); - Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(deferred)); - - ctx.sealFlightSqlDeferredExecutors(); - Assert.assertFalse("a publication in flight before teardown must not commit after the terminal seal", - ctx.endFlightSqlResultPublication()); - Mockito.verify(deferred).finalizeArrowFlightQuery(); - } - - @Test - public void testRejectedConcurrentPublisherDoesNotTouchActiveQueryState() throws Exception { - ConnectContext ctx = new ConnectContext(); - StmtExecutor activeDeferred = Mockito.mock(StmtExecutor.class); - Assert.assertTrue(ctx.beginFlightSqlResultPublication()); - Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(activeDeferred)); - ctx.setCommand(MysqlCommand.COM_QUERY); - FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class); - Mockito.when(sessionsManager.getConnectContext(Mockito.anyString())).thenReturn(ctx); - CallContext callContext = Mockito.mock(CallContext.class); - Mockito.when(callContext.peerIdentity()).thenReturn("token"); - DorisFlightSqlProducer producer = new DorisFlightSqlProducer( - Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager); - - try { - CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 2").build(); - FlightDescriptor descriptor = FlightDescriptor.command(new byte[0]); - Throwable rejected = Assert.assertThrows(Throwable.class, - () -> producer.getFlightInfoStatement(request, callContext, descriptor)); - Assert.assertTrue(rejected.getMessage(), - rejected.getMessage().contains("active result publisher")); - - Mockito.verify(activeDeferred, Mockito.never()).finalizeArrowFlightQuery(); - Assert.assertEquals("a rejected publisher must not mark the active query idle", - MysqlCommand.COM_QUERY, ctx.getCommand()); - Assert.assertTrue(ctx.endFlightSqlResultPublication()); - ctx.closeFlightSqlDeferredExecutors(); - Mockito.verify(activeDeferred).finalizeArrowFlightQuery(); - } finally { - producer.close(); - } - } - - private void assertTeardownPreventsFlightInfoPublication(boolean registerBeforeSeal) throws Exception { - ConnectContext ctx = Mockito.spy(new ConnectContext()); - Mockito.doReturn(Mockito.mock(FlightSqlChannel.class)).when(ctx).getFlightSqlChannel(); - StmtExecutor deferred = Mockito.mock(StmtExecutor.class); - FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class); - Mockito.when(sessionsManager.getConnectContext(Mockito.anyString())).thenReturn(ctx); - CallContext callContext = Mockito.mock(CallContext.class); - Mockito.when(callContext.peerIdentity()).thenReturn("token"); - - DorisFlightSqlProducer producer = new DorisFlightSqlProducer( - Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager); - try (MockedConstruction mocked = Mockito.mockConstruction( - FlightSqlConnectProcessor.class, (mock, context) -> { - Mockito.doAnswer(invocation -> { - ctx.setReturnResultFromLocal(false); - if (registerBeforeSeal) { - Assert.assertTrue(ctx.addFlightSqlDeferredExecutor(deferred)); - } - ctx.sealFlightSqlDeferredExecutors(); - if (!registerBeforeSeal) { - Assert.assertFalse(ctx.addFlightSqlDeferredExecutor(deferred)); - } - return null; - }).when(mock).handleQuery(Mockito.anyString()); - Mockito.when(mock.getArrowSchema()).thenReturn(new Schema(Collections.emptyList())); - })) { - CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 1").build(); - FlightDescriptor descriptor = FlightDescriptor.command(new byte[0]); - - try { - producer.getFlightInfoStatement(request, callContext, descriptor); - Assert.fail("teardown must prevent publishing a ticket for finalized query resources"); - } catch (Throwable expected) { - Assert.assertTrue(expected.getMessage().contains("torn down")); - } - - Mockito.verify(deferred, Mockito.times(registerBeforeSeal ? 1 : 0)).finalizeArrowFlightQuery(); - } finally { - producer.close(); - } - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java index 1558e97a42f890..0513221569d9e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java @@ -22,7 +22,6 @@ import org.junit.Assert; import org.junit.Test; -import org.mockito.InOrder; import org.mockito.Mockito; public class FlightSqlConnectPoolMgrTest { @@ -44,14 +43,7 @@ public void testUnregisterConnectionFinalizesDeferredExecutors() { // The deferred coordinators must be released on teardown even though this connection was // never registered in the pool (an abandoned connection is still cleaned up, not leaked). Mockito.verify(channel).close(); - Mockito.verify(ctx).sealFlightSqlDeferredExecutors(); - Mockito.verify(ctx).cancelQuery(Mockito.any()); - Mockito.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); - InOrder teardownOrder = Mockito.inOrder(ctx, channel); - teardownOrder.verify(ctx).sealFlightSqlDeferredExecutors(); - teardownOrder.verify(ctx).cancelQuery(Mockito.any()); - teardownOrder.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); - teardownOrder.verify(channel).close(); + Mockito.verify(ctx).closeFlightSqlDeferredExecutors(); } // Cleanup must run before the connection bookkeeping (closeTxn / map removal), so that a failure @@ -71,14 +63,7 @@ public void testUnregisterRegisteredConnectionFinalizesDeferredExecutors() { poolMgr.unregisterConnection(ctx); Mockito.verify(channel).close(); - Mockito.verify(ctx).sealFlightSqlDeferredExecutors(); - Mockito.verify(ctx).cancelQuery(Mockito.any()); - Mockito.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); - InOrder teardownOrder = Mockito.inOrder(ctx, channel); - teardownOrder.verify(ctx).sealFlightSqlDeferredExecutors(); - teardownOrder.verify(ctx).cancelQuery(Mockito.any()); - teardownOrder.verify(ctx).awaitAndCloseFlightSqlDeferredExecutors(); - teardownOrder.verify(channel).close(); + Mockito.verify(ctx).closeFlightSqlDeferredExecutors(); Assert.assertNull(poolMgr.getConnectionMap().get(7)); } } From 2a3840d2af0749988f25c8d9002c4f398e8f4c61 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 24 Aug 2026 09:25:22 +0800 Subject: [PATCH 18/38] [fix](iceberg) preserve async refresh and close HMS FileIO --- .../datasource/hive/HMSExternalCatalog.java | 10 +++ .../datasource/iceberg/DorisHiveCatalog.java | 77 +++++++++++++++++++ .../datasource/iceberg/IcebergUtils.java | 12 ++- .../IcebergHMSMetaStoreProperties.java | 6 +- .../iceberg/DorisHiveCatalogTest.java | 41 ++++++++++ .../datasource/iceberg/IcebergUtilsTest.java | 3 +- 6 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index b87da756f57aec..1e6be442b33c74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -47,6 +47,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.math.NumberUtils; import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.hive.HiveCatalog; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -183,6 +184,8 @@ public synchronized void onClose() { ThreadPoolExecutor retiredExecutor = threadPoolWithPreAuth; threadPoolWithPreAuth = null; IcebergMetadataOps retiredIcebergMetadataOps = icebergMetadataOps; + Catalog retiredIcebergCatalog = retiredIcebergMetadataOps == null + ? null : retiredIcebergMetadataOps.getCatalog(); icebergMetadataOps = null; super.onClose(); if (null != fileSystemExecutor) { @@ -196,6 +199,13 @@ public synchronized void onClose() { if (retiredIcebergMetadataOps != null) { retiredIcebergMetadataOps.close(); } + if (retiredIcebergCatalog instanceof AutoCloseable) { + try { + ((AutoCloseable) retiredIcebergCatalog).close(); + } catch (Exception e) { + LOG.warn("Failed to close HMS Iceberg catalog: {}", getName(), e); + } + } if (retiredExecutor != null) { ThreadPoolManager.shutdownExecutorService(retiredExecutor); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java new file mode 100644 index 00000000000000..5ab72dbe140812 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.iceberg.hive.HiveCatalog; +import org.apache.iceberg.io.FileIO; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Owns and closes the shared FileIO created by one Iceberg HiveCatalog generation. */ +public class DorisHiveCatalog extends HiveCatalog { + private final AtomicBoolean closed = new AtomicBoolean(); + private FileIO ownedFileIO; + + @Override + public void initialize(String name, Map properties) { + super.initialize(name, properties); + ownedFileIO = extractFileIO(); + } + + @Override + public void close() throws IOException { + if (!closed.compareAndSet(false, true)) { + return; + } + IOException closeFailure = null; + try { + super.close(); + } catch (IOException e) { + closeFailure = e; + } + try { + if (ownedFileIO != null) { + ownedFileIO.close(); + } + } catch (RuntimeException e) { + if (closeFailure != null) { + closeFailure.addSuppressed(e); + } else { + throw e; + } + } finally { + ownedFileIO = null; + } + if (closeFailure != null) { + throw closeFailure; + } + } + + private FileIO extractFileIO() { + try { + Field fileIOField = HiveCatalog.class.getDeclaredField("fileIO"); + fileIOField.setAccessible(true); + return (FileIO) fileIOField.get(this); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Failed to capture Iceberg HiveCatalog FileIO ownership", e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 518301c4753e4d..186d06e2fc0fcd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -82,6 +82,7 @@ import com.google.gson.reflect.TypeToken; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; @@ -1749,9 +1750,6 @@ public static String dataLocation(Table table) { } public static HiveCatalog createIcebergHiveCatalog(ExternalCatalog externalCatalog, String name) { - HiveCatalog hiveCatalog = new HiveCatalog(); - hiveCatalog.setConf(externalCatalog.getConfiguration()); - Map catalogProperties = externalCatalog.getProperties(); if (!catalogProperties.containsKey(HiveCatalog.LIST_ALL_TABLES)) { // This configuration will display all tables (including non-Iceberg type tables), @@ -1761,6 +1759,14 @@ public static HiveCatalog createIcebergHiveCatalog(ExternalCatalog externalCatal } String metastoreUris = catalogProperties.getOrDefault(HMSBaseProperties.HIVE_METASTORE_URIS, ""); catalogProperties.put(CatalogProperties.URI, metastoreUris); + return createIcebergHiveCatalog(name, catalogProperties, externalCatalog.getConfiguration()); + } + + /** Creates an HMS catalog whose generation owns the FileIO allocated by Iceberg. */ + public static HiveCatalog createIcebergHiveCatalog( + String name, Map catalogProperties, Configuration configuration) { + DorisHiveCatalog hiveCatalog = new DorisHiveCatalog(); + hiveCatalog.setConf(configuration); hiveCatalog.initialize(name, catalogProperties); return hiveCatalog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java index 94c4d19f9eba8d..858d71fb624a31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java @@ -19,14 +19,13 @@ import org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.foundation.property.ConnectorProperty; import lombok.Getter; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.hive.HiveCatalog; @@ -68,10 +67,9 @@ public void initNormalizeAndCheckProps() { public Catalog initCatalog(String catalogName, Map catalogProps, List storagePropertiesList) { try { - catalogProps.put(CatalogProperties.CATALOG_IMPL, CatalogUtil.ICEBERG_CATALOG_HIVE); Configuration conf = buildHiveConfiguration(storagePropertiesList); return this.executionAuthenticator.execute(() -> - buildIcebergCatalog(catalogName, catalogProps, conf)); + IcebergUtils.createIcebergHiveCatalog(catalogName, catalogProps, conf)); } catch (Exception e) { throw new RuntimeException("Failed to initialize HiveCatalog for Iceberg. " + "CatalogName=" + catalogName + ", msg :" + ExceptionUtils.getRootCauseMessage(e), e); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java new file mode 100644 index 00000000000000..f21956835d5532 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.iceberg.io.FileIO; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; + +class DorisHiveCatalogTest { + + @Test + void closesOwnedFileIOOnceAcrossRepeatedRetirement() throws Exception { + DorisHiveCatalog catalog = new DorisHiveCatalog(); + FileIO fileIO = Mockito.mock(FileIO.class); + Field ownedFileIO = DorisHiveCatalog.class.getDeclaredField("ownedFileIO"); + ownedFileIO.setAccessible(true); + ownedFileIO.set(catalog, fileIO); + + catalog.close(); + catalog.close(); + + Mockito.verify(fileIO, Mockito.times(1)).close(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 4c88db354b0f22..d7e091da6d1069 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -229,6 +229,7 @@ public void testParseTableName() { IcebergHMSExternalCatalog c1 = new IcebergHMSExternalCatalog(1, "name", null, new HashMap<>(), ""); HiveCatalog i1 = IcebergUtils.createIcebergHiveCatalog(c1, "i1"); + Assert.assertTrue(i1 instanceof DorisHiveCatalog); Assert.assertTrue(getListAllTables(i1)); IcebergHMSExternalCatalog c2 = @@ -259,7 +260,7 @@ public void testParseTableName() { } private boolean getListAllTables(HiveCatalog hiveCatalog) throws IllegalAccessException, NoSuchFieldException { - Field declaredField = hiveCatalog.getClass().getDeclaredField("listAllTables"); + Field declaredField = HiveCatalog.class.getDeclaredField("listAllTables"); declaredField.setAccessible(true); return declaredField.getBoolean(hiveCatalog); } From 86fc52b36ad3cdb8bb070a15397800c613ae9690 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 25 Aug 2026 20:06:50 +0800 Subject: [PATCH 19/38] [fix](fe) Reconcile Iceberg lifecycle with metadata cache governance Issue Number: close #66913 Related PR: #66914 Problem Summary: Rebasing the Iceberg and Hudi handle leak fix onto branch-4.1 introduced the new external metadata cache memory-governance implementation. The old lifecycle APIs no longer compiled, and a textual merge would either lose exact-generation ownership or bypass the new weight, refresh, and snapshot-isolation contracts. This change integrates statement and asynchronous borrowers with the governed cache, retires replaced and removed Iceberg generations only after their final borrower, closes table-owned FileIO and catalog generations in the captured authentication scope, and preserves Hudi removal cleanup when a cache group closes. It also removes stale pre-governance API residue and updates lifecycle regression tests to exercise the governed refresh path. Fix Iceberg and Hudi metadata handle leaks during cache refresh, eviction, invalidation, and catalog reset. - Test: Unit Test and FE build - ./run-fe-ut.sh targeted Iceberg, Hudi, and StatementContext tests - ./build.sh --fe - Behavior changed: Yes (external metadata handles are closed after their exact generation is no longer borrowed) - Does this need documentation: No --- .../datasource/hive/HMSExternalCatalog.java | 3 +- .../iceberg/IcebergExternalCatalog.java | 2 +- .../iceberg/IcebergExternalMetaCache.java | 254 +++++++++++++++--- .../iceberg/IcebergTableCacheValue.java | 105 ++++++++ .../datasource/metacache/MetaCacheEntry.java | 35 ++- .../metacache/MetaCacheEntryDef.java | 2 - .../StaleMetaCacheEntryException.java | 25 -- .../iceberg/IcebergExternalMetaCacheTest.java | 36 ++- .../iceberg/IcebergTableCacheValueTest.java | 72 +++-- .../iceberg/source/IcebergScanNodeTest.java | 23 -- 10 files changed, 428 insertions(+), 129 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 1e6be442b33c74..4f259e9289483d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractHiveProperties; import org.apache.doris.datasource.property.metastore.MetastoreProperties; @@ -298,7 +299,7 @@ public synchronized IcebergTableLoadContext beginIcebergTableLoad() { @Override public synchronized void resetToUninitialized(boolean invalidCache) { ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - cacheMgr.runCatalogLifecycle(getId(), () -> resetCatalogRuntime(cacheMgr, invalidCache)); + resetCatalogRuntime(cacheMgr, invalidCache); } private void resetCatalogRuntime(ExternalMetaCacheMgr cacheMgr, boolean invalidCache) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 007c3b1a0ed9af..35a5616a7340f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -205,7 +205,7 @@ public synchronized void onClose() { @Override public synchronized void resetToUninitialized(boolean invalidCache) { ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - cacheMgr.runCatalogLifecycle(getId(), () -> resetCatalogRuntime(cacheMgr, invalidCache)); + resetCatalogRuntime(cacheMgr, invalidCache); } private void resetCatalogRuntime(ExternalMetaCacheMgr cacheMgr, boolean invalidCache) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index b58e659abfce76..c8eeb51e5b7ff4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -36,6 +36,8 @@ import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.qe.ConnectContext; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.ManifestContent; @@ -49,6 +51,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; @@ -56,6 +59,7 @@ import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.Nullable; @@ -110,7 +114,12 @@ public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCac this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) .withSizeEstimator(this::prepareTableForCachePublication) - .withReplacementListener(this::retireTableGeneration)); + .withReplacementListener(this::retireTableGeneration) + .withRemovalListener(value -> value, (key, value) -> { + if (value != null) { + retireRemovedTableGeneration(key, value); + } + })); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) @@ -133,7 +142,32 @@ IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCach public Table getIcebergTable(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getIcebergTable(); + IcebergTableCacheValue.Lease lease = statementLease(nameMapping); + if (lease != null) { + return lease.getIcebergTable(); + } + // Background callers have no deterministic statement boundary. Use a live catalog load + // instead of returning a cache generation that can be evicted immediately after lookup. + return getWritableIcebergTable(dorisTable); + } + + ThreadPoolExecutor getIcebergTableExecutor(ExternalTable dorisTable) { + IcebergTableCacheValue.Lease lease = statementLease(dorisTable.getOrBuildNameMapping()); + if (lease == null || lease.getPlanningExecutor() == null) { + return dorisTable.getCatalog().getThreadPoolWithPreAuth(); + } + return lease.getPlanningExecutor(); + } + + T withIcebergTable(ExternalTable dorisTable, Function action) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + IcebergTableCacheValue.Lease lease = statementLease(nameMapping); + if (lease != null) { + return action.apply(lease.getIcebergTable()); + } + try (IcebergTableCacheValue.Lease operationLease = borrow(nameMapping)) { + return action.apply(operationLease.getIcebergTable()); + } } public Table getWritableIcebergTable(ExternalTable dorisTable) { @@ -172,17 +206,14 @@ public Table getWritableIcebergTable(ExternalTable dorisTable, @Nullable Iceberg Table getQueryScopedIcebergTable(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - MetaCacheEntry entry = - tableEntry.get(nameMapping.getCtlId()); - IcebergTableCacheValue tableValue = - entry.get(nameMapping); + IcebergTableCacheValue tableValue = statementValue(nameMapping); return createQueryTable(nameMapping, tableValue); } /** Resolve the current table generation, exposing the handle and its captured context together. */ IcebergTableCacheValue getTableCacheValue(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return statementValue(nameMapping); } /** Query-scoped view of an already-resolved generation; see {@link #getTableCacheValue}. */ @@ -204,8 +235,7 @@ private Table createQueryTable( public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - IcebergTableCacheValue tableValue = - tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + IcebergTableCacheValue tableValue = statementValue(nameMapping); Table retainedTable = tableValue.getRetainedIcebergTable(); java.util.Optional optionalKey = IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); @@ -282,7 +312,7 @@ public View getIcebergView(ExternalTable dorisTable) { } public IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId) { - IcebergTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + IcebergTableCacheValue tableValue = statementValue(nameMapping); return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable()); } @@ -358,16 +388,52 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); } - // One catalog generation must supply the ops, the loaded table and the bound - // authenticator together: re-reading the mutable catalog after the load could bind a - // handle of the old generation to the execution context a concurrent ALTER installed. - // The acquisition is re-validated before publication; a mid-flight reset fails the miss - // (the caller retries against the reinitialized catalog) instead of publishing a splice. - ExecutionAuthenticator authenticator = requireExecutionAuthenticator(catalog); - IcebergMetadataOps ops = resolveMetadataOps(catalog); - IcebergTableCacheValue value = execute(authenticator, () -> { - Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); - IcebergTableCacheValue loaded = new IcebergTableCacheValue(table); + if (catalog instanceof IcebergExternalCatalog) { + IcebergExternalCatalog icebergCatalog = (IcebergExternalCatalog) catalog; + try (IcebergExternalCatalog.TableLoadContext context = icebergCatalog.beginTableLoad()) { + IcebergMetadataOps ops = context.getOps(); + Table table; + try { + table = context.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping); + Runnable tableCleanup = tableCleanup(context.getCatalogType(), ops, table); + IcebergCatalogResourceTracker.ResourceLease catalogLease = context.promote(); + Runnable cleanup = () -> { + try { + tableCleanup.run(); + } finally { + catalogLease.close(); + } + }; + return execute(context.getAuthenticator(), () -> createLoadedTableValue( + nameMapping, table, ops.getThreadPoolWithPreAuth(), context.getAuthenticator(), cleanup)); + } + } + if (catalog instanceof HMSExternalCatalog) { + HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; + try (HMSExternalCatalog.IcebergTableLoadContext context = hmsCatalog.beginIcebergTableLoad()) { + Table table; + try { + table = context.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + IcebergCatalogResourceTracker.ResourceLease catalogLease = context.promote(); + return execute(context.getAuthenticator(), () -> createLoadedTableValue( + nameMapping, table, context.getExecutor(), context.getAuthenticator(), catalogLease::close)); + } + } + + throw new RuntimeException("Only support 'hms' and 'iceberg' type for iceberg table"); + } + + private IcebergTableCacheValue createLoadedTableValue(NameMapping nameMapping, Table table, + ThreadPoolExecutor planningExecutor, ExecutionAuthenticator authenticator, Runnable cleanup) { + IcebergTableCacheValue loaded = new IcebergTableCacheValue(table, planningExecutor, () -> null, cleanup); + try { loaded.bindAuthenticator(authenticator); MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); @@ -375,11 +441,96 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { prepareTableForCachePublication(nameMapping, loaded); } return loaded; - }); - ensureCatalogGenerationStable(catalog, ops, authenticator, nameMapping); + } catch (RuntimeException | Error e) { + loaded.retire(); + throw e; + } + } + + private IcebergTableCacheValue statementValue(NameMapping nameMapping) { + IcebergTableCacheValue.Lease lease = statementLease(nameMapping); + if (lease != null) { + return lease.getValue(); + } + IcebergTableCacheValue value = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + value.releaseLoaderReference(); return value; } + @Nullable + private IcebergTableCacheValue.Lease statementLease(NameMapping nameMapping) { + ConnectContext connectContext = ConnectContext.get(); + StatementContext statementContext = connectContext == null ? null : connectContext.getStatementContext(); + if (statementContext == null) { + return null; + } + String resourceKey = "iceberg-table:" + nameMapping.getCtlId() + "\u0000" + + nameMapping.getRemoteDbName() + "\u0000" + nameMapping.getRemoteTblName(); + return statementContext.getOrRegisterStatementResource(resourceKey, () -> borrow(nameMapping)); + } + + private IcebergTableCacheValue.Lease borrow(NameMapping nameMapping) { + MetaCacheEntry entry = tableEntry.get(nameMapping.getCtlId()); + while (true) { + IcebergTableCacheValue value = entry.get(nameMapping); + IcebergTableCacheValue.Lease lease = value.tryAcquire(); + if (entry.peekIfPresent(nameMapping) != value) { + // Disabled, weight-rejected and invalidation-suppressed loads have no cache owner. + // Retire that owner here; the just-acquired lease remains the sole use boundary. + value.releaseCacheReference(); + } + value.releaseLoaderReference(); + if (lease != null) { + return lease; + } + } + } + + private Runnable tableCleanup(String catalogType, IcebergMetadataOps ops, Table table) { + FileIO catalogFileIO = IcebergExternalCatalog.ICEBERG_REST.equals(catalogType) ? catalogFileIO(ops) : null; + if (!shouldCloseTableFileIO(catalogType, table.io(), catalogFileIO)) { + return () -> { }; + } + FileIO tableFileIO = table.io(); + return () -> { + try { + tableFileIO.close(); + } catch (Exception e) { + LOG.warn("Failed to close Iceberg table FileIO", e); + } + }; + } + + static boolean shouldCloseTableFileIO(String catalogType, FileIO tableFileIO, FileIO catalogFileIO) { + if (IcebergExternalCatalog.ICEBERG_GLUE.equals(catalogType) + || IcebergExternalCatalog.ICEBERG_S3_TABLES.equals(catalogType)) { + return true; + } + return IcebergExternalCatalog.ICEBERG_REST.equals(catalogType) + && catalogFileIO != null && tableFileIO != catalogFileIO; + } + + @Nullable + private FileIO catalogFileIO(IcebergMetadataOps ops) { + Object catalog = ops.getCatalog(); + try { + if (catalog instanceof org.apache.iceberg.rest.RESTCatalog) { + Field sessionCatalogField = org.apache.iceberg.rest.RESTCatalog.class + .getDeclaredField("sessionCatalog"); + sessionCatalogField.setAccessible(true); + catalog = sessionCatalogField.get(catalog); + } + if (catalog instanceof org.apache.iceberg.rest.RESTSessionCatalog) { + Field ioField = org.apache.iceberg.rest.RESTSessionCatalog.class.getDeclaredField("io"); + ioField.setAccessible(true); + return (FileIO) ioField.get(catalog); + } + } catch (Exception e) { + LOG.warn("Failed to identify REST catalog FileIO; skip per-table close to protect shared IO", e); + } + return null; + } + private static ExecutionAuthenticator requireExecutionAuthenticator(CatalogIf catalog) { if (!(catalog instanceof ExternalCatalog)) { throw new RuntimeException("Iceberg metadata cache requires an external catalog"); @@ -490,24 +641,51 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table r private void retireTableGeneration(NameMapping nameMapping, @Nullable IcebergTableCacheValue previousValue, IcebergTableCacheValue currentValue) { - if (previousValue != null && previousValue.isSameOperationalGeneration(currentValue)) { + if (previousValue == null) { return; } - MetaCacheEntry snapshots = - snapshotEntry.getIfInitialized(nameMapping.getCtlId()); - if (snapshots != null) { - // Projections of another metadata generation are unreachable. Projections of the same - // generation frozen on a previous handle keep that handle's FileIO (vended credentials) - // and location provider; scans bind to them, so they must be rebuilt from the new handle. - snapshots.invalidateIf((key, value) -> key.getNameMapping().equals(nameMapping) - && (!key.belongsTo(currentValue) || !sharesOperationalResources(currentValue, value))); - } - Optional currentUuid = currentValue.getTableUuid(); - MetaCacheEntry schemas = - schemaEntry.getIfInitialized(nameMapping.getCtlId()); - if (schemas != null) { - schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) - && !key.getTableUuid().equals(currentUuid)); + try { + if (previousValue.isSameOperationalGeneration(currentValue)) { + return; + } + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + // Projections of another metadata generation are unreachable. Projections of the same + // generation frozen on a previous handle keep that handle's FileIO (vended credentials) + // and location provider; scans bind to them, so they must be rebuilt from the new handle. + snapshots.invalidateIf((key, value) -> key.getNameMapping().equals(nameMapping) + && (!key.belongsTo(currentValue) || !sharesOperationalResources(currentValue, value))); + } + Optional currentUuid = currentValue.getTableUuid(); + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.getTableUuid().equals(currentUuid)); + } + } finally { + // Caffeine REPLACED notifications intentionally do not run the removal listener because + // the cache reservation transfers to the new generation. Resource ownership does not: + // retire the old value here and let active statement/async leases delay physical close. + previousValue.retire(); + } + } + + private void retireRemovedTableGeneration(NameMapping nameMapping, IcebergTableCacheValue removedValue) { + try { + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping)); + } + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping)); + } + } finally { + removedValue.retire(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index 8d0dd33641260e..4245d02c94ec5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -27,12 +27,23 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.SupportsStorageCredentials; +import java.io.Closeable; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import javax.annotation.Nullable; public class IcebergTableCacheValue { private volatile Table icebergTable; + @Nullable + private final ThreadPoolExecutor planningExecutor; + private final Runnable cleanup; + private final AtomicInteger references; + private final AtomicBoolean cacheReferenceReleased = new AtomicBoolean(); + private final AtomicBoolean loaderReferenceReleased; // The execution authenticator active when this generation was loaded; see the Paimon // counterpart for the concurrent catalog-reset rationale. @Nullable @@ -43,7 +54,64 @@ public class IcebergTableCacheValue { private MetaCacheSizeEstimate sizeEstimate; public IcebergTableCacheValue(Table icebergTable) { + this(icebergTable, null, () -> null, () -> { }, false); + } + + IcebergTableCacheValue(Table icebergTable, Supplier ignoredSnapshotSupplier, + Runnable cleanup) { + this(icebergTable, null, ignoredSnapshotSupplier, cleanup, true); + } + + IcebergTableCacheValue(Table icebergTable, ThreadPoolExecutor planningExecutor, + Supplier ignoredSnapshotSupplier, Runnable cleanup) { + this(icebergTable, planningExecutor, ignoredSnapshotSupplier, cleanup, true); + } + + private IcebergTableCacheValue(Table icebergTable, @Nullable ThreadPoolExecutor planningExecutor, + Supplier ignoredSnapshotSupplier, Runnable cleanup, boolean loading) { this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); + this.planningExecutor = planningExecutor; + this.cleanup = Objects.requireNonNull(cleanup, "cleanup"); + Objects.requireNonNull(ignoredSnapshotSupplier, "snapshot supplier"); + this.references = new AtomicInteger(loading ? 2 : 1); + this.loaderReferenceReleased = new AtomicBoolean(!loading); + } + + Lease tryAcquire() { + int current = references.get(); + while (current != 0) { + if (references.compareAndSet(current, current + 1)) { + return new Lease(this); + } + current = references.get(); + } + return null; + } + + void releaseCacheReference() { + if (cacheReferenceReleased.compareAndSet(false, true)) { + release(); + } + } + + void releaseLoaderReference() { + if (loaderReferenceReleased.compareAndSet(false, true)) { + release(); + } + } + + void retire() { + releaseCacheReference(); + releaseLoaderReference(); + } + + private void release() { + int remaining = references.decrementAndGet(); + if (remaining == 0) { + cleanup.run(); + } else if (remaining < 0) { + throw new IllegalStateException("Iceberg table cache value released too many times"); + } } void bindAuthenticator( @@ -228,4 +296,41 @@ private TableMetadata retainedMetadata() { return retainedTable instanceof HasTableOperations ? ((HasTableOperations) retainedTable).operations().current() : null; } + + static final class Lease implements Closeable { + private final IcebergTableCacheValue value; + private final AtomicBoolean closed = new AtomicBoolean(); + + private Lease(IcebergTableCacheValue value) { + this.value = value; + } + + Table getIcebergTable() { + return value.getIcebergTable(); + } + + @Nullable + ThreadPoolExecutor getPlanningExecutor() { + return value.planningExecutor; + } + + IcebergTableCacheValue getValue() { + return value; + } + + Lease retain() { + Lease retained = value.tryAcquire(); + if (retained == null) { + throw new IllegalStateException("Iceberg table cache generation was already retired"); + } + return retained; + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + value.release(); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 64a52f1a307dc2..0566a2dd8024c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -521,7 +521,7 @@ public void close() { return; } invalidateAll(); - pendingRemovalNotifications.clear(); + drainRemovalNotificationsOnClose(); if (entryBudget != null) { entryBudget.close(); } @@ -835,14 +835,15 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (!weightBounded && !generationFencedRefresh && removalListener == null) { return; } - if (closed.get()) { - return; - } // Replacement transfers the existing reservation to the newly published generation. A // soft-value collection instead reports a null value with COLLECTED and must release it. if (cause == RemovalCause.REPLACED) { return; } + if (closed.get()) { + invokeRemovalListener(key, removalToken(value)); + return; + } // The dead reservation is queued before the dependency notification so the shared // cleanup worker can always release quota before entering potentially expensive listener // work; see drainRemovalCleanups. @@ -986,12 +987,26 @@ private void drainRemovalNotifications() { if (removed == null || closed.get()) { return; } - try { - removalListener.onRemoval(removed.key, removed.token); - } catch (RuntimeException e) { - LOG.warn("Failed to retire dependencies after removing external metadata cache entry {}", - name, e); - } + invokeRemovalListener(removed.key, removed.token); + } + } + + private void drainRemovalNotificationsOnClose() { + RemovedToken removed; + while ((removed = pendingRemovalNotifications.poll()) != null) { + invokeRemovalListener(removed.key, removed.token); + } + } + + private void invokeRemovalListener(K key, @Nullable Object token) { + if (removalListener == null) { + return; + } + try { + removalListener.onRemoval(key, token); + } catch (RuntimeException e) { + LOG.warn("Failed to retire dependencies after removing external metadata cache entry {}", + name, e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index da843acae27b0f..25f7a21626af73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -17,8 +17,6 @@ package org.apache.doris.datasource.metacache; -import com.github.benmanes.caffeine.cache.RemovalListener; - import java.util.Objects; import java.util.function.Function; import javax.annotation.Nullable; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java deleted file mode 100644 index c7e19360e15cb2..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/StaleMetaCacheEntryException.java +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.metacache; - -/** Signals that a caller raced catalog-group retirement and must resolve the current entry again. */ -public class StaleMetaCacheEntryException extends IllegalStateException { - public StaleMetaCacheEntryException(String message) { - super(message); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index a6975eb020ed79..3e61a28ae2eb8d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -85,6 +85,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; @@ -179,7 +180,7 @@ public void testSnapshotKeyIncludesMetadataGeneration() { } @Test - public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); try { @@ -212,6 +213,12 @@ public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { tables.invalidateKey(mapping); tables.put(mapping, second); + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while ((snapshots.peekIfPresent(oldSnapshotKey) != null + || schemas.peekIfPresent(oldSchemaKey) != null) + && System.nanoTime() < deadlineNanos) { + TimeUnit.MILLISECONDS.sleep(10L); + } Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); } finally { @@ -360,6 +367,7 @@ public T execute(java.util.concurrent.Callable task) throws Exception { Mockito.when(catalog.getMetadataOps()).thenAnswer(invocation -> currentOps.get()); Mockito.when(catalog.getExecutionAuthenticator()).thenAnswer( invocation -> currentAuthenticator.get()); + stubTableLoadContext(catalog); Table table = tableWithMetadataLocation("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/metadata/coherent-acquisition-v1.json"); org.mockito.stubbing.Answer
loadFlipsGeneration = invocation -> { if (alterCompletesDuringLoad.get()) { @@ -445,6 +453,7 @@ public T execute(java.util.concurrent.Callable task) throws Exception { return authenticator; }); Mockito.when(catalog.getMetadataOps()).thenReturn(currentOps); + stubTableLoadContext(catalog); Table table = tableWithMetadataLocation("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/metadata/reset-before-capture-v1.json"); Mockito.when(currentOps.loadTable("remote_db", "remote_tbl")).thenReturn(table); ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -643,6 +652,26 @@ private static boolean exceptionChainContains(Throwable throwable, String fragme return false; } + private static void stubTableLoadContext(IcebergExternalCatalog catalog) { + Mockito.when(catalog.beginTableLoad()).thenAnswer(invocation -> { + catalog.makeSureInitialized(); + IcebergMetadataOps ops = (IcebergMetadataOps) catalog.getMetadataOps(); + ExecutionAuthenticator authenticator = catalog.getExecutionAuthenticator(); + String catalogType = catalog.getIcebergCatalogType(); + IcebergExternalCatalog.TableLoadContext context = + Mockito.mock(IcebergExternalCatalog.TableLoadContext.class); + Mockito.when(context.getOps()).thenReturn(ops); + Mockito.when(context.getAuthenticator()).thenReturn(authenticator); + Mockito.when(context.getCatalogType()).thenReturn(catalogType); + Mockito.when(context.loadTable(Mockito.anyString(), Mockito.anyString())).thenAnswer(load -> + authenticator.execute(() -> ops.loadTable(load.getArgument(0), load.getArgument(1)))); + IcebergCatalogResourceTracker.ResourceLease lease = + Mockito.mock(IcebergCatalogResourceTracker.ResourceLease.class); + Mockito.when(context.promote()).thenReturn(lease); + return context; + }); + } + @Test public void testRejectedTableGenerationsDoNotAccumulateSnapshotOrSchemaProjections() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -655,6 +684,7 @@ public T execute(Callable task) throws Exception { return task.call(); } }); + stubTableLoadContext(catalog); // Every reload advances the metadata location; every publication is rejected. Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenReturn( tableWithMetadataLocation("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/metadata/rejected-v1.json"), @@ -737,6 +767,7 @@ public T execute(Callable task) throws Exception { return task.call(); } }); + stubTableLoadContext(catalog); TableMetadata metadata = metadataWithLocation("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/metadata/ineffective-base.json"); Table firstHandle = tableWithMetadata(metadata, new PropertiesFileIO("token", "one")); Table sameCredentials = tableWithMetadata(metadata, new PropertiesFileIO("token", "one")); @@ -801,6 +832,7 @@ public T execute(Callable task) throws Exception { return task.call(); } }); + stubTableLoadContext(catalog); Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenAnswer( invocation -> tableWithMetadataLocation("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/metadata/ineffective-weighted.json")); AtomicInteger preparations = new AtomicInteger(); @@ -1733,6 +1765,7 @@ public T execute(Callable task) throws Exception { } } }); + stubTableLoadContext(catalog); IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { @Override protected CatalogIf getCatalog(long catalogId) { @@ -1892,6 +1925,7 @@ public void testPinnedGenerationSurvivesMetadataFileRetirement() throws Exceptio IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + stubTableLoadContext(catalog); Mockito.when(metadataOps.loadTable("db", "tbl")).thenReturn(freshTable); IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java index 4c306cf0bacf60..75a4b40bbb847a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -21,17 +21,15 @@ import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.nereids.StatementContext; -import com.github.benmanes.caffeine.cache.LoadingCache; import org.apache.iceberg.Table; import org.apache.iceberg.io.FileIO; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; @@ -56,22 +54,6 @@ void leaseKeepsExecutorFromItsTableGeneration() { } } - @Test - void backgroundSnapshotCopyDropsRuntimeGenerationOwners() { - ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); - try { - IcebergSnapshotCacheValue runtimeValue = new IcebergSnapshotCacheValue( - null, null, Optional.empty(), newProxy(Table.class), executor); - - IcebergSnapshotCacheValue detached = runtimeValue.metadataOnlyCopy(); - - Assertions.assertFalse(detached.getIcebergTable().isPresent()); - Assertions.assertNull(detached.getPlanningExecutor()); - } finally { - executor.shutdownNow(); - } - } - @Test void classifiesOnlyPerTableFileIOAsOwned() { FileIO tableIo = newProxy(FileIO.class); @@ -190,13 +172,19 @@ void refreshPublishesNewGenerationWithoutClosingActiveOldBorrower() throws Excep cleanupCounts.add(cleanupCount); return newValue(cleanupCount); }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, true, false, - (key, value) -> value.retire()); + null, null, + (key, previous, current) -> { + if (previous != null) { + previous.retire(); + } + }, value -> value, + (key, value) -> ((IcebergTableCacheValue) value).retire()); IcebergTableCacheValue first = entry.get("table"); IcebergTableCacheValue.Lease oldBorrower = first.tryAcquire(); Assertions.assertNotNull(oldBorrower); first.releaseLoaderReference(); - extractLoadingCache(entry).refresh("table"); + triggerRefresh(entry, "table"); refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); Assertions.assertEquals(2, cleanupCounts.size()); @@ -205,15 +193,37 @@ void refreshPublishesNewGenerationWithoutClosingActiveOldBorrower() throws Excep Assertions.assertEquals(0, cleanupCounts.get(0).get()); oldBorrower.close(); + awaitCleanup(cleanupCounts.get(0)); Assertions.assertEquals(1, cleanupCounts.get(0).get()); entry.invalidateKey("table"); - refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + awaitCleanup(cleanupCounts.get(1)); Assertions.assertEquals(1, cleanupCounts.get(1).get()); } finally { refreshExecutor.shutdownNow(); } } + @Test + void closingCacheRetiresPublishedGeneration() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger cleanupCount = new AtomicInteger(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>("iceberg-table", + key -> newValue(cleanupCount), CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, null, null, null, value -> value, + (key, value) -> ((IcebergTableCacheValue) value).retire()); + IcebergTableCacheValue value = entry.get("table"); + value.releaseLoaderReference(); + + entry.close(); + + awaitCleanup(cleanupCount); + Assertions.assertEquals(1, cleanupCount.get()); + } finally { + refreshExecutor.shutdownNow(); + } + } + @Test void catalogRetirementWaitsForTableEvictionAndBorrower() { IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); @@ -240,12 +250,18 @@ private IcebergTableCacheValue newValue(AtomicInteger cleanupCount) { return new IcebergTableCacheValue(table, () -> null, cleanupCount::incrementAndGet); } - @SuppressWarnings("unchecked") - private LoadingCache extractLoadingCache( - MetaCacheEntry entry) throws Exception { - Field field = MetaCacheEntry.class.getDeclaredField("loadingData"); - field.setAccessible(true); - return (LoadingCache) field.get(entry); + private void awaitCleanup(AtomicInteger cleanupCount) throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (cleanupCount.get() == 0 && System.nanoTime() < deadlineNanos) { + TimeUnit.MILLISECONDS.sleep(10L); + } + } + + private void triggerRefresh(MetaCacheEntry entry, String key) + throws Exception { + Method method = MetaCacheEntry.class.getDeclaredMethod("triggerRefreshForTest", Object.class); + method.setAccessible(true); + method.invoke(entry, key); } @SuppressWarnings("unchecked") diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 2525dd21ba6b67..bb5c4ec419ad35 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -553,29 +553,6 @@ public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefres } } - @Test - public void testPlanningExecutorComesFromPinnedSnapshotGeneration() throws Exception { - IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); - IcebergSource source = Mockito.mock(IcebergSource.class); - Mockito.when(source.getTargetTable()).thenReturn(targetTable); - TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); - setIcebergSource(node, source); - - ThreadPoolExecutor frozenExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); - Table frozenTable = Mockito.mock(Table.class); - node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( - new IcebergSnapshotCacheValue(new IcebergPartitionInfo( - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), - new IcebergSnapshot(1L, 1L), Optional.empty(), frozenTable, frozenExecutor)))); - try { - Method method = IcebergScanNode.class.getDeclaredMethod("getPlanningExecutor"); - method.setAccessible(true); - Assert.assertSame(frozenExecutor, method.invoke(node)); - } finally { - frozenExecutor.shutdownNow(); - } - } - private static class CountPlanningIcebergScanNode extends IcebergScanNode { private final TableScan tableScan; private final long snapshotCount; From 062edf620295c2d18036ce2733c22ad13bbf4edd Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 26 Aug 2026 10:37:01 +0800 Subject: [PATCH 20/38] [fix](fe) Close external metadata lifecycle races ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: Closing a branch-4.1 external metadata cache could race asynchronous removal publication or cleanup after a worker had claimed a token, dropping dependency retirement callbacks. Hudi batch split cancellation was also only owned by StatementContext, so SplitAssignment cancellation could leave started filesystem tasks running without promptly requesting interruption. Give each removal token an exact single owner across publication, cleanup, and close; register the Hudi batch owner with SplitAssignment and make cancellation idempotently interrupt all tracked tasks while retaining the fs-view lease until terminal accounting completes. ### Release note None ### Check List (For Author) - Test: Unit Test - MetaCacheEntryTest, SplitAssignmentTest, HudiFsViewCacheValueTest, HudiBatchFsViewOwnerTest, HudiScanNodeTest - ./build.sh --fe - Behavior changed: No - Does this need documentation: No --- .../datasource/hudi/source/HudiScanNode.java | 17 +++-- .../datasource/metacache/MetaCacheEntry.java | 22 +++++- .../hudi/source/HudiBatchFsViewOwnerTest.java | 39 ++++++++++ .../metacache/MetaCacheEntryTest.java | 76 +++++++++++++++++++ 4 files changed, 144 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 45cb91e01087ba..df8f3ef3dd9df7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -662,6 +662,7 @@ public void startSplit(int numBackends) { } BatchFsViewOwner finalBatchOwner = batchOwner; + splitAssignment.addCloseable(finalBatchOwner); AtomicInteger pendingTasks = new AtomicInteger(1); // producer reference Runnable taskFinished = () -> { if (pendingTasks.decrementAndGet() == 0) { @@ -842,16 +843,16 @@ void track(TerminalTask task) { @Override public void close() { - if (!finished.get()) { - stopping.set(true); - try { - splitAssignment.stop(); - } catch (RuntimeException e) { - tasks.forEach(TerminalTask::requestStop); - throw e; - } + if (finished.get() || !stopping.compareAndSet(false, true)) { + return; + } + try { + splitAssignment.stop(); + } catch (RuntimeException e) { tasks.forEach(TerminalTask::requestStop); + throw e; } + tasks.forEach(TerminalTask::requestStop); // Already-started filesystem calls may be blocked in storage code that does not respond to // interruption. Their TerminalTask.done callbacks retain exact task accounting and eventually call // finish(), which releases the fs-view lease only after the last task exits. Cancellation must return diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 0566a2dd8024c5..debb3eaa8cf031 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -897,7 +897,18 @@ private void queueRemovalNotification(K key, @Nullable V value) { if (removalListener == null) { return; } - pendingRemovalNotifications.add(new RemovedToken<>(key, removalToken(value))); + RemovedToken removed = new RemovedToken<>(key, removalToken(value)); + if (closed.get()) { + invokeRemovalListener(removed.key, removed.token); + return; + } + pendingRemovalNotifications.add(removed); + if (closed.get() && pendingRemovalNotifications.remove(removed)) { + // close() drained before this publication. Claim the exact token here; if removal + // fails, the close drain or a worker already owns it and must invoke the listener. + invokeRemovalListener(removed.key, removed.token); + return; + } scheduleRemovalCleanup(); } @@ -984,9 +995,12 @@ private void drainRemovalNotifications() { } for (int processed = 0; processed < REMOVAL_CLEANUP_BATCH_SIZE; processed++) { RemovedToken removed = pendingRemovalNotifications.poll(); - if (removed == null || closed.get()) { + if (removed == null) { return; } + afterRemovalNotificationPollForTest(removed.key); + // A successful poll transfers ownership to this worker. close() cannot see this token + // in its final drain, so the worker must invoke it even when close raced the poll. invokeRemovalListener(removed.key, removed.token); } } @@ -1352,6 +1366,10 @@ void afterRemovalCleanupForTest(K key) { void beforeRemovalCleanupLockForTest(K key) { } + // Called after a cleanup worker claims a dependency-retirement token from the queue. + void afterRemovalNotificationPollForTest(K key) { + } + // Let tests establish admissionLock -> Caffeine eviction-lock ordering deterministically. void beforeWeightedInvalidateAllForTest() { } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index dab6094d5e52eb..f0ffd4b51e70ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.Collections; import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -119,6 +120,44 @@ void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { } } + @Test + void assignmentStopInterruptsStartedTaskAndRetainsLeaseUntilTerminal() throws Exception { + SplitAssignment assignment = new SplitAssignment( + null, null, null, Collections.emptyMap(), Collections.emptyList(), false); + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + assignment.addCloseable(owner); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask(() -> { + started.countDown(); + while (release.getCount() > 0) { + try { + release.await(3, TimeUnit.SECONDS); + } catch (InterruptedException e) { + interrupted.countDown(); + } + } + }, owner::finish); + owner.track(task); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + executor.execute(task); + Assertions.assertTrue(started.await(3, TimeUnit.SECONDS)); + + assignment.stop(); + + Assertions.assertTrue(interrupted.await(3, TimeUnit.SECONDS)); + Mockito.verify(lease, Mockito.never()).close(); + release.countDown(); + Mockito.verify(lease, Mockito.timeout(3000)).close(); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + @Test void synchronousListingCancellationReturnsBeforeBlockedTaskAndRetainsLease() throws Exception { HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 6c84c576655b28..3e16ec48fde035 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1915,6 +1915,82 @@ public void testRemovalListenerReceivesRemovedValuesButNotReplacements() throws } } + @Test + public void testCloseDoesNotDropRemovalTokenClaimedByCleanupWorker() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch tokenClaimed = new CountDownLatch(1); + CountDownLatch releaseClaimedToken = new CountDownLatch(1); + CountDownLatch retired = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry( + "close-after-token-poll", key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false, false, null, null, null, + value -> value, (key, token) -> retired.countDown()) { + @Override + void afterRemovalNotificationPollForTest(String key) { + tokenClaimed.countDown(); + awaitLatch(releaseClaimedToken); + } + }; + try { + entry.put("k", 1); + entry.invalidateKey("k"); + Assert.assertTrue(tokenClaimed.await(3L, TimeUnit.SECONDS)); + + entry.close(); + Assert.assertEquals("the worker still owns the polled token", 1L, retired.getCount()); + releaseClaimedToken.countDown(); + + Assert.assertTrue(retired.await(3L, TimeUnit.SECONDS)); + } finally { + releaseClaimedToken.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRemovalPublicationAfterCloseDrainInvokesListenerInline() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService removalExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch callbackPassedOpenCheck = new CountDownLatch(1); + CountDownLatch releaseCallback = new CountDownLatch(1); + CountDownLatch retired = new CountDownLatch(1); + AtomicBoolean pauseCallback = new AtomicBoolean(); + MetaCacheEntry entry = new MetaCacheEntry( + "publish-after-close-drain", key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false, false, null, null, null, + value -> value, (key, token) -> retired.countDown()) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (pauseCallback.compareAndSet(true, false)) { + callbackPassedOpenCheck.countDown(); + awaitLatch(releaseCallback); + } + } + }; + try { + entry.put("k", 1); + pauseCallback.set(true); + LoadingCache loadingCache = extractLoadingCache(entry); + Future removal = removalExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(callbackPassedOpenCheck.await(3L, TimeUnit.SECONDS)); + + entry.close(); + Assert.assertEquals("close drained before the callback published", 1L, retired.getCount()); + releaseCallback.countDown(); + removal.get(3L, TimeUnit.SECONDS); + + Assert.assertTrue(retired.await(3L, TimeUnit.SECONDS)); + } finally { + releaseCallback.countDown(); + entry.close(); + removalExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testLocalEvictionStopsOnceTheDeficitIsReclaimed() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); From a6dd59b1df5247203e396988dc56ddee8f8a8c95 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 27 Aug 2026 08:11:41 +0800 Subject: [PATCH 21/38] [fix](fe) Restore external catalog runtime state after replay ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: After an FE upgrade or restart loads an Iceberg or HMS catalog from a metadata checkpoint, Gson bypasses Java field initializers. The runtime-only Iceberg resource trackers and HMS runtime generation therefore remain null, causing Iceberg table loads to fail and leaving HMS lifecycle fencing unusable. Recreate these runtime-only objects in each catalog's gsonPostProcess hook after the persisted state is restored. ### Release note Fix Iceberg catalog queries and writes after FE metadata checkpoint recovery. ### Check List (For Author) - Test: Unit Test / Manual test - ExternalCatalogRuntimeStateTest - ./build.sh --fe - Docker upgrade and checkpoint restart with Iceberg REST SELECT and INSERT - Behavior changed: Yes, replayed Iceberg and HMS catalogs restore their runtime lifecycle state - Does this need documentation: No --- .../datasource/hive/HMSExternalCatalog.java | 12 +++- .../iceberg/IcebergExternalCatalog.java | 9 ++- .../ExternalCatalogRuntimeStateTest.java | 71 +++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogRuntimeStateTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 4f259e9289483d..97e102f5698607 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -53,6 +53,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -85,10 +86,10 @@ public class HMSExternalCatalog extends ExternalCatalog { //for "type" = "hms" , but is iceberg table. private IcebergMetadataOps icebergMetadataOps; - private final IcebergCatalogResourceTracker icebergResourceTracker = new IcebergCatalogResourceTracker(); + private IcebergCatalogResourceTracker icebergResourceTracker = new IcebergCatalogResourceTracker(); private volatile AbstractHiveProperties hmsProperties; - private final AtomicLong runtimeGeneration = new AtomicLong(); + private AtomicLong runtimeGeneration = new AtomicLong(); public long getRuntimeGeneration() { return runtimeGeneration.get(); @@ -132,6 +133,13 @@ public HMSExternalCatalog(long catalogId, String name, String resource, Map> T roundTrip(CatalogIf catalog, Class expectedType) { + String json = GsonUtils.GSON.toJson(catalog, CatalogIf.class); + CatalogIf restored = GsonUtils.GSON.fromJson(json, CatalogIf.class); + return expectedType.cast(restored); + } + + private static void assertTrackerCanRetainAndRelease(Object catalog, Class owner, String fieldName) + throws Exception { + Field trackerField = owner.getDeclaredField(fieldName); + trackerField.setAccessible(true); + IcebergCatalogResourceTracker tracker = (IcebergCatalogResourceTracker) trackerField.get(catalog); + Assertions.assertNotNull(tracker); + Assertions.assertDoesNotThrow(() -> { + try (IcebergCatalogResourceTracker.LoadGuard ignored = tracker.beginLoad()) { + // Closing the guard exercises the same retain/release pair used by table loading. + } + }); + } +} From 76ad1e5b718f5d73100deed5b2e99ede356fce0d Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 27 Aug 2026 14:55:04 +0800 Subject: [PATCH 22/38] [fix](fe) Make external cleanup failure atomic ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: SplitAssignment.stop rethrew an already-reported planning failure after releasing assignment resources, which could prevent split-source and sibling scan cleanup during query cancellation. DorisHiveCatalog also skipped its owned FileIO when Iceberg inherited close threw an unchecked reporter failure. Keep assignment stop no-throw and execute both catalog close legs while preserving the primary failure and suppressing later failures. ### Release note None ### Check List (For Author) - Test: Unit Test - SplitAssignmentTest, FileQueryScanNodeTest, DorisHiveCatalogTest, ExternalCatalogRuntimeStateTest - ./build.sh --fe - Behavior changed: Yes. Cleanup now completes after an existing planning or close failure. - Does this need documentation: No --- .../doris/datasource/SplitAssignment.java | 3 -- .../datasource/iceberg/DorisHiveCatalog.java | 13 +++++---- .../datasource/FileQueryScanNodeTest.java | 29 +++++++++++++++++++ .../doris/datasource/SplitAssignmentTest.java | 15 ++++++++++ .../iceberg/DorisHiveCatalogTest.java | 25 ++++++++++++++++ 5 files changed, 77 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java index d2bcd935e2294e..48abb6db871a82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java @@ -209,9 +209,6 @@ public void stop() { } }); notifyAssignment(); - if (exception != null) { - throw new RuntimeException(exception); - } } public boolean isStop() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java index 5ab72dbe140812..08296b2f88005c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.iceberg; +import com.google.common.base.Throwables; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.FileIO; @@ -41,27 +42,29 @@ public void close() throws IOException { if (!closed.compareAndSet(false, true)) { return; } - IOException closeFailure = null; + Throwable closeFailure = null; try { super.close(); - } catch (IOException e) { + } catch (Throwable e) { closeFailure = e; } try { if (ownedFileIO != null) { ownedFileIO.close(); } - } catch (RuntimeException e) { + } catch (Throwable e) { if (closeFailure != null) { closeFailure.addSuppressed(e); } else { - throw e; + closeFailure = e; } } finally { ownedFileIO = null; } if (closeFailure != null) { - throw closeFailure; + Throwables.throwIfInstanceOf(closeFailure, IOException.class); + Throwables.throwIfUnchecked(closeFailure); + throw new IOException(closeFailure); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index a29471aff20233..fecb5e466e3767 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; @@ -45,6 +46,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.lang.reflect.Method; @@ -87,6 +89,10 @@ void setTargetTable(TableIf targetTable) { this.targetTable = targetTable; } + void setSplitAssignment(SplitAssignment splitAssignment) { + this.splitAssignment = splitAssignment; + } + long selectFeSplitSize(long fallbackSize, TFileFormatType format, boolean supportsBeSplit) { return selectFeSplitSizeForBe(fallbackSize, format, supportsBeSplit); } @@ -127,6 +133,29 @@ public void testApplyMaxFileSplitNumLimitRaisesTargetSize() { Assert.assertEquals(100 * MB, target); } + @Test + public void testStopRemovesEverySourceAfterPlanningFailure() { + SplitAssignment assignment = new SplitAssignment( + null, null, null, Collections.emptyMap(), Collections.emptyList(), false); + assignment.registerSource(11L); + assignment.registerSource(12L); + assignment.setException(new UserException("planning failed")); + TestFileQueryScanNode node = new TestFileQueryScanNode(new SessionVariable()); + node.setSplitAssignment(assignment); + Env env = Mockito.mock(Env.class); + SplitSourceManager manager = Mockito.mock(SplitSourceManager.class); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getSplitSourceManager()).thenReturn(manager); + + node.stop(); + } + + Mockito.verify(manager).removeSplitSource(11L); + Mockito.verify(manager).removeSplitSource(12L); + } + @Test public void testApplyMaxFileSplitNumLimitKeepsTargetSizeWhenSmall() { SessionVariable sv = new SessionVariable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java index 3cc30ea61ac824..050749a6d11f7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/SplitAssignmentTest.java @@ -92,6 +92,21 @@ void testCloseableRegisteredAfterStopIsClosedImmediately() { Assertions.assertTrue(closed.get()); } + @Test + void testStopAfterPlanningFailureClosesEveryResourceWithoutThrowing() { + AtomicBoolean firstClosed = new AtomicBoolean(); + AtomicBoolean secondClosed = new AtomicBoolean(); + splitAssignment.addCloseable(() -> firstClosed.set(true)); + splitAssignment.addCloseable(() -> secondClosed.set(true)); + splitAssignment.setException(new UserException("planning failed")); + + Assertions.assertDoesNotThrow(splitAssignment::stop); + + Assertions.assertTrue(firstClosed.get()); + Assertions.assertTrue(secondClosed.get()); + Assertions.assertTrue(splitAssignment.isStop()); + } + // ==================== init() method tests ==================== @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java index f21956835d5532..89bc20383556cb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java @@ -17,7 +17,10 @@ package org.apache.doris.datasource.iceberg; +import org.apache.iceberg.BaseMetastoreCatalog; import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.metrics.MetricsReporter; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -38,4 +41,26 @@ void closesOwnedFileIOOnceAcrossRepeatedRetirement() throws Exception { Mockito.verify(fileIO, Mockito.times(1)).close(); } + + @Test + void closesOwnedFileIOWhenConfiguredReporterThrows() throws Exception { + DorisHiveCatalog catalog = new DorisHiveCatalog(); + FileIO fileIO = Mockito.mock(FileIO.class); + Field ownedFileIO = DorisHiveCatalog.class.getDeclaredField("ownedFileIO"); + ownedFileIO.setAccessible(true); + ownedFileIO.set(catalog, fileIO); + MetricsReporter reporter = Mockito.mock(MetricsReporter.class); + RuntimeException reporterFailure = new RuntimeException("reporter close failed"); + Mockito.doThrow(reporterFailure).when(reporter).close(); + Field metricsReporter = BaseMetastoreCatalog.class.getDeclaredField("metricsReporter"); + metricsReporter.setAccessible(true); + metricsReporter.set(catalog, reporter); + + RuntimeException actual = Assertions.assertThrows(RuntimeException.class, catalog::close); + Assertions.assertSame(reporterFailure, actual); + Mockito.verify(fileIO).close(); + + catalog.close(); + Mockito.verify(fileIO, Mockito.times(1)).close(); + } } From 316363620f2933300e0c3030e7e2deda0f57652f Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 27 Aug 2026 16:47:27 +0800 Subject: [PATCH 23/38] [fix](fe) Close external refresh lifecycle gaps ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: Hudi held its generation-owner monitor while synchronizing a remote filesystem view, so a stalled timeline sync could block catalog reset or ALTER cleanup. Iceberg background refreshes also leaked loaded table generations when mutation fencing or weighted admission prevented publication. Limit the Hudi monitor to exact lease ownership changes, and retire unpublished metadata refresh values through a dedicated callback that does not invalidate the retained Iceberg generation child caches. ### Release note None ### Check List (For Author) - Test: Unit Test - HudiFsViewCacheValueTest, MetaCacheEntryTest, IcebergExternalMetaCacheTest - ./build.sh --fe - Behavior changed: Yes. External metadata cleanup no longer waits for Hudi timeline I/O, and unpublished Iceberg refresh generations release their resources without invalidating the retained generation. - Does this need documentation: No --- .../datasource/hudi/HudiFsViewCacheValue.java | 23 +++--- .../iceberg/IcebergExternalMetaCache.java | 1 + .../metacache/AbstractExternalMetaCache.java | 3 +- .../datasource/metacache/MetaCacheEntry.java | 52 ++++++++++++- .../metacache/MetaCacheEntryDef.java | 28 +++++-- .../hudi/HudiFsViewCacheValueTest.java | 37 ++++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 73 +++++++++++++++++++ .../metacache/MetaCacheEntryTest.java | 60 ++++++++++++++- 8 files changed, 255 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java index 664f5417ccfd11..94048a4dff4d93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java @@ -38,20 +38,23 @@ public HudiFsViewCacheValue(HoodieTableFileSystemView fsView) { this.fsView = fsView; } - public synchronized Lease tryAcquire() { + public Lease tryAcquire() { Lease lease; - if (loaderReferenceAvailable) { - loaderReferenceAvailable = false; - lease = new Lease(this, fsView); - } else if (evicted) { - return null; - } else { - refCount++; - lease = new Lease(this, fsView); + synchronized (this) { + if (loaderReferenceAvailable) { + loaderReferenceAvailable = false; + lease = new Lease(this, fsView); + } else if (evicted) { + return null; + } else { + refCount++; + lease = new Lease(this, fsView); + } } try { // The cache uses expire-after-access without detached refresh. Sync every foreground generation handoff - // so a continuously hot key still observes newly completed commits. + // so a continuously hot key still observes newly completed commits. The exact lease keeps the view alive; + // do not hold the generation-owner monitor across remote timeline I/O. fsView.sync(); return lease; } catch (RuntimeException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index c8eeb51e5b7ff4..19d5d1218c62da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -115,6 +115,7 @@ public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCac MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) .withSizeEstimator(this::prepareTableForCachePublication) .withReplacementListener(this::retireTableGeneration) + .withUnpublishedValueRetirer(IcebergTableCacheValue::retire) .withRemovalListener(value -> value, (key, value) -> { if (value != null) { retireRemovedTableGeneration(key, value); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 79ee1ab78bf5e5..44fa787a5f8b94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -494,7 +494,8 @@ private MetaCacheEntry newMetaCacheEntry( cacheSpec, refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener(), - entryDef.getRemovalTokenExtractor(), entryDef.getRemovalListener()); + entryDef.getRemovalTokenExtractor(), entryDef.getRemovalListener(), + entryDef.getUnpublishedValueRetirer()); } catch (RuntimeException | Error e) { if (entryBudget != null) { entryBudget.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index debb3eaa8cf031..2659497e8ae954 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -48,6 +48,7 @@ import java.util.function.BiConsumer; import java.util.function.BiPredicate; import java.util.function.BooleanSupplier; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import javax.annotation.Nullable; @@ -93,6 +94,8 @@ public class MetaCacheEntry { private final Function removalTokenExtractor; @Nullable private final MetaCacheEntryRemovalListener removalListener; + @Nullable + private final Consumer unpublishedValueRetirer; // Removed (key, token) pairs awaiting the asynchronous removal listener; drained together with // the reservation cleanups so Caffeine's synchronous callback stays lock-free. Only the token // is queued: the removed value's reservation is released with the removal, so keeping the @@ -183,6 +186,18 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca @Nullable MetaCacheEntryReplacementListener replacementListener, @Nullable Function removalTokenExtractor, @Nullable MetaCacheEntryRemovalListener removalListener) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + sizeEstimator, entryBudget, replacementListener, removalTokenExtractor, removalListener, null); + } + + @SuppressWarnings("unchecked") + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, + @Nullable MetaCacheEntryReplacementListener replacementListener, + @Nullable Function removalTokenExtractor, + @Nullable MetaCacheEntryRemovalListener removalListener, + @Nullable Consumer unpublishedValueRetirer) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -206,9 +221,11 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca } this.removalTokenExtractor = (Function) removalTokenExtractor; this.removalListener = (MetaCacheEntryRemovalListener) removalListener; + this.unpublishedValueRetirer = unpublishedValueRetirer; this.weightBounded = this.cacheSpec.isWeightBounded(); this.generationFencedRefresh = autoRefresh - && (sizeEstimator != null || replacementListener != null || removalListener != null); + && (sizeEstimator != null || replacementListener != null + || removalListener != null || unpublishedValueRetirer != null); if (weightBounded && (sizeEstimator == null || entryBudget == null)) { throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); } @@ -1185,11 +1202,13 @@ private void submitNonWeightedRefresh( K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { try { refreshExecutor.execute(() -> { + V refreshed = null; + boolean ownershipTransferred = false; try { if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { return; } - V refreshed = loadAndTrack(key, this::applyDefaultLoader); + refreshed = loadAndTrack(key, this::applyDefaultLoader); if (refreshed == null) { return; } @@ -1199,11 +1218,15 @@ private void submitNonWeightedRefresh( } advanceKeyMutation(key); putNonWeightedValue(key, refreshed); + ownershipTransferred = true; } } catch (RuntimeException e) { LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + "retaining the previous value", name, key, e); } finally { + if (refreshed != null && !ownershipTransferred) { + retireUnpublishedValue(key, refreshed); + } endKeyMutation(key, expectedMutation); refreshesInFlight.remove(key); } @@ -1228,15 +1251,18 @@ private void submitWeightedRefresh( K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { try { refreshExecutor.execute(() -> { + V refreshed = null; + boolean ownershipTransferred = false; try { if (!isReservationCurrent(key, expectedReservationGeneration, expectedMutation)) { return; } - V refreshed = loadAndTrack(key, this::applyDefaultLoader); + refreshed = loadAndTrack(key, this::applyDefaultLoader); if (refreshed != null && isKeyMutationCurrent(key, expectedMutation)) { - admitWeightedValue( + AdmissionResult result = admitWeightedValue( key, refreshed, null, false, expectedMutation, expectedReservationGeneration, true); + ownershipTransferred = result == AdmissionResult.ADMITTED; // Admission rejection leaves the already reserved, known-good generation // in place. A larger refresh must not turn a transient quota shortage into // a forced cache miss for every subsequent reader. @@ -1245,6 +1271,9 @@ private void submitWeightedRefresh( LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + "retaining the previous value", name, key, e); } finally { + if (refreshed != null && !ownershipTransferred) { + retireUnpublishedValue(key, refreshed); + } endKeyMutation(key, expectedMutation); refreshesInFlight.remove(key); } @@ -1265,6 +1294,21 @@ private boolean isReservationCurrent( && record.published && data.asMap().get(key) != null; } + private void retireUnpublishedValue(K key, V value) { + // A refresh has no borrower to own a value that loses its generation fence or admission. + // Its cleanup must not run admitted-removal side effects against the retained generation. + if (unpublishedValueRetirer != null) { + try { + unpublishedValueRetirer.accept(value); + } catch (RuntimeException e) { + LOG.warn("Failed to retire an unpublished external metadata cache value for entry {}", + name, e); + } + return; + } + invokeRemovalListener(key, removalToken(value)); + } + // Read the config dynamically so existing cache entries follow runtime config updates. private boolean isManualMissLoadEnabled() { return weightBounded || generationFencedRefresh || Config.enable_external_meta_cache_manual_miss_load; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 25f7a21626af73..bbbcebb1d5680c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.metacache; import java.util.Objects; +import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.Nullable; @@ -109,13 +110,15 @@ public final class MetaCacheEntryDef { private final Function removalTokenExtractor; @Nullable private final MetaCacheEntryRemovalListener removalListener; + @Nullable + private final Consumer unpublishedValueRetirer; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable MetaCacheEntryReplacementListener replacementListener) { this(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, - sizeEstimator, replacementListener, null, null); + sizeEstimator, replacementListener, null, null, null); } private MetaCacheEntryDef(String name, Class keyType, Class valueType, @@ -123,7 +126,8 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable MetaCacheEntryReplacementListener replacementListener, @Nullable Function removalTokenExtractor, - @Nullable MetaCacheEntryRemovalListener removalListener) { + @Nullable MetaCacheEntryRemovalListener removalListener, + @Nullable Consumer unpublishedValueRetirer) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -146,6 +150,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.replacementListener = replacementListener; this.removalTokenExtractor = removalTokenExtractor; this.removalListener = removalListener; + this.unpublishedValueRetirer = unpublishedValueRetirer; } /** @@ -210,7 +215,7 @@ public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator es return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, Objects.requireNonNull(estimator, "estimator"), replacementListener, - removalTokenExtractor, removalListener); + removalTokenExtractor, removalListener, unpublishedValueRetirer); } /** Return a definition that synchronously retires dependencies after a value replacement. */ @@ -218,7 +223,8 @@ public MetaCacheEntryDef withReplacementListener( MetaCacheEntryReplacementListener listener) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, - Objects.requireNonNull(listener, "listener"), removalTokenExtractor, removalListener); + Objects.requireNonNull(listener, "listener"), removalTokenExtractor, removalListener, + unpublishedValueRetirer); } /** @@ -230,7 +236,14 @@ public MetaCacheEntryDef withRemovalListener( return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, replacementListener, Objects.requireNonNull(tokenExtractor, "tokenExtractor"), - Objects.requireNonNull(listener, "listener")); + Objects.requireNonNull(listener, "listener"), unpublishedValueRetirer); + } + + /** Return a definition that retires a refresh value whose ownership was never published. */ + public MetaCacheEntryDef withUnpublishedValueRetirer(Consumer retirer) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, sizeEstimator, replacementListener, + removalTokenExtractor, removalListener, Objects.requireNonNull(retirer, "retirer")); } /** @@ -303,4 +316,9 @@ public MetaCacheEntryReplacementListener getReplacementListener() { public MetaCacheEntryRemovalListener getRemovalListener() { return removalListener; } + + @Nullable + public Consumer getUnpublishedValueRetirer() { + return unpublishedValueRetirer; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java index f3a2be19541575..af2a10d3b3d0ad 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java @@ -22,6 +22,12 @@ import org.junit.Test; import org.mockito.Mockito; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + public class HudiFsViewCacheValueTest { @Test @@ -68,4 +74,35 @@ public void testLeaseSynchronizesHotCachedView() { Mockito.verify(view, Mockito.times(2)).sync(); } + + @Test + public void testEvictionDoesNotWaitForBlockedSync() throws Exception { + HoodieTableFileSystemView view = Mockito.mock(HoodieTableFileSystemView.class); + HudiFsViewCacheValue value = new HudiFsViewCacheValue(view); + CountDownLatch syncStarted = new CountDownLatch(1); + CountDownLatch allowSync = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + syncStarted.countDown(); + Assert.assertTrue(allowSync.await(3L, TimeUnit.SECONDS)); + return null; + }).when(view).sync(); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future acquisition = executor.submit(value::tryAcquire); + Assert.assertTrue(syncStarted.await(3L, TimeUnit.SECONDS)); + + Future eviction = executor.submit(value::evict); + eviction.get(3L, TimeUnit.SECONDS); + Mockito.verify(view, Mockito.never()).close(); + + allowSync.countDown(); + HudiFsViewCacheValue.Lease lease = acquisition.get(3L, TimeUnit.SECONDS); + Mockito.verify(view, Mockito.never()).close(); + lease.close(); + Mockito.verify(view).close(); + } finally { + allowSync.countDown(); + executor.shutdownNow(); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 3e61a28ae2eb8d..2a6ea1562c9465 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -73,6 +73,7 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -744,6 +745,78 @@ MetaCacheSizeEstimate prepareTableForCachePublication( } } + @Test + public void testRejectedTableRefreshKeepsCurrentGenerationProjections() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + stubTableLoadContext(catalog); + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")) + .thenReturn(tableWithMetadataLocation("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/metadata/rejected-refresh.json")); + AtomicBoolean rejectRefresh = new AtomicBoolean(false); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + if (rejectRefresh.get()) { + return MetaCacheSizeEstimate.incomplete("test_refresh_rejection"); + } + return super.prepareTableForCachePublication(nameMapping, value); + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + IcebergTableCacheValue current = new IcebergTableCacheValue( + tableWithMetadataLocation("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/metadata/current.json")); + MetaCacheEntry tables = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); + MetaCacheEntry snapshots = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + MetaCacheEntry schemas = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + tables.put(mapping, current); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, current.getRetainedIcebergTable()).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L)); + snapshots.put(snapshotKey, snapshotValue); + IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey( + mapping, current.getTableUuid().get(), 0L); + SchemaCacheValue schemaValue = new SchemaCacheValue(Collections.emptyList()); + schemas.put(schemaKey, schemaValue); + + rejectRefresh.set(true); + Method triggerRefresh = MetaCacheEntry.class.getDeclaredMethod("triggerRefreshForTest", Object.class); + triggerRefresh.setAccessible(true); + triggerRefresh.invoke(tables, mapping); + executor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, tables.peekIfPresent(mapping)); + Assert.assertSame(snapshotValue, snapshots.peekIfPresent(snapshotKey)); + Assert.assertSame(schemaValue, schemas.peekIfPresent(schemaKey)); + Assert.assertEquals(1L, tables.stats().getWeightAdmissionRejectedCount()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testIneffectiveTableEntryRevalidatesSnapshotResourcesOnEveryLookup() { // A base entry can be ineffective while the snapshot entry caches: the physical key hits, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 3e16ec48fde035..769bb6d1af8274 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1700,11 +1700,21 @@ public void testRejectedWeightedRefreshRetainsPreviousValue() throws Exception { ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( 1L, "test", "refresh-reject", OptionalLong.empty(), OptionalLong.empty()); byte[] current = new byte[1]; + AtomicReference refreshedValue = new AtomicReference<>(); + AtomicReference retiredValue = new AtomicReference<>(); + AtomicReference removedValue = new AtomicReference<>(); + MetaCacheEntryRemovalListener removalListener = + (key, value) -> removedValue.compareAndSet(null, value); MetaCacheEntry entry = new MetaCacheEntry<>( - "refresh-reject", key -> new byte[100], + "refresh-reject", key -> { + byte[] value = new byte[100]; + refreshedValue.set(value); + return value; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), refreshExecutor, true, false, - (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget, null, + value -> value, removalListener, value -> retiredValue.compareAndSet(null, value)); try { entry.put("k", current); entry.triggerRefreshForTest("k"); @@ -1713,7 +1723,53 @@ public void testRejectedWeightedRefreshRetainsPreviousValue() throws Exception { Assert.assertSame(current, entry.peekIfPresent("k")); Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + Assert.assertSame(refreshedValue.get(), retiredValue.get()); + Assert.assertNull(removedValue.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testFencedNonWeightedRefreshRetiresUnpublishedValue() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch refreshStarted = new CountDownLatch(1); + CountDownLatch allowRefresh = new CountDownLatch(1); + String current = new String("current"); + String refreshed = new String("refreshed"); + AtomicBoolean refreshedRetired = new AtomicBoolean(); + AtomicReference removedValue = new AtomicReference<>(); + MetaCacheEntryRemovalListener removalListener = + (key, value) -> removedValue.set(value); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-fenced", key -> { + refreshStarted.countDown(); + try { + Assert.assertTrue(allowRefresh.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + return refreshed; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, null, null, null, + value -> value, removalListener, + value -> refreshedRetired.set(value == refreshed)); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + Assert.assertTrue(refreshStarted.await(3L, TimeUnit.SECONDS)); + + entry.invalidateKey("k"); + allowRefresh.countDown(); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertTrue(refreshedRetired.get()); + Assert.assertNotSame(refreshed, removedValue.get()); } finally { + allowRefresh.countDown(); entry.close(); refreshExecutor.shutdownNow(); } From ae3dfbe06fd303b4fa7e7c26440cae08a234a579 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 27 Aug 2026 18:37:32 +0800 Subject: [PATCH 24/38] [fix](fe) Close remaining external lifecycle gaps Issue Number: None Related PR: #66914 Problem Summary: Hudi incremental planning acquired and synchronized an unused shared filesystem view, while its task-lifecycle tests could race the terminal callback. Iceberg resource-owning weighted cache values could lose their cleanup identity through soft collection; snapshot projection and table-property DDL could also cross catalog generations. Hive catalog initialization could allocate FileIO before a later client-pool failure and leak it. Keep resource owners strongly reachable until bounded eviction, use one captured authenticator and generation-bound writable table, make partial catalog initialization failure-atomic, and avoid the unused Hudi view acquisition. None - Test: Unit Test - HudiBatchFsViewOwnerTest, HudiScanNodeTest, HudiExternalMetaCacheTest - DorisHiveCatalogTest, IcebergMetadataOpsValidationTest, IcebergExternalMetaCacheTest - MetaCacheEntryTest - Behavior changed: Yes. Iceberg and Hudi resource generations now preserve cleanup ownership across cache eviction, initialization failure, and catalog refresh races. - Does this need documentation: No --- .../hudi/HudiExternalMetaCache.java | 3 +- .../datasource/hudi/source/HudiScanNode.java | 6 +- .../datasource/iceberg/DorisHiveCatalog.java | 20 +++++- .../iceberg/IcebergExternalMetaCache.java | 39 +++++++----- .../datasource/iceberg/IcebergUtils.java | 10 ++- .../metacache/AbstractExternalMetaCache.java | 2 +- .../datasource/metacache/MetaCacheEntry.java | 15 ++++- .../metacache/MetaCacheEntryDef.java | 25 ++++++-- .../hudi/source/HudiBatchFsViewOwnerTest.java | 18 ++++-- .../hudi/source/HudiScanNodeTest.java | 54 +++++++++++++++- .../iceberg/DorisHiveCatalogTest.java | 28 +++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 39 ++++++++++++ .../metacache/MetaCacheEntryTest.java | 62 +++++++++++++++++-- 13 files changed, 281 insertions(+), 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index 91cdcc6034a5c1..865da42299c0d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -95,7 +95,8 @@ public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheB fsViewEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_FS_VIEW, HudiFsViewCacheKey.class, HudiFsViewCacheValue.class, this::createFsView, defaultEntryCacheSpec(), false, MetaCacheEntryInvalidation.forNameMapping(HudiFsViewCacheKey::getNameMapping)) - .withRemovalListener(value -> value, this::evictFsView)); + .withRemovalListener(value -> value, this::evictFsView) + .withStrongValues()); metaClientEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_META_CLIENT, HudiMetaClientCacheKey.class, HoodieTableMetaClient.class, this::createHoodieTableMetaClient, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiMetaClientCacheKey::getNameMapping))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index df8f3ef3dd9df7..9d1cc7ebaade08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -592,11 +592,13 @@ private void getPartitionsSplits(List partitions, List spl @Override public List getSplits(int numBackends) throws UserException { ensureHmsRuntimeGeneration(); - acquireFsView(); try { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { - return getIncrementalSplits(); + List splits = getIncrementalSplits(); + ensureHmsRuntimeGeneration(); + return splits; } + acquireFsView(); List splits = Collections.synchronizedList(new ArrayList<>()); initPrunedPartitions(); hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java index 08296b2f88005c..b28e66b6f6f05f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHiveCatalog.java @@ -33,8 +33,24 @@ public class DorisHiveCatalog extends HiveCatalog { @Override public void initialize(String name, Map properties) { - super.initialize(name, properties); - ownedFileIO = extractFileIO(); + try { + super.initialize(name, properties); + ownedFileIO = extractFileIO(); + } catch (RuntimeException | Error failure) { + closePartiallyInitializedFileIO(failure); + throw failure; + } + } + + private void closePartiallyInitializedFileIO(Throwable initializationFailure) { + try { + FileIO fileIO = extractFileIO(); + if (fileIO != null) { + fileIO.close(); + } + } catch (Throwable closeFailure) { + initializationFailure.addSuppressed(closeFailure); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 19d5d1218c62da..97f51129d536d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -120,7 +120,8 @@ public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCac if (value != null) { retireRemovedTableGeneration(key, value); } - })); + }) + .withStrongValues()); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) @@ -243,13 +244,14 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { if (!optionalKey.isPresent()) { boolean isolateForQueries = tableValue.isQueryIsolationPrepared(); return executeForGeneration(tableValue, nameMapping.getCtlId(), - () -> loadSnapshotProjection( + authenticator -> loadSnapshotProjection( dorisTable, isolateForQueries ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(), tableValue.getRetainedIcebergTable(), - tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries)) - .bindCapturedAuthenticator(tableValue.getAuthenticator()); + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries, + authenticator) + .bindCapturedAuthenticator(authenticator)); } IcebergSnapshotEntryKey key = optionalKey.get(); MetaCacheEntry entry = @@ -257,14 +259,15 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { boolean isolateForQueries = tableValue.isQueryIsolationPrepared() || entry.isWeightAccounting(); Function projectionLoader = - ignored -> executeForGeneration(tableValue, nameMapping.getCtlId(), () -> { + ignored -> executeForGeneration(tableValue, nameMapping.getCtlId(), authenticator -> { Table projectionTable = isolateForQueries ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); IcebergSnapshotCacheValue value = loadSnapshotProjection( dorisTable, projectionTable, tableValue.getRetainedIcebergTable(), - tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries) - .bindCapturedAuthenticator(tableValue.getAuthenticator()); + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries, + authenticator) + .bindCapturedAuthenticator(authenticator); if (entry.isWeightAccounting()) { value.prepareForCachePublication(key); } @@ -714,7 +717,8 @@ private static boolean sharesOperationalResources( private IcebergSnapshotCacheValue loadSnapshotProjection( ExternalTable dorisTable, Table projectionTable, Table retainedTable, - String retainedCurrentSnapshotJson, boolean isolateForQueries) { + String retainedCurrentSnapshotJson, boolean isolateForQueries, + ExecutionAuthenticator authenticator) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); @@ -727,7 +731,7 @@ private IcebergSnapshotCacheValue loadSnapshotProjection( icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, projectionTable, - latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); + latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId(), authenticator); } Optional>> nameMapping = IcebergUtils.getNameMapping(projectionTable); @@ -758,15 +762,20 @@ private IcebergMetadataOps resolveMetadataOps(CatalogIf catalog) { * property ALTER resets the catalog before retiring the group, so a lookup that already * owns the old generation must not resolve authentication from the resetting catalog. */ - private T executeForGeneration( - IcebergTableCacheValue tableValue, long catalogId, Callable task) { - org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator = - tableValue.getAuthenticator(); + private T executeForGeneration(IcebergTableCacheValue tableValue, long catalogId, + Function task) { + ExecutionAuthenticator authenticator = tableValue.getAuthenticator(); if (authenticator == null) { - return executeAuthenticated(catalogId, task); + CatalogIf catalog = getCatalog(catalogId); + if (!(catalog instanceof ExternalCatalog)) { + throw new RuntimeException("Iceberg metadata cache requires an external catalog"); + } + ((ExternalCatalog) catalog).makeSureInitialized(); + authenticator = ((ExternalCatalog) catalog).getExecutionAuthenticator(); } + ExecutionAuthenticator generationAuthenticator = authenticator; try { - return authenticator.execute(task); + return generationAuthenticator.execute(() -> task.apply(generationAuthenticator)); } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 186d06e2fc0fcd..800af5332004d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -50,6 +50,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; @@ -1937,13 +1938,18 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, Table table, long snapshotId, long schemaId) throws AnalysisException { + return loadPartitionInfo(dorisTable, table, snapshotId, schemaId, + dorisTable.getCatalog().getExecutionAuthenticator()); + } + + static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, Table table, long snapshotId, + long schemaId, ExecutionAuthenticator authenticator) throws AnalysisException { if (snapshotId == IcebergUtils.UNKNOWN_SNAPSHOT_ID) { return IcebergPartitionInfo.empty(); } List icebergPartitions; try { - icebergPartitions = dorisTable.getCatalog().getExecutionAuthenticator() - .execute(() -> loadIcebergPartition(table, snapshotId)); + icebergPartitions = authenticator.execute(() -> loadIcebergPartition(table, snapshotId)); } catch (Exception e) { String errorMsg = String.format("Failed to get iceberg partition info, table: %s.%s.%s, snapshotId: %s", dorisTable.getCatalog().getName(), dorisTable.getDbName(), dorisTable.getName(), snapshotId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 44fa787a5f8b94..85433677ec278b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -495,7 +495,7 @@ private MetaCacheEntry newMetaCacheEntry( refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener(), entryDef.getRemovalTokenExtractor(), entryDef.getRemovalListener(), - entryDef.getUnpublishedValueRetirer()); + entryDef.getUnpublishedValueRetirer(), entryDef.usesSoftValues()); } catch (RuntimeException | Error e) { if (entryBudget != null) { entryBudget.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 2659497e8ae954..9f484f83233af2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -198,6 +198,19 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca @Nullable Function removalTokenExtractor, @Nullable MetaCacheEntryRemovalListener removalListener, @Nullable Consumer unpublishedValueRetirer) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + sizeEstimator, entryBudget, replacementListener, removalTokenExtractor, + removalListener, unpublishedValueRetirer, true); + } + + @SuppressWarnings("unchecked") + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, + @Nullable MetaCacheEntryReplacementListener replacementListener, + @Nullable Function removalTokenExtractor, + @Nullable MetaCacheEntryRemovalListener removalListener, + @Nullable Consumer unpublishedValueRetirer, boolean softValues) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -250,7 +263,7 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca cacheWeigher, true, null); - if (weightBounded) { + if (weightBounded && softValues) { cacheFactory.withSoftValues(); } if (weightBounded || generationFencedRefresh || removalListener != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index bbbcebb1d5680c..81970d2d5396a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -112,13 +112,14 @@ public final class MetaCacheEntryDef { private final MetaCacheEntryRemovalListener removalListener; @Nullable private final Consumer unpublishedValueRetirer; + private final boolean softValues; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable MetaCacheEntryReplacementListener replacementListener) { this(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, - sizeEstimator, replacementListener, null, null, null); + sizeEstimator, replacementListener, null, null, null, true); } private MetaCacheEntryDef(String name, Class keyType, Class valueType, @@ -127,7 +128,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable MetaCacheEntryReplacementListener replacementListener, @Nullable Function removalTokenExtractor, @Nullable MetaCacheEntryRemovalListener removalListener, - @Nullable Consumer unpublishedValueRetirer) { + @Nullable Consumer unpublishedValueRetirer, boolean softValues) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -151,6 +152,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.removalTokenExtractor = removalTokenExtractor; this.removalListener = removalListener; this.unpublishedValueRetirer = unpublishedValueRetirer; + this.softValues = softValues; } /** @@ -215,7 +217,7 @@ public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator es return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, Objects.requireNonNull(estimator, "estimator"), replacementListener, - removalTokenExtractor, removalListener, unpublishedValueRetirer); + removalTokenExtractor, removalListener, unpublishedValueRetirer, softValues); } /** Return a definition that synchronously retires dependencies after a value replacement. */ @@ -224,7 +226,7 @@ public MetaCacheEntryDef withReplacementListener( return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, Objects.requireNonNull(listener, "listener"), removalTokenExtractor, removalListener, - unpublishedValueRetirer); + unpublishedValueRetirer, softValues); } /** @@ -236,14 +238,21 @@ public MetaCacheEntryDef withRemovalListener( return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, replacementListener, Objects.requireNonNull(tokenExtractor, "tokenExtractor"), - Objects.requireNonNull(listener, "listener"), unpublishedValueRetirer); + Objects.requireNonNull(listener, "listener"), unpublishedValueRetirer, softValues); } /** Return a definition that retires a refresh value whose ownership was never published. */ public MetaCacheEntryDef withUnpublishedValueRetirer(Consumer retirer) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, replacementListener, - removalTokenExtractor, removalListener, Objects.requireNonNull(retirer, "retirer")); + removalTokenExtractor, removalListener, Objects.requireNonNull(retirer, "retirer"), softValues); + } + + /** Keep resource-owning weighted values strongly reachable until normal cache eviction. */ + public MetaCacheEntryDef withStrongValues() { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, sizeEstimator, replacementListener, + removalTokenExtractor, removalListener, unpublishedValueRetirer, false); } /** @@ -321,4 +330,8 @@ public MetaCacheEntryReplacementListener getReplacementListener() { public Consumer getUnpublishedValueRetirer() { return unpublishedValueRetirer; } + + public boolean usesSoftValues() { + return softValues; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index f0ffd4b51e70ae..ba5bbe4eec732a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -91,6 +91,7 @@ void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { CountDownLatch started = new CountDownLatch(1); CountDownLatch interrupted = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); + CountDownLatch terminal = new CountDownLatch(1); HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask(() -> { started.countDown(); while (release.getCount() > 0) { @@ -100,7 +101,10 @@ void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { interrupted.countDown(); } } - }, owner::finish); + }, () -> { + owner.finish(); + terminal.countDown(); + }); owner.track(task); ExecutorService executor = Executors.newSingleThreadExecutor(); try { @@ -113,7 +117,8 @@ void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { Assertions.assertTrue(interrupted.await(3, TimeUnit.SECONDS)); Mockito.verify(lease, Mockito.never()).close(); release.countDown(); - Mockito.verify(lease, Mockito.timeout(3000)).close(); + Assertions.assertTrue(terminal.await(3, TimeUnit.SECONDS)); + Mockito.verify(lease).close(); } finally { release.countDown(); executor.shutdownNow(); @@ -130,6 +135,7 @@ void assignmentStopInterruptsStartedTaskAndRetainsLeaseUntilTerminal() throws Ex CountDownLatch started = new CountDownLatch(1); CountDownLatch interrupted = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); + CountDownLatch terminal = new CountDownLatch(1); HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask(() -> { started.countDown(); while (release.getCount() > 0) { @@ -139,7 +145,10 @@ void assignmentStopInterruptsStartedTaskAndRetainsLeaseUntilTerminal() throws Ex interrupted.countDown(); } } - }, owner::finish); + }, () -> { + owner.finish(); + terminal.countDown(); + }); owner.track(task); ExecutorService executor = Executors.newSingleThreadExecutor(); try { @@ -151,7 +160,8 @@ void assignmentStopInterruptsStartedTaskAndRetainsLeaseUntilTerminal() throws Ex Assertions.assertTrue(interrupted.await(3, TimeUnit.SECONDS)); Mockito.verify(lease, Mockito.never()).close(); release.countDown(); - Mockito.verify(lease, Mockito.timeout(3000)).close(); + Assertions.assertTrue(terminal.await(3, TimeUnit.SECONDS)); + Mockito.verify(lease).close(); } finally { release.countDown(); executor.shutdownNow(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java index f955d8ed8fa68e..456b40514cd208 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java @@ -22,6 +22,7 @@ import org.apache.doris.datasource.ExternalScanTaskCacheKey; import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hive.source.HiveScanNode; @@ -33,6 +34,7 @@ import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.Option; import org.apache.hudi.storage.StoragePath; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -48,6 +50,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; @@ -260,12 +263,50 @@ public void testIncrementalPlanningBypassesStatementCache() throws Exception { Assertions.assertEquals(Collections.singletonList("p=1"), duplicateSplit.getPartitionValues()); } + @Test + public void testIncrementalPlanningDoesNotAcquireUnusedFsView() throws Exception { + AtomicInteger loads = new AtomicInteger(); + IncrementalRelation relation = incrementalRelation( + "10", "20", ImmutableMap.of("hoodie.datasource.query.type", "incremental"), + loads, "incremental.parquet"); + Mockito.when(relation.fallbackFullTableScan()).thenReturn(false); + HudiScanNode node = incrementalScanNode( + new StatementContext.ExternalScanTaskCache(), relation, true); + setField(node, HudiScanNode.class, "incrementalRead", true); + + List splits = node.getSplits(1); + + Assertions.assertEquals(1, loads.get()); + Assertions.assertEquals(1, splits.size()); + Assertions.assertNull(getField(node, HudiScanNode.class, "fsViewLease")); + } + + @Test + public void testEmptyMorIncrementalPlanningDoesNotAcquireUnusedFsView() throws Exception { + IncrementalRelation relation = Mockito.mock(IncrementalRelation.class); + Mockito.when(relation.fallbackFullTableScan()).thenReturn(false); + Mockito.when(relation.collectFileSlices()).thenReturn(Collections.emptyList()); + HudiScanNode node = incrementalScanNode( + new StatementContext.ExternalScanTaskCache(), relation, false); + HoodieTableMetaClient metaClient = Mockito.mock(HoodieTableMetaClient.class, Answers.RETURNS_DEEP_STUBS); + Mockito.when(metaClient.getTableConfig().getPartitionFields()).thenReturn(Option.empty()); + setField(node, HudiScanNode.class, "hudiClient", metaClient); + setField(node, HudiScanNode.class, "incrementalRead", true); + + List splits = node.getSplits(1); + + Assertions.assertTrue(splits.isEmpty()); + Assertions.assertNull(getField(node, HudiScanNode.class, "fsViewLease")); + } + private static HudiScanNode partitionScanNode( StatementContext.ExternalScanTaskCache cache, HoodieTableFileSystemView fsView, String queryInstant, boolean nativeReader, boolean runtimePrune) throws Exception { HudiScanNode node = Mockito.mock(HudiScanNode.class, Answers.CALLS_REAL_METHODS); HMSExternalTable table = Mockito.mock(HMSExternalTable.class, Answers.RETURNS_DEEP_STUBS); - Mockito.when(table.getCatalog().getId()).thenReturn(1L); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getId()).thenReturn(1L); Mockito.when(table.getId()).thenReturn(2L); Mockito.when(table.getStoragePropertiesMap()).thenReturn(Collections.emptyMap()); SessionVariable sessionVariable = new SessionVariable(); @@ -302,7 +343,9 @@ private static HudiScanNode incrementalScanNode( boolean nativeReader) throws Exception { HudiScanNode node = Mockito.mock(HudiScanNode.class, Answers.CALLS_REAL_METHODS); HMSExternalTable table = Mockito.mock(HMSExternalTable.class, Answers.RETURNS_DEEP_STUBS); - Mockito.when(table.getCatalog().getId()).thenReturn(1L); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getId()).thenReturn(1L); Mockito.when(table.getId()).thenReturn(2L); SessionVariable sessionVariable = new SessionVariable(); sessionVariable.setForceJniScanner(!nativeReader); @@ -314,6 +357,7 @@ private static HudiScanNode incrementalScanNode( setField(node, HudiScanNode.class, "isCowTable", true); setField(node, HudiScanNode.class, "incrementalRelation", relation); setField(node, HudiScanNode.class, "noLogsSplitNum", new AtomicLong()); + setField(node, HudiScanNode.class, "fsViewReleased", new AtomicBoolean()); return node; } @@ -421,4 +465,10 @@ private static void setField(Object target, Class owner, String name, Object field.setAccessible(true); field.set(target, value); } + + private static Object getField(Object target, Class owner, String name) throws Exception { + Field field = owner.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java index 89bc20383556cb..52af68b2bcd873 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHiveCatalogTest.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.iceberg; import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.hadoop.HadoopFileIO; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.metrics.MetricsReporter; import org.junit.jupiter.api.Assertions; @@ -25,9 +27,21 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; class DorisHiveCatalogTest { + public static class TrackingFileIO extends HadoopFileIO { + private static final AtomicInteger CLOSE_COUNT = new AtomicInteger(); + + @Override + public void close() { + CLOSE_COUNT.incrementAndGet(); + } + } + @Test void closesOwnedFileIOOnceAcrossRepeatedRetirement() throws Exception { DorisHiveCatalog catalog = new DorisHiveCatalog(); @@ -63,4 +77,18 @@ void closesOwnedFileIOWhenConfiguredReporterThrows() throws Exception { catalog.close(); Mockito.verify(fileIO, Mockito.times(1)).close(); } + + @Test + void closesFileIOWhenHiveClientPoolInitializationFails() { + TrackingFileIO.CLOSE_COUNT.set(0); + Map properties = new HashMap<>(); + properties.put(CatalogProperties.FILE_IO_IMPL, TrackingFileIO.class.getName()); + properties.put("client-pool-cache-keys", "invalid-key-element"); + DorisHiveCatalog catalog = new DorisHiveCatalog(); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> catalog.initialize("test", properties)); + + Assertions.assertEquals(1, TrackingFileIO.CLOSE_COUNT.get()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 2a6ea1562c9465..ece8e0a8edb242 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -502,6 +502,45 @@ protected CatalogIf getCatalog(long catalogId) { } } + @Test + public void testSnapshotPartitionLoadUsesCapturedAuthenticator() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.isValidRelatedTable()).thenReturn(true); + Table projectionTable = Mockito.mock(Table.class); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Schema schema = Mockito.mock(Schema.class); + Mockito.when(projectionTable.currentSnapshot()).thenReturn(snapshot); + Mockito.when(snapshot.snapshotId()).thenReturn(11L); + Mockito.when(projectionTable.schema()).thenReturn(schema); + Mockito.when(schema.schemaId()).thenReturn(3); + ExecutionAuthenticator capturedAuthenticator = new ExecutionAuthenticator() { + }; + Method loader = IcebergExternalMetaCache.class.getDeclaredMethod( + "loadSnapshotProjection", ExternalTable.class, Table.class, Table.class, + String.class, boolean.class, ExecutionAuthenticator.class); + loader.setAccessible(true); + try (MockedStatic icebergUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + icebergUtils.when(() -> IcebergUtils.loadPartitionInfo( + dorisTable, projectionTable, 11L, 3L, capturedAuthenticator)) + .thenReturn(IcebergPartitionInfo.empty()); + icebergUtils.when(() -> IcebergUtils.getNameMapping(projectionTable)) + .thenReturn(Optional.empty()); + icebergUtils.clearInvocations(); + + loader.invoke(cache, dorisTable, projectionTable, projectionTable, + null, false, capturedAuthenticator); + + icebergUtils.verify(() -> IcebergUtils.loadPartitionInfo( + dorisTable, projectionTable, 11L, 3L, capturedAuthenticator)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testRetainedExecutionContextAllowanceAndPlanningFence() { ExecutionAuthenticator captured = new ExecutionAuthenticator() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 769bb6d1af8274..ca3be9b9d74545 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1008,6 +1008,38 @@ public void testWeightedCacheUsesSoftValuesAndReleasesCollectedReservation() thr } } + @Test + public void testResourceOwningWeightedCacheCanUseStrongValues() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + AtomicReference retired = new AtomicReference<>(); + AtomicInteger retireCount = new AtomicInteger(); + CountDownLatch retiredLatch = new CountDownLatch(1); + ResourceOwningWeightedExternalMetaCache cache = new ResourceOwningWeightedExternalMetaCache( + refreshExecutor, manager, retired, retireCount, retiredLatch); + try { + cache.initCatalog(1L, Maps.newHashMap()); + MetaCacheEntry entry = cache.entry( + 1L, "resource", String.class, byte[].class); + byte[] value = new byte[1]; + entry.put("k", value); + Object node = extractNode(extractLoadingCache(entry)); + Object valueReference = findMethod(node.getClass(), "getValueReference").invoke(node); + + Assert.assertSame(value, findMethod(node.getClass(), "getValue").invoke(node)); + Assert.assertSame("strong-value node must not wrap V in a java.lang.ref.Reference", + value, valueReference); + + entry.invalidateAll(); + Assert.assertTrue(retiredLatch.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(value, retired.get()); + Assert.assertEquals(1, retireCount.get()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testAutomaticEvictionTelemetryKeepsExactWeightAboveWeigherLimit() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); @@ -2155,16 +2187,20 @@ private void awaitGlobalWeight(ExternalMetaCacheBudgetManager manager, long expe } private Reference extractValueReference(LoadingCache loadingCache) throws Exception { - Object boundedLocalCache = readField(loadingCache, "cache"); - Map nodes = (Map) readField(boundedLocalCache, "data"); - Assert.assertEquals(1, nodes.size()); - Object node = nodes.values().iterator().next(); + Object node = extractNode(loadingCache); Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); Object valueReference = valueReferenceMethod.invoke(node); Assert.assertTrue(valueReference instanceof Reference); return (Reference) valueReference; } + private Object extractNode(LoadingCache loadingCache) throws Exception { + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + Assert.assertEquals(1, nodes.size()); + return nodes.values().iterator().next(); + } + private Object readField(Object target, String name) throws Exception { for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { try { @@ -2203,4 +2239,22 @@ private LoadingCache extractLoadingCache(MetaCacheEntry retired, + AtomicInteger retireCount, CountDownLatch retiredLatch) { + super("strong_value_plumbing", refreshExecutor, budgetManager); + registerEntry(MetaCacheEntryDef.of( + "resource", String.class, byte[].class, key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(value.length)) + .withRemovalListener(value -> value, (key, value) -> { + retired.set(value); + retireCount.incrementAndGet(); + retiredLatch.countDown(); + }) + .withStrongValues()); + } + } } From 21f11d0168968008fbc85ccbc226be7a362d3099 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 28 Aug 2026 10:50:31 +0800 Subject: [PATCH 25/38] [fix](fe) Close Iceberg and Hudi metadata resources safely Issue Number: None Related PR: #66913 Problem Summary: Iceberg table, snapshot, schema, writable-table, and Hudi file-system-view handles could cross catalog generations or lose their close boundary during empty scans, refresh, property updates, cache replacement, and failure paths. Capture one coherent catalog generation, retain exact ownership through leases and cleanup tokens, reject mixed generations, and close resources exactly once after the final borrower releases them. Fix Iceberg and Hudi external metadata resource lifecycle leaks. - Test: Unit Test - HudiScanNodeTest - IcebergExternalMetaCacheTest - IcebergMetadataOpsValidationTest - Behavior changed: Yes. External metadata handles are retained and closed at their owning generation boundary. - Does this need documentation: No --- .../datasource/hive/HMSExternalCatalog.java | 40 ++- .../datasource/hudi/source/HudiScanNode.java | 6 +- .../iceberg/IcebergExternalCatalog.java | 57 +++- .../iceberg/IcebergExternalMetaCache.java | 296 +++++++++++++++--- .../iceberg/IcebergMetadataOps.java | 19 +- .../iceberg/IcebergSchemaCacheKey.java | 44 ++- .../iceberg/IcebergSnapshotCacheValue.java | 17 + .../iceberg/IcebergTableCacheValue.java | 96 +++++- .../datasource/iceberg/IcebergUtils.java | 64 +++- .../hudi/source/HudiScanNodeTest.java | 13 + .../iceberg/IcebergExternalMetaCacheTest.java | 241 +++++++++++++- .../IcebergMetadataOpsValidationTest.java | 20 +- 12 files changed, 820 insertions(+), 93 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 97e102f5698607..48ecde29f4c697 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -142,9 +142,24 @@ public void gsonPostProcess() throws IOException { @Override public void checkProperties() throws DdlException { - super.checkProperties(); + checkProperties(catalogProperty); + } + + @Override + public boolean validatePropertiesBeforeUpdate( + Map currentProperties, Map updatedProperties) throws DdlException { + Map candidateProperties = currentProperties == null + ? new HashMap<>() : new HashMap<>(currentProperties); + candidateProperties.putAll(updatedProperties); + checkProperties(new CatalogProperty(null, candidateProperties)); + return true; + } + + @Override + protected void checkProperties(CatalogProperty property) throws DdlException { + super.checkProperties(property); // check file.meta.cache.ttl-second parameter - String fileMetaCacheTtlSecond = catalogProperty.getOrDefault(FILE_META_CACHE_TTL_SECOND, null); + String fileMetaCacheTtlSecond = property.getOrDefault(FILE_META_CACHE_TTL_SECOND, null); if (Objects.nonNull(fileMetaCacheTtlSecond) && NumberUtils.toInt(fileMetaCacheTtlSecond, CACHE_NO_TTL) < CACHE_TTL_DISABLE_CACHE) { throw new DdlException( @@ -152,13 +167,13 @@ public void checkProperties() throws DdlException { } // check partition.cache.ttl-second parameter - String partitionCacheTtlSecond = catalogProperty.getOrDefault(PARTITION_CACHE_TTL_SECOND, null); + String partitionCacheTtlSecond = property.getOrDefault(PARTITION_CACHE_TTL_SECOND, null); if (Objects.nonNull(partitionCacheTtlSecond) && NumberUtils.toInt(partitionCacheTtlSecond, CACHE_NO_TTL) < CACHE_TTL_DISABLE_CACHE) { throw new DdlException( "The parameter " + PARTITION_CACHE_TTL_SECOND + " is wrong, value is " + partitionCacheTtlSecond); } - catalogProperty.checkMetaStoreAndStorageProperties(AbstractHiveProperties.class); + property.checkMetaStoreAndStorageProperties(AbstractHiveProperties.class); } @Override @@ -301,7 +316,9 @@ public synchronized IcebergTableLoadContext beginIcebergTableLoad() { IcebergMetadataOps ops = getIcebergMetadataOps(); return new IcebergTableLoadContext(ops, threadPoolWithPreAuth, executionAuthenticator, catalogProperty.getMetastoreProperties(), - new HashMap<>(catalogProperty.getStoragePropertiesMap()), icebergResourceTracker.beginLoad()); + new HashMap<>(catalogProperty.getStoragePropertiesMap()), + catalogProperty.getEnableMappingVarbinary(), catalogProperty.getEnableMappingTimestampTz(), + icebergResourceTracker.beginLoad()); } @Override @@ -324,17 +341,22 @@ public final class IcebergTableLoadContext implements AutoCloseable { private final ExecutionAuthenticator authenticator; private final MetastoreProperties metastoreProperties; private final Map storageProperties; + private final boolean enableMappingVarbinary; + private final boolean enableMappingTimestampTz; private final IcebergCatalogResourceTracker.LoadGuard guard; private IcebergTableLoadContext(IcebergMetadataOps ops, ThreadPoolExecutor executor, ExecutionAuthenticator authenticator, MetastoreProperties metastoreProperties, Map storageProperties, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz, IcebergCatalogResourceTracker.LoadGuard guard) { this.ops = ops; this.executor = executor; this.authenticator = authenticator; this.metastoreProperties = metastoreProperties; this.storageProperties = storageProperties; + this.enableMappingVarbinary = enableMappingVarbinary; + this.enableMappingTimestampTz = enableMappingTimestampTz; this.guard = guard; } @@ -358,6 +380,14 @@ public Map getStorageProperties() { return storageProperties; } + public boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + public boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + public Table loadTable(String dbName, String tableName) throws Exception { return authenticator.execute(() -> ops.loadTable(dbName, tableName)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 9d1cc7ebaade08..2a2f439b30a9f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -598,9 +598,13 @@ public List getSplits(int numBackends) throws UserException { ensureHmsRuntimeGeneration(); return splits; } + initPrunedPartitions(); + if (prunedPartitions.isEmpty()) { + ensureHmsRuntimeGeneration(); + return Collections.emptyList(); + } acquireFsView(); List splits = Collections.synchronizedList(new ArrayList<>()); - initPrunedPartitions(); hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> { getPartitionsSplits(prunedPartitions, splits); return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index da0090ca10e2cb..f79612e74c2775 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -25,6 +25,7 @@ import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalObjectLog; @@ -79,6 +80,14 @@ public IcebergExternalCatalog(long catalogId, String name, String comment) { super(catalogId, name, InitCatalogLog.Type.ICEBERG, comment); } + @Override + public synchronized void modifyCatalogProps(Map props) { + // Keep property publication and runtime reset under the same monitor used by + // beginTableLoad(), so a load context cannot splice old runtime objects with new mapping + // options while ALTER CATALOG is between those two operations. + super.modifyCatalogProps(props); + } + @Override public void gsonPostProcess() throws IOException { super.gsonPostProcess(); @@ -100,21 +109,36 @@ protected void initCatalog() { @Override public void checkProperties() throws DdlException { - super.checkProperties(); - CacheSpec.checkBooleanProperty(catalogProperty.getOrDefault(ICEBERG_TABLE_CACHE_ENABLE, null), + checkProperties(catalogProperty); + } + + @Override + public boolean validatePropertiesBeforeUpdate( + Map currentProperties, Map updatedProperties) throws DdlException { + Map candidateProperties = currentProperties == null + ? new HashMap<>() : new HashMap<>(currentProperties); + candidateProperties.putAll(updatedProperties); + checkProperties(new CatalogProperty(null, candidateProperties)); + return true; + } + + @Override + protected void checkProperties(CatalogProperty property) throws DdlException { + super.checkProperties(property); + CacheSpec.checkBooleanProperty(property.getOrDefault(ICEBERG_TABLE_CACHE_ENABLE, null), ICEBERG_TABLE_CACHE_ENABLE); - CacheSpec.checkLongProperty(catalogProperty.getOrDefault(ICEBERG_TABLE_CACHE_TTL_SECOND, null), + CacheSpec.checkLongProperty(property.getOrDefault(ICEBERG_TABLE_CACHE_TTL_SECOND, null), -1L, ICEBERG_TABLE_CACHE_TTL_SECOND); - CacheSpec.checkLongProperty(catalogProperty.getOrDefault(ICEBERG_TABLE_CACHE_CAPACITY, null), + CacheSpec.checkLongProperty(property.getOrDefault(ICEBERG_TABLE_CACHE_CAPACITY, null), 0L, ICEBERG_TABLE_CACHE_CAPACITY); - CacheSpec.checkBooleanProperty(catalogProperty.getOrDefault(ICEBERG_MANIFEST_CACHE_ENABLE, null), + CacheSpec.checkBooleanProperty(property.getOrDefault(ICEBERG_MANIFEST_CACHE_ENABLE, null), ICEBERG_MANIFEST_CACHE_ENABLE); - CacheSpec.checkLongProperty(catalogProperty.getOrDefault(ICEBERG_MANIFEST_CACHE_TTL_SECOND, null), + CacheSpec.checkLongProperty(property.getOrDefault(ICEBERG_MANIFEST_CACHE_TTL_SECOND, null), -1L, ICEBERG_MANIFEST_CACHE_TTL_SECOND); - CacheSpec.checkLongProperty(catalogProperty.getOrDefault(ICEBERG_MANIFEST_CACHE_CAPACITY, null), + CacheSpec.checkLongProperty(property.getOrDefault(ICEBERG_MANIFEST_CACHE_CAPACITY, null), 0L, ICEBERG_MANIFEST_CACHE_CAPACITY); - catalogProperty.checkMetaStoreAndStorageProperties(AbstractIcebergProperties.class); + property.checkMetaStoreAndStorageProperties(AbstractIcebergProperties.class); } @Override @@ -170,7 +194,9 @@ synchronized TableLoadContext beginTableLoad() { makeSureInitialized(); return new TableLoadContext((IcebergMetadataOps) metadataOps, executionAuthenticator, icebergCatalogType, catalogProperty.getMetastoreProperties(), - new HashMap<>(catalogProperty.getStoragePropertiesMap()), resourceTracker.beginLoad()); + new HashMap<>(catalogProperty.getStoragePropertiesMap()), + catalogProperty.getEnableMappingVarbinary(), catalogProperty.getEnableMappingTimestampTz(), + resourceTracker.beginLoad()); } public String getIcebergCatalogType() { @@ -239,17 +265,22 @@ final class TableLoadContext implements AutoCloseable { private final String catalogType; private final MetastoreProperties metastoreProperties; private final Map storageProperties; + private final boolean enableMappingVarbinary; + private final boolean enableMappingTimestampTz; private final IcebergCatalogResourceTracker.LoadGuard guard; private TableLoadContext(IcebergMetadataOps ops, ExecutionAuthenticator authenticator, String catalogType, MetastoreProperties metastoreProperties, Map storageProperties, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz, IcebergCatalogResourceTracker.LoadGuard guard) { this.ops = ops; this.authenticator = authenticator; this.catalogType = catalogType; this.metastoreProperties = metastoreProperties; this.storageProperties = storageProperties; + this.enableMappingVarbinary = enableMappingVarbinary; + this.enableMappingTimestampTz = enableMappingTimestampTz; this.guard = guard; } @@ -277,6 +308,14 @@ Map getStorageProperties() { return storageProperties; } + boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + IcebergCatalogResourceTracker.ResourceLease promote() { return guard.promote(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 97f51129d536d0..302dba7c84cc80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -60,6 +60,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.Nullable; @@ -206,6 +207,68 @@ public Table getWritableIcebergTable(ExternalTable dorisTable, @Nullable Iceberg return table; } + WritableTableLease acquireWritableIcebergTable( + ExternalTable dorisTable, @Nullable IcebergMetadataOps expectedOps) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + + " when loading a writable Iceberg table"); + } + if (catalog instanceof IcebergExternalCatalog) { + IcebergExternalCatalog icebergCatalog = (IcebergExternalCatalog) catalog; + try (IcebergExternalCatalog.TableLoadContext context = icebergCatalog.beginTableLoad()) { + IcebergMetadataOps ops = context.getOps(); + if (expectedOps != null && ops != expectedOps) { + throw catalogGenerationMoved(nameMapping); + } + Table table; + try { + table = context.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + try (TableResourceOwner owner = new TableResourceOwner( + tableCleanup(context.getCatalogType(), ops, table))) { + ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, + context.isEnableMappingVarbinary(), context.isEnableMappingTimestampTz()); + owner.add(context.promote()::close); + WritableTableLease lease = new WritableTableLease( + table, context.getAuthenticator(), owner.cleanup()); + owner.transfer(); + return lease; + } + } + } + if (catalog instanceof HMSExternalCatalog) { + HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; + try (HMSExternalCatalog.IcebergTableLoadContext context = hmsCatalog.beginIcebergTableLoad()) { + IcebergMetadataOps ops = context.getOps(); + if (expectedOps != null && ops != expectedOps) { + throw catalogGenerationMoved(nameMapping); + } + boolean enableMappingVarbinary = context.isEnableMappingVarbinary(); + boolean enableMappingTimestampTz = context.isEnableMappingTimestampTz(); + Table table; + try { + table = context.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + try (TableResourceOwner owner = new TableResourceOwner(() -> { })) { + ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, + enableMappingVarbinary, enableMappingTimestampTz); + owner.add(context.promote()::close); + WritableTableLease lease = new WritableTableLease( + table, context.getAuthenticator(), owner.cleanup()); + owner.transfer(); + return lease; + } + } + } + throw new RuntimeException("Only support 'hms' and 'iceberg' type for iceberg table"); + } + Table getQueryScopedIcebergTable(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); IcebergTableCacheValue tableValue = statementValue(nameMapping); @@ -250,8 +313,11 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { : tableValue.getIcebergTable(), tableValue.getRetainedIcebergTable(), tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries, - authenticator) - .bindCapturedAuthenticator(authenticator)); + authenticator, tableValue.isEnableMappingVarbinary(), + tableValue.isEnableMappingTimestampTz()) + .bindCapturedAuthenticator(authenticator) + .bindSchemaMappingOptions(tableValue.isEnableMappingVarbinary(), + tableValue.isEnableMappingTimestampTz())); } IcebergSnapshotEntryKey key = optionalKey.get(); MetaCacheEntry entry = @@ -266,8 +332,11 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { dorisTable, projectionTable, tableValue.getRetainedIcebergTable(), tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries, - authenticator) - .bindCapturedAuthenticator(authenticator); + authenticator, tableValue.isEnableMappingVarbinary(), + tableValue.isEnableMappingTimestampTz()) + .bindCapturedAuthenticator(authenticator) + .bindSchemaMappingOptions(tableValue.isEnableMappingVarbinary(), + tableValue.isEnableMappingTimestampTz()); if (entry.isWeightAccounting()) { value.prepareForCachePublication(key); } @@ -317,21 +386,41 @@ public View getIcebergView(ExternalTable dorisTable) { public IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId) { IcebergTableCacheValue tableValue = statementValue(nameMapping); - return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable()); + return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable(), + tableValue.getAuthenticator(), tableValue.isEnableMappingVarbinary(), + tableValue.isEnableMappingTimestampTz()); } IcebergSchemaCacheValue getIcebergSchemaCacheValue( NameMapping nameMapping, long schemaId, Table retainedTable) { + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); + if (!(catalog instanceof ExternalCatalog)) { + return getIcebergSchemaCacheValue(nameMapping, schemaId, retainedTable, + new ExecutionAuthenticator() { }, false, false); + } + ExecutionAuthenticator authenticator = requireExecutionAuthenticator(catalog); + ExternalCatalog externalCatalog = (ExternalCatalog) catalog; + return getIcebergSchemaCacheValue(nameMapping, schemaId, retainedTable, authenticator, + externalCatalog.getEnableMappingVarbinary(), externalCatalog.getEnableMappingTimestampTz()); + } + + IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId, Table retainedTable, + ExecutionAuthenticator authenticator, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { Optional generation = IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); if (!generation.isPresent()) { return (IcebergSchemaCacheValue) loadSchemaCacheValue( - new IcebergSchemaCacheKey(nameMapping, schemaId), retainedTable); + new IcebergSchemaCacheKey(nameMapping, "", schemaId, + retainedTable.spec().specId(), + enableMappingVarbinary, enableMappingTimestampTz), + retainedTable, authenticator); } IcebergSchemaCacheKey key = new IcebergSchemaCacheKey( - nameMapping, generation.get().getTableUuid(), schemaId); + nameMapping, generation.get().getTableUuid(), schemaId, retainedTable.spec().specId(), + enableMappingVarbinary, enableMappingTimestampTz); MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); SchemaCacheValue schemaCacheValue = entry - .get(key, ignored -> loadSchemaCacheValue(key, retainedTable)); + .get(key, ignored -> loadSchemaCacheValue(key, retainedTable, authenticator)); MetaCacheEntry tables = tableEntry.getIfInitialized(nameMapping.getCtlId()); if (tables == null) { @@ -396,38 +485,52 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { IcebergExternalCatalog icebergCatalog = (IcebergExternalCatalog) catalog; try (IcebergExternalCatalog.TableLoadContext context = icebergCatalog.beginTableLoad()) { IcebergMetadataOps ops = context.getOps(); + boolean enableMappingVarbinary = context.isEnableMappingVarbinary(); + boolean enableMappingTimestampTz = context.isEnableMappingTimestampTz(); Table table; try { table = context.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } - ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping); - Runnable tableCleanup = tableCleanup(context.getCatalogType(), ops, table); - IcebergCatalogResourceTracker.ResourceLease catalogLease = context.promote(); - Runnable cleanup = () -> { - try { - tableCleanup.run(); - } finally { - catalogLease.close(); + try (TableResourceOwner owner = new TableResourceOwner( + tableCleanup(context.getCatalogType(), ops, table))) { + ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, + enableMappingVarbinary, enableMappingTimestampTz); + try (TableResourceOwner catalogOwner = new TableResourceOwner(context.promote()::close)) { + IcebergTableCacheValue value = execute(context.getAuthenticator(), () -> createLoadedTableValue( + nameMapping, table, ops.getThreadPoolWithPreAuth(), context.getAuthenticator(), + enableMappingVarbinary, enableMappingTimestampTz, + owner, catalogOwner.cleanup())); + owner.transfer(); + catalogOwner.transfer(); + return value; } - }; - return execute(context.getAuthenticator(), () -> createLoadedTableValue( - nameMapping, table, ops.getThreadPoolWithPreAuth(), context.getAuthenticator(), cleanup)); + } } } if (catalog instanceof HMSExternalCatalog) { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; try (HMSExternalCatalog.IcebergTableLoadContext context = hmsCatalog.beginIcebergTableLoad()) { + boolean enableMappingVarbinary = context.isEnableMappingVarbinary(); + boolean enableMappingTimestampTz = context.isEnableMappingTimestampTz(); Table table; try { table = context.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } - IcebergCatalogResourceTracker.ResourceLease catalogLease = context.promote(); - return execute(context.getAuthenticator(), () -> createLoadedTableValue( - nameMapping, table, context.getExecutor(), context.getAuthenticator(), catalogLease::close)); + try (TableResourceOwner owner = new TableResourceOwner(() -> { })) { + try (TableResourceOwner catalogOwner = new TableResourceOwner(context.promote()::close)) { + IcebergTableCacheValue value = execute(context.getAuthenticator(), () -> createLoadedTableValue( + nameMapping, table, context.getExecutor(), context.getAuthenticator(), + enableMappingVarbinary, enableMappingTimestampTz, + owner, catalogOwner.cleanup())); + owner.transfer(); + catalogOwner.transfer(); + return value; + } + } } } @@ -435,12 +538,31 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { } private IcebergTableCacheValue createLoadedTableValue(NameMapping nameMapping, Table table, - ThreadPoolExecutor planningExecutor, ExecutionAuthenticator authenticator, Runnable cleanup) { - IcebergTableCacheValue loaded = new IcebergTableCacheValue(table, planningExecutor, () -> null, cleanup); + ThreadPoolExecutor planningExecutor, ExecutionAuthenticator authenticator, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz, + TableResourceOwner tableOwner, Runnable cleanup) { + IcebergTableCacheValue loaded = new IcebergTableCacheValue( + table, planningExecutor, () -> null, tableOwner.cleanup(), cleanup); try { loaded.bindAuthenticator(authenticator); + loaded.bindSchemaMappingOptions(enableMappingVarbinary, enableMappingTimestampTz); MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); + IcebergTableCacheValue currentValue = currentEntry == null + ? null : currentEntry.peekIfPresent(nameMapping); + if (currentValue != null && currentValue.sharesFileIoIdentity(table)) { + // Adopt the current generation's cleanup token before cache publication. If this + // load is rejected, its reference is released without closing the shared FileIO; + // if it replaces current, the last retired generation closes the FileIO once. + if (!loaded.shareTableCleanupWith(currentValue)) { + // The exact shared FileIO was closed after peek but before ownership could be + // retained. Suppress the duplicate close and reject this unusable generation. + loaded.abandonTableCleanup(); + tableOwner.transfer(); + throw catalogGenerationMoved(nameMapping); + } + tableOwner.transfer(); + } if (currentEntry != null && currentEntry.isWeightAccounting()) { prepareTableForCachePublication(nameMapping, loaded); } @@ -563,12 +685,22 @@ private T execute(ExecutionAuthenticator authenticator, Callable task) { */ private void ensureCatalogGenerationStable(CatalogIf catalog, IcebergMetadataOps ops, ExecutionAuthenticator authenticator, NameMapping nameMapping) { + ExternalCatalog externalCatalog = (ExternalCatalog) catalog; + ensureCatalogGenerationStable(catalog, ops, authenticator, nameMapping, + externalCatalog.getEnableMappingVarbinary(), externalCatalog.getEnableMappingTimestampTz()); + } + + private void ensureCatalogGenerationStable(CatalogIf catalog, IcebergMetadataOps ops, + ExecutionAuthenticator authenticator, NameMapping nameMapping, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { boolean stable; try { // Read without re-initializing: a catalog that was reset mid-flight must count as // unstable here, not get quietly reinitialized by the validation itself. stable = resolveMetadataOps(catalog) == ops - && ((ExternalCatalog) catalog).getExecutionAuthenticator() == authenticator; + && ((ExternalCatalog) catalog).getExecutionAuthenticator() == authenticator + && ((ExternalCatalog) catalog).getEnableMappingVarbinary() == enableMappingVarbinary + && ((ExternalCatalog) catalog).getEnableMappingTimestampTz() == enableMappingTimestampTz; } catch (RuntimeException e) { stable = false; } @@ -627,13 +759,18 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } - private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table retainedTable) { + private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table retainedTable, + ExecutionAuthenticator authenticator) { ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); dorisTable.setUpdateTime(System.currentTimeMillis()); boolean isView = dorisTable instanceof IcebergExternalTable && ((IcebergExternalTable) dorisTable).isView(); - SchemaCacheValue value = IcebergUtils.loadSchemaCacheValue( - dorisTable, key.getSchemaId(), isView, retainedTable).orElseThrow(() -> + SchemaCacheValue value = (isView + ? IcebergUtils.loadSchemaCacheValue(dorisTable, key.getSchemaId(), true, retainedTable) + : Optional.of(IcebergUtils.buildTableSchemaCacheValue( + dorisTable, key.getSchemaId(), retainedTable, authenticator, + key.isEnableMappingVarbinary(), key.isEnableMappingTimestampTz()))) + .orElseThrow(() -> new CacheException("failed to load iceberg schema cache value for: %s.%s.%s, schemaId: %s", null, key.getNameMapping().getCtlId(), key.getNameMapping().getLocalDbName(), key.getNameMapping().getLocalTblName(), key.getSchemaId())); @@ -650,7 +787,9 @@ private void retireTableGeneration(NameMapping nameMapping, } try { if (previousValue.isSameOperationalGeneration(currentValue)) { - return; + if (previousValue.sharesFileIoIdentity(currentValue.getRetainedIcebergTable())) { + return; + } } MetaCacheEntry snapshots = snapshotEntry.getIfInitialized(nameMapping.getCtlId()); @@ -666,7 +805,10 @@ private void retireTableGeneration(NameMapping nameMapping, schemaEntry.getIfInitialized(nameMapping.getCtlId()); if (schemas != null) { schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) - && !key.getTableUuid().equals(currentUuid)); + && (!key.getTableUuid().equals(currentUuid) + || key.isEnableMappingVarbinary() != currentValue.isEnableMappingVarbinary() + || key.isEnableMappingTimestampTz() + != currentValue.isEnableMappingTimestampTz())); } } finally { // Caffeine REPLACED notifications intentionally do not run the removal listener because @@ -698,10 +840,6 @@ private static boolean sharesOperationalResources( if (projection == null) { return false; } - if (projection.getRetainedIcebergTable().map(table -> table == currentValue.getRetainedIcebergTable()) - .orElse(false)) { - return true; - } Optional
retainedTable = projection.getRetainedIcebergTable(); // Count-mode projections do not retain a table handle; nothing to rebind (and nothing is // planned through a frozen generation, so a stale captured context is inert). @@ -711,14 +849,95 @@ private static boolean sharesOperationalResources( // The captured execution context must match as well: after an auth-only ALTER the frozen // handle is operationally equivalent but unplannable under the new context, and serving // it would make every retried statement hit the same rejected projection until expiry. - return currentValue.sharesOperationalResources(retainedTable.get()) - && currentValue.getAuthenticator() == projection.getCapturedAuthenticator(); + return currentValue.sharesFileIoIdentity(retainedTable.get()) + && currentValue.getAuthenticator() == projection.getCapturedAuthenticator() + && currentValue.isEnableMappingVarbinary() == projection.isEnableMappingVarbinary() + && currentValue.isEnableMappingTimestampTz() == projection.isEnableMappingTimestampTz(); + } + + static final class WritableTableLease implements AutoCloseable { + private final Table table; + private final ExecutionAuthenticator authenticator; + private final Runnable cleanup; + private final AtomicBoolean closed = new AtomicBoolean(); + + private WritableTableLease(Table table, ExecutionAuthenticator authenticator, Runnable cleanup) { + this.table = table; + this.authenticator = authenticator; + this.cleanup = cleanup; + } + + Table getTable() { + return table; + } + + ExecutionAuthenticator getAuthenticator() { + return authenticator; + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + cleanup.run(); + } + } + } + + private static final class TableResourceOwner implements AutoCloseable { + private final List cleanups = new ArrayList<>(); + private final AtomicBoolean released = new AtomicBoolean(); + private boolean transferred; + + private TableResourceOwner(Runnable cleanup) { + cleanups.add(cleanup); + } + + private void add(Runnable cleanup) { + cleanups.add(cleanup); + } + + private Runnable cleanup() { + return this::release; + } + + private void transfer() { + transferred = true; + } + + private void release() { + if (!released.compareAndSet(false, true)) { + return; + } + RuntimeException failure = null; + for (Runnable cleanup : cleanups) { + try { + cleanup.run(); + } catch (RuntimeException e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; + } + } + + @Override + public void close() { + if (!transferred) { + release(); + } + } } private IcebergSnapshotCacheValue loadSnapshotProjection( ExternalTable dorisTable, Table projectionTable, Table retainedTable, String retainedCurrentSnapshotJson, boolean isolateForQueries, - ExecutionAuthenticator authenticator) { + ExecutionAuthenticator authenticator, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); @@ -731,7 +950,8 @@ private IcebergSnapshotCacheValue loadSnapshotProjection( icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, projectionTable, - latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId(), authenticator); + latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId(), authenticator, + enableMappingVarbinary, enableMappingTimestampTz); } Optional>> nameMapping = IcebergUtils.getNameMapping(projectionTable); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 6cbb51ab62fdbd..9ed866d57ca27b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -891,14 +891,17 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd @Override public void updateTableProperties(ExternalTable dorisTable, Map properties, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - UpdateProperties updateProperties = icebergTable.updateProperties(); - properties.forEach(updateProperties::set); - try { - executionAuthenticator.execute(updateProperties::commit); - } catch (Exception e) { - throw new UserException("Failed to update properties for table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + UpdateProperties updateProperties = icebergTable.updateProperties(); + properties.forEach(updateProperties::set); + try { + lease.getAuthenticator().execute(updateProperties::commit); + } catch (Exception e) { + throw new UserException("Failed to update properties for table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java index 9916d0afbb17e1..6bbd38617e20cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java @@ -27,15 +27,36 @@ public class IcebergSchemaCacheKey extends SchemaCacheKey { private final String tableUuid; private final long schemaId; + private final int partitionSpecId; + private final boolean enableMappingVarbinary; + private final boolean enableMappingTimestampTz; public IcebergSchemaCacheKey(NameMapping nameMapping, long schemaId) { - this(nameMapping, "", schemaId); + this(nameMapping, "", schemaId, -1, false, false); } public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId) { + this(nameMapping, tableUuid, schemaId, -1, false, false); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId, int partitionSpecId) { + this(nameMapping, tableUuid, schemaId, partitionSpecId, false, false); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { + this(nameMapping, tableUuid, schemaId, -1, + enableMappingVarbinary, enableMappingTimestampTz); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId, int partitionSpecId, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { super(nameMapping); this.tableUuid = java.util.Objects.requireNonNull(tableUuid, "tableUuid can not be null"); this.schemaId = schemaId; + this.partitionSpecId = partitionSpecId; + this.enableMappingVarbinary = enableMappingVarbinary; + this.enableMappingTimestampTz = enableMappingTimestampTz; } public Optional getTableUuid() { @@ -46,6 +67,18 @@ public long getSchemaId() { return schemaId; } + public int getPartitionSpecId() { + return partitionSpecId; + } + + public boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + public boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -58,11 +91,16 @@ public boolean equals(Object o) { return false; } IcebergSchemaCacheKey that = (IcebergSchemaCacheKey) o; - return schemaId == that.schemaId && tableUuid.equals(that.tableUuid); + return schemaId == that.schemaId + && partitionSpecId == that.partitionSpecId + && enableMappingVarbinary == that.enableMappingVarbinary + && enableMappingTimestampTz == that.enableMappingTimestampTz + && tableUuid.equals(that.tableUuid); } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), tableUuid, schemaId); + return Objects.hashCode(super.hashCode(), tableUuid, schemaId, partitionSpecId, + enableMappingVarbinary, enableMappingTimestampTz); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index d1a4e1be59dd4b..66b1b29c7371e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -63,6 +63,8 @@ public class IcebergSnapshotCacheValue { */ @Nullable private transient volatile ExecutionAuthenticator capturedAuthenticator; + private transient volatile boolean enableMappingVarbinary; + private transient volatile boolean enableMappingTimestampTz; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { this(partitionInfo, snapshot, Optional.empty(), Optional.empty(), null, false); @@ -136,11 +138,26 @@ public IcebergSnapshotCacheValue bindCapturedAuthenticator(@Nullable ExecutionAu return this; } + public IcebergSnapshotCacheValue bindSchemaMappingOptions( + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { + this.enableMappingVarbinary = enableMappingVarbinary; + this.enableMappingTimestampTz = enableMappingTimestampTz; + return this; + } + @Nullable public ExecutionAuthenticator getCapturedAuthenticator() { return capturedAuthenticator; } + public boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + public boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + /** * A relation pinned to this projection plans and scans the retained frozen table. That work * must run on the execution context captured with the projection's table generation: after a diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index 4245d02c94ec5a..3129693ba0f426 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -40,6 +40,7 @@ public class IcebergTableCacheValue { private volatile Table icebergTable; @Nullable private final ThreadPoolExecutor planningExecutor; + private TableCleanupOwner tableCleanupOwner; private final Runnable cleanup; private final AtomicInteger references; private final AtomicBoolean cacheReferenceReleased = new AtomicBoolean(); @@ -48,6 +49,8 @@ public class IcebergTableCacheValue { // counterpart for the concurrent catalog-reset rationale. @Nullable private volatile org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator; + private volatile boolean enableMappingVarbinary; + private volatile boolean enableMappingTimestampTz; private String retainedCurrentSnapshotJson; private volatile boolean queryIsolationPrepared; private long retainedTablePayloadBytes; @@ -69,8 +72,22 @@ public IcebergTableCacheValue(Table icebergTable) { private IcebergTableCacheValue(Table icebergTable, @Nullable ThreadPoolExecutor planningExecutor, Supplier ignoredSnapshotSupplier, Runnable cleanup, boolean loading) { + this(icebergTable, planningExecutor, ignoredSnapshotSupplier, () -> { }, cleanup, loading); + } + + IcebergTableCacheValue(Table icebergTable, @Nullable ThreadPoolExecutor planningExecutor, + Supplier ignoredSnapshotSupplier, + Runnable tableCleanup, Runnable cleanup) { + this(icebergTable, planningExecutor, ignoredSnapshotSupplier, tableCleanup, cleanup, true); + } + + private IcebergTableCacheValue(Table icebergTable, @Nullable ThreadPoolExecutor planningExecutor, + Supplier ignoredSnapshotSupplier, + Runnable tableCleanup, Runnable cleanup, boolean loading) { this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); this.planningExecutor = planningExecutor; + this.tableCleanupOwner = new TableCleanupOwner( + Objects.requireNonNull(tableCleanup, "table cleanup")); this.cleanup = Objects.requireNonNull(cleanup, "cleanup"); Objects.requireNonNull(ignoredSnapshotSupplier, "snapshot supplier"); this.references = new AtomicInteger(loading ? 2 : 1); @@ -108,7 +125,11 @@ void retire() { private void release() { int remaining = references.decrementAndGet(); if (remaining == 0) { - cleanup.run(); + try { + tableCleanupOwner.release(); + } finally { + cleanup.run(); + } } else if (remaining < 0) { throw new IllegalStateException("Iceberg table cache value released too many times"); } @@ -119,11 +140,76 @@ void bindAuthenticator( this.authenticator = authenticator; } + void bindSchemaMappingOptions(boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { + this.enableMappingVarbinary = enableMappingVarbinary; + this.enableMappingTimestampTz = enableMappingTimestampTz; + } + + boolean shareTableCleanupWith(IcebergTableCacheValue currentValue) { + TableCleanupOwner shared = currentValue.tableCleanupOwner; + if (shared.tryRetain()) { + tableCleanupOwner.abandon(); + tableCleanupOwner = shared; + return true; + } + return false; + } + + void abandonTableCleanup() { + tableCleanupOwner.abandon(); + tableCleanupOwner = new TableCleanupOwner(() -> { }); + } + @Nullable org.apache.doris.common.security.authentication.ExecutionAuthenticator getAuthenticator() { return authenticator; } + boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + + /** One exact closeable shared by every cache generation that publishes the same FileIO. */ + private static final class TableCleanupOwner { + private final Runnable cleanup; + private final AtomicInteger owners = new AtomicInteger(1); + + private TableCleanupOwner(Runnable cleanup) { + this.cleanup = cleanup; + } + + private boolean tryRetain() { + int current = owners.get(); + while (current != 0) { + if (owners.compareAndSet(current, current + 1)) { + return true; + } + current = owners.get(); + } + return false; + } + + private void abandon() { + int remaining = owners.decrementAndGet(); + if (remaining != 0) { + throw new IllegalStateException("unpublished Iceberg table cleanup has multiple owners"); + } + } + + private void release() { + int remaining = owners.decrementAndGet(); + if (remaining == 0) { + cleanup.run(); + } else if (remaining < 0) { + throw new IllegalStateException("Iceberg table cleanup released too many times"); + } + } + } + public Table getIcebergTable() { Table retainedTable = icebergTable; return queryIsolationPrepared || IcebergSnapshotCacheValue.isNonGrowingGeneration(retainedTable) @@ -228,13 +314,19 @@ boolean isSameOperationalGeneration(IcebergTableCacheValue other) { // and projections frozen on the old context would fail the planning fence forever // instead of being rebuilt. return isSamePhysicalGeneration(other) && sharesOperationalResources(other.icebergTable) - && authenticator == other.authenticator; + && authenticator == other.authenticator + && enableMappingVarbinary == other.enableMappingVarbinary + && enableMappingTimestampTz == other.enableMappingTimestampTz; } boolean sharesOperationalResources(Table table) { return sharesOperationalResources(icebergTable, table); } + boolean sharesFileIoIdentity(Table table) { + return table != null && icebergTable.io() == table.io(); + } + /** True when both tables read and write through the same FileIO, encryption and locations. */ static boolean sharesOperationalResources(Table left, Table right) { if (left == right) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 800af5332004d3..de9c8ce7b97371 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1155,6 +1155,11 @@ public static Table getWritableIcebergTable(ExternalTable dorisTable, IcebergMet return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable, expectedOps); } + static IcebergExternalMetaCache.WritableTableLease acquireWritableIcebergTable( + ExternalTable dorisTable, IcebergMetadataOps expectedOps) { + return icebergExternalMetaCache(dorisTable).acquireWritableIcebergTable(dorisTable, expectedOps); + } + public static ThreadPoolExecutor getIcebergTableExecutor(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTableExecutor(dorisTable); } @@ -1379,8 +1384,17 @@ private static void updateIcebergColumnMetadata(Column column, Types.NestedField */ private static List getSchema(ExternalTable dorisTable, long schemaId, boolean isView, Table icebergTable) { + return getSchema(dorisTable, schemaId, isView, icebergTable, + dorisTable.getCatalog().getExecutionAuthenticator(), + dorisTable.getCatalog().getEnableMappingVarbinary(), + dorisTable.getCatalog().getEnableMappingTimestampTz()); + } + + private static List getSchema(ExternalTable dorisTable, long schemaId, boolean isView, + Table icebergTable, ExecutionAuthenticator authenticator, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { try { - return dorisTable.getCatalog().getExecutionAuthenticator().execute(() -> { + return authenticator.execute(() -> { Schema schema; if (isView) { View icebergView = getIcebergView(dorisTable); @@ -1401,8 +1415,7 @@ private static List getSchema(ExternalTable dorisTable, long schemaId, b Preconditions.checkNotNull(schema, "Schema for " + type + " " + dorisTable.getCatalog().getName() + "." + dorisTable.getDbName() + "." + dorisTable.getName() + " is null"); - return parseSchema(schema, dorisTable.getCatalog().getEnableMappingVarbinary(), - dorisTable.getCatalog().getEnableMappingTimestampTz()); + return parseSchema(schema, enableMappingVarbinary, enableMappingTimestampTz); }); } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); @@ -1918,6 +1931,17 @@ static IcebergSchemaCacheValue getSchemaCacheValue( dorisTable.getOrBuildNameMapping(), schemaId, retainedTable); } + private static IcebergSchemaCacheValue getSchemaCacheValue( + ExternalTable dorisTable, long schemaId, IcebergSnapshotCacheValue snapshotValue, Table retainedTable) { + if (snapshotValue.getCapturedAuthenticator() == null) { + return getSchemaCacheValue(dorisTable, schemaId, retainedTable); + } + return icebergExternalMetaCache(dorisTable).getIcebergSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), schemaId, retainedTable, + snapshotValue.getCapturedAuthenticator(), snapshotValue.isEnableMappingVarbinary(), + snapshotValue.isEnableMappingTimestampTz()); + } + public static IcebergSnapshot getLatestIcebergSnapshot(Table table) { Snapshot snapshot = table.currentSnapshot(); long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); @@ -1939,11 +1963,14 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, Table table, long snapshotId, long schemaId) throws AnalysisException { return loadPartitionInfo(dorisTable, table, snapshotId, schemaId, - dorisTable.getCatalog().getExecutionAuthenticator()); + dorisTable.getCatalog().getExecutionAuthenticator(), + dorisTable.getCatalog().getEnableMappingVarbinary(), + dorisTable.getCatalog().getEnableMappingTimestampTz()); } static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, Table table, long snapshotId, - long schemaId, ExecutionAuthenticator authenticator) throws AnalysisException { + long schemaId, ExecutionAuthenticator authenticator, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) throws AnalysisException { if (snapshotId == IcebergUtils.UNKNOWN_SNAPSHOT_ID) { return IcebergPartitionInfo.empty(); } @@ -1960,8 +1987,9 @@ static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, Table ta Map nameToPartitionItem = Maps.newHashMap(); long retainedPayloadBytes = 0L; - List partitionColumns = IcebergUtils.getSchemaCacheValue( - dorisTable, schemaId, table).getPartitionColumns(); + List partitionColumns = icebergExternalMetaCache(dorisTable).getIcebergSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), schemaId, table, authenticator, + enableMappingVarbinary, enableMappingTimestampTz).getPartitionColumns(); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( @@ -2209,7 +2237,7 @@ public int compare(Map.Entry p1, Map.Entry retainedTable = sv.getRetainedIcebergTable(); return retainedTable.isPresent() - ? getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId(), retainedTable.get()) + ? getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId(), sv, retainedTable.get()) : getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId()); } @@ -2258,7 +2286,9 @@ static IcebergSnapshotCacheValue newExplicitSnapshotValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), getNameMapping(queryScopedTable), queryScopedTable) - .bindCapturedAuthenticator(generation.getAuthenticator()); + .bindCapturedAuthenticator(generation.getAuthenticator()) + .bindSchemaMappingOptions(generation.isEnableMappingVarbinary(), + generation.isEnableMappingTimestampTz()); } public static List getIcebergSchema(ExternalTable dorisTable) { @@ -2276,8 +2306,8 @@ public static List getIcebergPartitionColumns(Optional sna if (snapshotTable.isPresent()) { // Schema ID alone cannot identify the partition spec; metadata-only evolution may keep // the same schema and snapshot IDs while changing spec(), so derive both from T0. - return buildTableSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId(), - snapshotTable.get()).getPartitionColumns(); + return getSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId(), + snapshotValue, snapshotTable.get()).getPartitionColumns(); } return getSchemaCacheValue(dorisTable, snapshotValue).getPartitionColumns(); } @@ -2305,6 +2335,14 @@ public static Optional loadSchemaCacheValue( : Optional.of(buildTableSchemaCacheValue(dorisTable, schemaId, retainedTable)); } + static IcebergSchemaCacheValue buildTableSchemaCacheValue(ExternalTable dorisTable, long schemaId, + Table icebergTable, ExecutionAuthenticator authenticator, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { + List schema = getSchema(dorisTable, schemaId, false, icebergTable, authenticator, + enableMappingVarbinary, enableMappingTimestampTz); + return buildTableSchemaCacheValue(icebergTable, schema); + } + private static Optional loadViewSchemaCacheValue(ExternalTable dorisTable, long schemaId) { List schema = IcebergUtils.getSchema(dorisTable, schemaId, true, null); return Optional.of(new IcebergSchemaCacheValue(schema, Lists.newArrayList())); @@ -2318,6 +2356,10 @@ private static Optional loadTableSchemaCacheValue(ExternalTabl private static IcebergSchemaCacheValue buildTableSchemaCacheValue(ExternalTable dorisTable, long schemaId, Table icebergTable) { List schema = IcebergUtils.getSchema(dorisTable, schemaId, false, icebergTable); + return buildTableSchemaCacheValue(icebergTable, schema); + } + + private static IcebergSchemaCacheValue buildTableSchemaCacheValue(Table icebergTable, List schema) { // get table partition column info List tmpColumns = Lists.newArrayList(); PartitionSpec spec = icebergTable.spec(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java index 456b40514cd208..570a42749eeca3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java @@ -299,6 +299,19 @@ public void testEmptyMorIncrementalPlanningDoesNotAcquireUnusedFsView() throws E Assertions.assertNull(getField(node, HudiScanNode.class, "fsViewLease")); } + @Test + public void testEmptyFullScanDoesNotAcquireUnusedFsView() throws Exception { + HudiScanNode node = incrementalScanNode( + new StatementContext.ExternalScanTaskCache(), Mockito.mock(IncrementalRelation.class), true); + setField(node, HudiScanNode.class, "partitionInit", true); + setField(node, HudiScanNode.class, "prunedPartitions", Collections.emptyList()); + + List splits = node.getSplits(1); + + Assertions.assertTrue(splits.isEmpty()); + Assertions.assertNull(getField(node, HudiScanNode.class, "fsViewLease")); + } + private static HudiScanNode partitionScanNode( StatementContext.ExternalScanTaskCache cache, HoodieTableFileSystemView fsView, String queryInstant, boolean nativeReader, boolean runtimePrune) throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index ece8e0a8edb242..b9cea8d71ccc85 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -84,6 +84,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -94,6 +95,21 @@ import java.util.stream.IntStream; public class IcebergExternalMetaCacheTest { + + @Test + public void testSchemaCacheKeySeparatesMappingOptions() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergSchemaCacheKey plain = new IcebergSchemaCacheKey( + mapping, "table-uuid", 7L, false, false); + + Assert.assertNotEquals(plain, new IcebergSchemaCacheKey( + mapping, "table-uuid", 7L, true, false)); + Assert.assertNotEquals(plain, new IcebergSchemaCacheKey( + mapping, "table-uuid", 7L, false, true)); + Assert.assertNotEquals( + new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 3, false, false), + new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 4, false, false)); + } // U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) lower-cases to two characters in Locale.ROOT. private static final String DOTTED_CAPITAL_I = String.valueOf((char) 0x0130); @Rule @@ -325,9 +341,10 @@ public void testSameGenerationRefreshWithRenewedFileIoRetiresSnapshotProjection( IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), rotated.getRetainedIcebergTable()); snapshots.put(snapshotKey, rotatedProjection); - // An equivalent reload (same credentials, new FileIO instance) keeps the projection. + // Even an equivalent reload owns a distinct FileIO instance. Retiring the old table + // closes that exact instance, so its frozen projection must be rebuilt first. tables.put(mapping, equivalent); - Assert.assertSame(rotatedProjection, snapshots.peekIfPresent(snapshotKey)); + Assert.assertNull(snapshots.peekIfPresent(snapshotKey)); // A count-mode projection retains no handle and is never bound to credentials. IcebergSnapshotCacheValue countProjection = new IcebergSnapshotCacheValue( @@ -342,6 +359,70 @@ public void testSameGenerationRefreshWithRenewedFileIoRetiresSnapshotProjection( } } + @Test + public void testReplacementTransfersSharedFileIoCloseToCurrentGeneration() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + AtomicInteger firstCatalogCloses = new AtomicInteger(); + AtomicInteger currentCatalogCloses = new AtomicInteger(); + CountDownLatch currentRetired = new CountDownLatch(1); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + PropertiesFileIO sharedIo = new PropertiesFileIO("token", "shared"); + TableMetadata metadata = metadataWithLocation("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/metadata/shared-io-v1.json"); + IcebergTableCacheValue first = new IcebergTableCacheValue( + tableWithMetadata(metadata, sharedIo), null, () -> null, + sharedIo::close, firstCatalogCloses::incrementAndGet); + IcebergTableCacheValue current = new IcebergTableCacheValue( + tableWithMetadata(metadata, sharedIo), null, () -> null, + sharedIo::close, () -> { + currentCatalogCloses.incrementAndGet(); + currentRetired.countDown(); + }); + Assert.assertTrue(current.shareTableCleanupWith(first)); + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + + tables.put(mapping, first); + first.releaseLoaderReference(); + tables.put(mapping, current); + current.releaseLoaderReference(); + + Assert.assertEquals("the replacement must keep its shared IO open", 0, sharedIo.getCloseCount()); + Assert.assertEquals(1, firstCatalogCloses.get()); + tables.invalidateKey(mapping); + Assert.assertTrue(currentRetired.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(1, sharedIo.getCloseCount()); + Assert.assertEquals(1, currentCatalogCloses.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testRetiredSharedFileIoOwnerCannotBeAdoptedOrClosedTwice() { + PropertiesFileIO sharedIo = new PropertiesFileIO("token", "retired-shared"); + TableMetadata metadata = metadataWithLocation("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/metadata/retired-shared-io-v1.json"); + IcebergTableCacheValue retired = new IcebergTableCacheValue( + tableWithMetadata(metadata, sharedIo), null, () -> null, + sharedIo::close, () -> { }); + retired.retire(); + Assert.assertEquals(1, sharedIo.getCloseCount()); + + IcebergTableCacheValue candidate = new IcebergTableCacheValue( + tableWithMetadata(metadata, sharedIo), null, () -> null, + sharedIo::close, () -> { }); + Assert.assertFalse(candidate.shareTableCleanupWith(retired)); + candidate.abandonTableCleanup(); + candidate.retire(); + Assert.assertEquals("a rejected generation must not close an already closed exact FileIO again", + 1, sharedIo.getCloseCount()); + } + @Test public void testAcquisitionsFailWhenCatalogResetsMidFlight() { IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); @@ -429,6 +510,72 @@ protected CatalogIf getCatalog(long catalogId) { } } + @Test + public void testWritableTableLeaseClosesOwnedFileIoOnSuccessAndGenerationFailure() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps firstOps = Mockito.mock(IcebergMetadataOps.class); + IcebergMetadataOps nextOps = Mockito.mock(IcebergMetadataOps.class); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { }; + java.util.concurrent.atomic.AtomicReference currentOps = + new java.util.concurrent.atomic.AtomicReference<>(firstOps); + Mockito.when(catalog.getMetadataOps()).thenAnswer(invocation -> currentOps.get()); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.when(catalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + IcebergExternalCatalog.TableLoadContext context = + Mockito.mock(IcebergExternalCatalog.TableLoadContext.class); + Mockito.when(catalog.beginTableLoad()).thenReturn(context); + Mockito.when(context.getOps()).thenReturn(firstOps); + Mockito.when(context.getAuthenticator()).thenReturn(authenticator); + Mockito.when(context.getCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + IcebergCatalogResourceTracker.ResourceLease catalogLease = + Mockito.mock(IcebergCatalogResourceTracker.ResourceLease.class); + Mockito.when(context.promote()).thenReturn(catalogLease); + PropertiesFileIO successfulIo = new PropertiesFileIO("token", "success"); + PropertiesFileIO rejectedIo = new PropertiesFileIO("token", "rejected"); + Table successfulTable = tableWithMetadata( + metadataWithLocation("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/metadata/writable-success.json"), successfulIo); + Table rejectedTable = tableWithMetadata( + metadataWithLocation("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/metadata/writable-rejected.json"), rejectedIo); + Mockito.when(context.loadTable("remote_db", "remote_tbl")) + .thenReturn(successfulTable) + .thenAnswer(invocation -> { + currentOps.set(nextOps); + return rejectedTable; + }); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + try (IcebergExternalMetaCache.WritableTableLease lease = + cache.acquireWritableIcebergTable(dorisTable, firstOps)) { + Assert.assertSame(successfulTable, lease.getTable()); + Assert.assertEquals(0, successfulIo.getCloseCount()); + } + Assert.assertEquals(1, successfulIo.getCloseCount()); + Mockito.verify(catalogLease).close(); + + try { + cache.acquireWritableIcebergTable(dorisTable, firstOps); + Assert.fail("generation replacement must reject the writable handle"); + } catch (RuntimeException expected) { + Assert.assertTrue(expected.getMessage().contains("please retry")); + } + Assert.assertEquals(1, rejectedIo.getCloseCount()); + Mockito.verify(context, Mockito.times(1)).promote(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testResetCatalogReinitializesBeforeCaptureAndWritableStaysOnDispatchGeneration() { IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); @@ -519,22 +666,23 @@ public void testSnapshotPartitionLoadUsesCapturedAuthenticator() throws Exceptio }; Method loader = IcebergExternalMetaCache.class.getDeclaredMethod( "loadSnapshotProjection", ExternalTable.class, Table.class, Table.class, - String.class, boolean.class, ExecutionAuthenticator.class); + String.class, boolean.class, ExecutionAuthenticator.class, + boolean.class, boolean.class); loader.setAccessible(true); try (MockedStatic icebergUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { icebergUtils.when(() -> IcebergUtils.loadPartitionInfo( - dorisTable, projectionTable, 11L, 3L, capturedAuthenticator)) + dorisTable, projectionTable, 11L, 3L, capturedAuthenticator, true, false)) .thenReturn(IcebergPartitionInfo.empty()); icebergUtils.when(() -> IcebergUtils.getNameMapping(projectionTable)) .thenReturn(Optional.empty()); icebergUtils.clearInvocations(); loader.invoke(cache, dorisTable, projectionTable, projectionTable, - null, false, capturedAuthenticator); + null, false, capturedAuthenticator, true, false); icebergUtils.verify(() -> IcebergUtils.loadPartitionInfo( - dorisTable, projectionTable, 11L, 3L, capturedAuthenticator)); + dorisTable, projectionTable, 11L, 3L, capturedAuthenticator, true, false)); } finally { cache.close(); executor.shutdownNow(); @@ -712,6 +860,59 @@ private static void stubTableLoadContext(IcebergExternalCatalog catalog) { }); } + @Test + public void testTableWrapperConstructionFailureReleasesTransferredResources() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { }; + PropertiesFileIO fileIo = new PropertiesFileIO("token", "constructor-failure"); + Table table = Mockito.mock(Table.class, Mockito.withSettings().extraInterfaces(HasTableOperations.class)); + TableOperations tableOperations = Mockito.mock(TableOperations.class); + Mockito.when(((HasTableOperations) table).operations()).thenReturn(tableOperations); + Mockito.when(tableOperations.current()).thenThrow(new RuntimeException("metadata unavailable")); + Mockito.when(table.io()).thenReturn(fileIo); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.when(catalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + IcebergExternalCatalog.TableLoadContext context = + Mockito.mock(IcebergExternalCatalog.TableLoadContext.class); + Mockito.when(catalog.beginTableLoad()).thenReturn(context); + Mockito.when(context.getOps()).thenReturn(metadataOps); + Mockito.when(context.getAuthenticator()).thenReturn(authenticator); + Mockito.when(context.getCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + try { + Mockito.when(context.loadTable("remote_db", "remote_tbl")).thenReturn(table); + } catch (Exception e) { + throw new AssertionError(e); + } + IcebergCatalogResourceTracker.ResourceLease catalogLease = + Mockito.mock(IcebergCatalogResourceTracker.ResourceLease.class); + Mockito.when(context.promote()).thenReturn(catalogLease); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + try { + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + Assert.fail("wrapper construction must fail"); + } catch (RuntimeException expected) { + Assert.assertTrue(exceptionChainContains(expected, "metadata unavailable")); + } + Assert.assertEquals(1, fileIo.getCloseCount()); + Mockito.verify(catalogLease).close(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testRejectedTableGenerationsDoNotAccumulateSnapshotOrSchemaProjections() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -771,7 +972,8 @@ MetaCacheSizeEstimate prepareTableForCachePublication( IcebergTableCacheValue rejected = new IcebergTableCacheValue( tableWithMetadataLocation("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/metadata/rejected-schema.json")); IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey( - mapping, rejected.getTableUuid().get(), 0L); + mapping, rejected.getTableUuid().get(), 0L, + rejected.getRetainedIcebergTable().spec().specId()); IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( Collections.emptyList(), Collections.emptyList()); schemas.put(schemaKey, schemaValue); @@ -916,11 +1118,13 @@ MetaCacheSizeEstimate prepareTableForCachePublication( IcebergSnapshotCacheValue first = cache.getSnapshotCache(dorisTable); Assert.assertNull(tables.peekIfPresent(mapping)); Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); - // Same credentials on a new handle instance: the physically keyed projection is reused. - Assert.assertSame(first, cache.getSnapshotCache(dorisTable)); + // A new handle owns a distinct FileIO instance. Even with equal properties, retiring + // that handle closes its exact IO, so the projection is rebound before retirement. + IcebergSnapshotCacheValue sameCredentialsProjection = cache.getSnapshotCache(dorisTable); + Assert.assertNotSame(first, sameCredentialsProjection); // Rotated credentials: the projection frozen on the first handle is rebuilt. IcebergSnapshotCacheValue rebound = cache.getSnapshotCache(dorisTable); - Assert.assertNotSame(first, rebound); + Assert.assertNotSame(sameCredentialsProjection, rebound); Assert.assertSame(rotatedHandle.io(), rebound.getIcebergTable().get().io()); Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); Mockito.verify(metadataOps, Mockito.times(3)).loadTable("remote_db", "remote_tbl"); @@ -1123,7 +1327,9 @@ public void testDisabledTableCacheKeepsPhysicallyKeyedSchemaProjections() { NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); IcebergTableCacheValue table = new IcebergTableCacheValue( tableWithMetadataLocation("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/metadata/disabled-table-cache.json")); - IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey(mapping, table.getTableUuid().get(), 0L); + IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey( + mapping, table.getTableUuid().get(), 0L, + table.getRetainedIcebergTable().spec().specId()); IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( Collections.emptyList(), Collections.emptyList()); MetaCacheEntry schemas = cache.entry( @@ -1157,7 +1363,8 @@ public void testOldGenerationSchemaLoadCannotRepopulateAfterTableReplacement() { cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class).put(mapping, newTable); IcebergSchemaCacheKey staleKey = new IcebergSchemaCacheKey( - mapping, oldTable.getTableUuid().get(), 0L); + mapping, oldTable.getTableUuid().get(), 0L, + oldTable.getRetainedIcebergTable().spec().specId()); IcebergSchemaCacheValue staleValue = new IcebergSchemaCacheValue( Collections.emptyList(), Collections.emptyList()); MetaCacheEntry schemas = cache.entry( @@ -2935,6 +3142,7 @@ private Table tableWithMetadata(TableMetadata metadata, FileIO io) { /** A FileIO whose identity is its configuration, like a catalog-vended S3 FileIO. */ private static final class PropertiesFileIO implements FileIO { private final Map properties; + private final AtomicInteger closeCount = new AtomicInteger(); private PropertiesFileIO(String key, String value) { this.properties = Collections.singletonMap(key, value); @@ -2959,6 +3167,15 @@ public org.apache.iceberg.io.OutputFile newOutputFile(String path) { public void deleteFile(String path) { throw new UnsupportedOperationException(); } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + + private int getCloseCount() { + return closeCount.get(); + } } private IcebergTableCacheValue tableValueWithFields(int fieldCount) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index f33c24b0535450..a0c998d5faed22 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -195,10 +195,16 @@ public void testUpdateTablePropertiesCommitsAllProperties() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable( - Mockito.eq(dorisTable), Mockito.eq(ops))).thenReturn(icebergTable); + IcebergExternalMetaCache.WritableTableLease lease = + Mockito.mock(IcebergExternalMetaCache.WritableTableLease.class); + ExecutionAuthenticator authenticator = dorisCatalog.getExecutionAuthenticator(); + Mockito.when(lease.getTable()).thenReturn(icebergTable); + Mockito.when(lease.getAuthenticator()).thenReturn(authenticator); + mockedIcebergUtils.when(() -> IcebergUtils.acquireWritableIcebergTable(dorisTable, ops)) + .thenReturn(lease); ops.updateTableProperties(dorisTable, properties, 123L); + Mockito.verify(lease).close(); } Mockito.verify(updateProperties).set("write.target-file-size-bytes", "134217728"); @@ -219,12 +225,18 @@ public void testUpdateTablePropertiesDoesNotRefreshAfterCommitFailure() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable( - Mockito.eq(dorisTable), Mockito.eq(ops))).thenReturn(icebergTable); + IcebergExternalMetaCache.WritableTableLease lease = + Mockito.mock(IcebergExternalMetaCache.WritableTableLease.class); + ExecutionAuthenticator authenticator = dorisCatalog.getExecutionAuthenticator(); + Mockito.when(lease.getTable()).thenReturn(icebergTable); + Mockito.when(lease.getAuthenticator()).thenReturn(authenticator); + mockedIcebergUtils.when(() -> IcebergUtils.acquireWritableIcebergTable(dorisTable, ops)) + .thenReturn(lease); assertUserException(() -> ops.updateTableProperties( dorisTable, Collections.singletonMap("write.target-file-size-bytes", "134217728"), 123L), "commit failed"); + Mockito.verify(lease).close(); } Mockito.verify(dorisCatalog, Mockito.never()).getDbForReplay(Mockito.anyString()); From 3dc35037d72b8c1782921ff6d2912fba6b12af6d Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 28 Aug 2026 11:02:40 +0800 Subject: [PATCH 26/38] [fix](fe) Fix Iceberg lifecycle checkstyle ### What problem does this PR solve? Issue Number: None Related PR: #66913 Problem Summary: Fix the FE Checkstyle field-separation violation introduced while rebasing the Iceberg lifecycle tests. This is a formatting-only repair and does not change Iceberg or Hudi behavior. ### Release note None ### Check List (For Author) - Test: No need to test (formatting-only change); full FE Checkstyle passed. - Behavior changed: No - Does this need documentation: No --- .../doris/datasource/iceberg/IcebergExternalMetaCacheTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index b9cea8d71ccc85..e7cca2a19b720e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -110,8 +110,10 @@ public void testSchemaCacheKeySeparatesMappingOptions() { new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 3, false, false), new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 4, false, false)); } + // U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) lower-cases to two characters in Locale.ROOT. private static final String DOTTED_CAPITAL_I = String.valueOf((char) 0x0130); + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); From 8ca0cc7632b0fe61776f65c82826674bba95e4c7 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 28 Aug 2026 12:41:48 +0800 Subject: [PATCH 27/38] [fix](fe) Close Hadoop Iceberg catalog resources ### What problem does this PR solve? Issue Number: None Related PR: #66913 Problem Summary: Iceberg Hadoop catalog generations did not own and close their catalog-wide FileIO, and historical partition projections joined current partition fields to historical schemas by mutable names. This change gives each Hadoop catalog generation idempotent FileIO ownership, matches partition columns by Iceberg field ID, keys cached projections by the current schema identity, and fixes the deterministic frozen-spec test fixture. ### Release note Fix Iceberg catalog FileIO retirement and historical partition-column projection across schema renames. ### Check List (For Author) - Test: Unit Test - DorisHadoopCatalogTest - IcebergUtilsTest - IcebergExternalMetaCacheTest - Behavior changed: Yes (Iceberg Hadoop catalog generations release their shared FileIO, and historical partition columns remain stable across renames) - Does this need documentation: No --- .../iceberg/DorisHadoopCatalog.java | 96 +++++++++++++++++++ .../iceberg/IcebergExternalMetaCache.java | 3 +- .../iceberg/IcebergSchemaCacheKey.java | 17 +++- .../datasource/iceberg/IcebergUtils.java | 3 +- .../IcebergFileSystemMetaStoreProperties.java | 4 +- .../iceberg/DorisHadoopCatalogTest.java | 84 ++++++++++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 12 ++- .../datasource/iceberg/IcebergUtilsTest.java | 27 +++--- 8 files changed, 224 insertions(+), 22 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalog.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalogTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalog.java new file mode 100644 index 00000000000000..fbbd7b2b78263d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalog.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import com.google.common.base.Throwables; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.io.FileIO; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Owns and closes the shared FileIO created by one Iceberg HadoopCatalog generation. */ +public class DorisHadoopCatalog extends HadoopCatalog { + private final AtomicBoolean closed = new AtomicBoolean(); + private FileIO ownedFileIO; + + @Override + public void initialize(String name, Map properties) { + try { + super.initialize(name, properties); + ownedFileIO = extractFileIO(); + } catch (RuntimeException | Error failure) { + closePartiallyInitializedFileIO(failure); + throw failure; + } + } + + private void closePartiallyInitializedFileIO(Throwable initializationFailure) { + try { + FileIO fileIO = extractFileIO(); + if (fileIO != null) { + fileIO.close(); + } + } catch (Throwable closeFailure) { + initializationFailure.addSuppressed(closeFailure); + } + } + + @Override + public void close() throws IOException { + if (!closed.compareAndSet(false, true)) { + return; + } + Throwable closeFailure = null; + try { + super.close(); + } catch (Throwable e) { + closeFailure = e; + } + try { + if (ownedFileIO != null) { + ownedFileIO.close(); + } + } catch (Throwable e) { + if (closeFailure != null) { + closeFailure.addSuppressed(e); + } else { + closeFailure = e; + } + } finally { + ownedFileIO = null; + } + if (closeFailure != null) { + Throwables.throwIfInstanceOf(closeFailure, IOException.class); + Throwables.throwIfUnchecked(closeFailure); + throw new IOException(closeFailure); + } + } + + private FileIO extractFileIO() { + try { + Field fileIOField = HadoopCatalog.class.getDeclaredField("fileIO"); + fileIOField.setAccessible(true); + return (FileIO) fileIOField.get(this); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Failed to capture Iceberg HadoopCatalog FileIO ownership", e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 302dba7c84cc80..be0ca32e87e981 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -411,12 +411,13 @@ IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long if (!generation.isPresent()) { return (IcebergSchemaCacheValue) loadSchemaCacheValue( new IcebergSchemaCacheKey(nameMapping, "", schemaId, - retainedTable.spec().specId(), + retainedTable.spec().specId(), retainedTable.schema().schemaId(), enableMappingVarbinary, enableMappingTimestampTz), retainedTable, authenticator); } IcebergSchemaCacheKey key = new IcebergSchemaCacheKey( nameMapping, generation.get().getTableUuid(), schemaId, retainedTable.spec().specId(), + retainedTable.schema().schemaId(), enableMappingVarbinary, enableMappingTimestampTz); MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); SchemaCacheValue schemaCacheValue = entry diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java index 6bbd38617e20cb..6647d26f3c282b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java @@ -28,6 +28,9 @@ public class IcebergSchemaCacheKey extends SchemaCacheKey { private final String tableUuid; private final long schemaId; private final int partitionSpecId; + // The requested schemaId may be historical while the frozen table has a newer current schema. + // A rename changes this ID even when the table UUID and partition spec ID stay unchanged. + private final int projectionSchemaId; private final boolean enableMappingVarbinary; private final boolean enableMappingTimestampTz; @@ -51,10 +54,17 @@ public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long sch public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId, int partitionSpecId, boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { + this(nameMapping, tableUuid, schemaId, partitionSpecId, -1, + enableMappingVarbinary, enableMappingTimestampTz); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId, int partitionSpecId, + int projectionSchemaId, boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { super(nameMapping); this.tableUuid = java.util.Objects.requireNonNull(tableUuid, "tableUuid can not be null"); this.schemaId = schemaId; this.partitionSpecId = partitionSpecId; + this.projectionSchemaId = projectionSchemaId; this.enableMappingVarbinary = enableMappingVarbinary; this.enableMappingTimestampTz = enableMappingTimestampTz; } @@ -71,6 +81,10 @@ public int getPartitionSpecId() { return partitionSpecId; } + public int getProjectionSchemaId() { + return projectionSchemaId; + } + public boolean isEnableMappingVarbinary() { return enableMappingVarbinary; } @@ -93,6 +107,7 @@ public boolean equals(Object o) { IcebergSchemaCacheKey that = (IcebergSchemaCacheKey) o; return schemaId == that.schemaId && partitionSpecId == that.partitionSpecId + && projectionSchemaId == that.projectionSchemaId && enableMappingVarbinary == that.enableMappingVarbinary && enableMappingTimestampTz == that.enableMappingTimestampTz && tableUuid.equals(that.tableUuid); @@ -100,7 +115,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), tableUuid, schemaId, partitionSpecId, + return Objects.hashCode(super.hashCode(), tableUuid, schemaId, partitionSpecId, projectionSchemaId, enableMappingVarbinary, enableMappingTimestampTz); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index de9c8ce7b97371..e6e935578d23c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -2364,9 +2364,8 @@ private static IcebergSchemaCacheValue buildTableSchemaCacheValue(Table icebergT List tmpColumns = Lists.newArrayList(); PartitionSpec spec = icebergTable.spec(); for (PartitionField field : spec.fields()) { - Types.NestedField col = icebergTable.schema().findField(field.sourceId()); for (Column c : schema) { - if (c.getName().equalsIgnoreCase(col.name())) { + if (c.getUniqueId() == field.sourceId()) { tmpColumns.add(c); break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java index 3f7327b786cfed..188ceb02d48dd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.metastore; import org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator; +import org.apache.doris.datasource.iceberg.DorisHadoopCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.property.storage.HdfsProperties; import org.apache.doris.datasource.property.storage.StorageProperties; @@ -25,7 +26,6 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.catalog.Catalog; import java.util.List; @@ -48,7 +48,7 @@ public Catalog initCatalog(String catalogName, Map catalogProps, try { Configuration configuration = new Configuration(); toFileIOProperties(storagePropertiesList, catalogProps, configuration); - catalogProps.put(CatalogProperties.CATALOG_IMPL, CatalogUtil.ICEBERG_CATALOG_HADOOP); + catalogProps.put(CatalogProperties.CATALOG_IMPL, DorisHadoopCatalog.class.getName()); buildExecutionAuthenticator(storagePropertiesList); return this.executionAuthenticator.execute(() -> buildIcebergCatalog(catalogName, catalogProps, configuration)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalogTest.java new file mode 100644 index 00000000000000..a284049df2e349 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/DorisHadoopCatalogTest.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.hadoop.HadoopFileIO; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +class DorisHadoopCatalogTest { + + public static class TrackingFileIO extends HadoopFileIO { + private static final AtomicInteger CLOSE_COUNT = new AtomicInteger(); + + @Override + public void close() { + CLOSE_COUNT.incrementAndGet(); + } + } + + @TempDir + Path warehouse; + + @Test + void closesOwnedFileIOOnceAcrossRepeatedRetirement() throws Exception { + TrackingFileIO.CLOSE_COUNT.set(0); + DorisHadoopCatalog catalog = newCatalog(); + + catalog.close(); + catalog.close(); + + Assertions.assertEquals(1, TrackingFileIO.CLOSE_COUNT.get()); + } + + @Test + void closesFileIOWhenLockManagerInitializationFails() { + TrackingFileIO.CLOSE_COUNT.set(0); + Map properties = properties(); + properties.put("lock-impl", "missing.LockManager"); + DorisHadoopCatalog catalog = new DorisHadoopCatalog(); + catalog.setConf(new Configuration()); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> catalog.initialize("test", properties)); + + Assertions.assertEquals(1, TrackingFileIO.CLOSE_COUNT.get()); + } + + private DorisHadoopCatalog newCatalog() { + DorisHadoopCatalog catalog = new DorisHadoopCatalog(); + catalog.setConf(new Configuration()); + catalog.initialize("test", properties()); + return catalog; + } + + private Map properties() { + Map properties = new HashMap<>(); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse.toUri().toString()); + properties.put(CatalogProperties.FILE_IO_IMPL, TrackingFileIO.class.getName()); + return properties; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index e7cca2a19b720e..8632f4a2c29dba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -109,6 +109,9 @@ public void testSchemaCacheKeySeparatesMappingOptions() { Assert.assertNotEquals( new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 3, false, false), new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 4, false, false)); + Assert.assertNotEquals( + new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 3, 8, false, false), + new IcebergSchemaCacheKey(mapping, "table-uuid", 7L, 3, 9, false, false)); } // U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) lower-cases to two characters in Locale.ROOT. @@ -975,7 +978,8 @@ MetaCacheSizeEstimate prepareTableForCachePublication( tableWithMetadataLocation("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/metadata/rejected-schema.json")); IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey( mapping, rejected.getTableUuid().get(), 0L, - rejected.getRetainedIcebergTable().spec().specId()); + rejected.getRetainedIcebergTable().spec().specId(), + rejected.getRetainedIcebergTable().schema().schemaId(), false, false); IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( Collections.emptyList(), Collections.emptyList()); schemas.put(schemaKey, schemaValue); @@ -1331,7 +1335,8 @@ public void testDisabledTableCacheKeepsPhysicallyKeyedSchemaProjections() { tableWithMetadataLocation("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/metadata/disabled-table-cache.json")); IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey( mapping, table.getTableUuid().get(), 0L, - table.getRetainedIcebergTable().spec().specId()); + table.getRetainedIcebergTable().spec().specId(), + table.getRetainedIcebergTable().schema().schemaId(), false, false); IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( Collections.emptyList(), Collections.emptyList()); MetaCacheEntry schemas = cache.entry( @@ -1366,7 +1371,8 @@ public void testOldGenerationSchemaLoadCannotRepopulateAfterTableReplacement() { NameMapping.class, IcebergTableCacheValue.class).put(mapping, newTable); IcebergSchemaCacheKey staleKey = new IcebergSchemaCacheKey( mapping, oldTable.getTableUuid().get(), 0L, - oldTable.getRetainedIcebergTable().spec().specId()); + oldTable.getRetainedIcebergTable().spec().specId(), + oldTable.getRetainedIcebergTable().schema().schemaId(), false, false); IcebergSchemaCacheValue staleValue = new IcebergSchemaCacheValue( Collections.emptyList(), Collections.emptyList()); MetaCacheEntry schemas = cache.entry( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index d7e091da6d1069..f495c97d25a1ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -197,29 +197,30 @@ public void testGetFileFormatUsesConfiguredTableFormat() { @Test public void testPartitionColumnsUseFrozenTableSpec() { - Schema frozenSchema = new Schema(17, Arrays.asList( + Schema historicalSchema = new Schema(17, Arrays.asList( Types.NestedField.required(1, "p", Types.IntegerType.get()), Types.NestedField.optional(2, "q", Types.IntegerType.get()))); + Schema currentSchema = new Schema(18, Arrays.asList( + Types.NestedField.required(1, "p_renamed", Types.IntegerType.get()), + Types.NestedField.optional(2, "q", Types.IntegerType.get()))); Table frozenTable = Mockito.mock(Table.class); - Mockito.when(frozenTable.schema()).thenReturn(frozenSchema); - Mockito.when(frozenTable.schemas()).thenReturn( - Collections.singletonMap(frozenSchema.schemaId(), frozenSchema)); - Mockito.when(frozenTable.spec()).thenReturn(PartitionSpec.builderFor(frozenSchema).identity("p").build()); + Mockito.when(frozenTable.schema()).thenReturn(currentSchema); + Mockito.when(frozenTable.schemas()).thenReturn(ImmutableMap.of( + historicalSchema.schemaId(), historicalSchema, + currentSchema.schemaId(), currentSchema)); + Mockito.when(frozenTable.spec()).thenReturn( + PartitionSpec.builderFor(currentSchema).identity("p_renamed").build()); Mockito.when(frozenTable.currentSnapshot()).thenReturn(Mockito.mock(Snapshot.class)); IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); Mockito.when(dorisTable.getCatalog()).thenReturn(catalog); - Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); Mockito.when(catalog.getName()).thenReturn("catalog"); - IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( - IcebergPartitionInfo.empty(), new IcebergSnapshot(101L, frozenSchema.schemaId()), - Optional.empty(), frozenTable); - - List partitionColumns = IcebergUtils.getIcebergPartitionColumns( - Optional.of(new IcebergMvccSnapshot(cacheValue)), dorisTable); + IcebergSchemaCacheValue cacheValue = IcebergUtils.buildTableSchemaCacheValue( + dorisTable, historicalSchema.schemaId(), frozenTable, + new ExecutionAuthenticator() { }, false, false); - Assert.assertEquals(Collections.singletonList("p"), partitionColumns.stream() + Assert.assertEquals(Collections.singletonList("p"), cacheValue.getPartitionColumns().stream() .map(Column::getName).collect(java.util.stream.Collectors.toList())); } From acda98d872ad27ef2fff952c3a3a0097e9b06700 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 28 Aug 2026 14:37:15 +0800 Subject: [PATCH 28/38] [fix](fe) Fence Iceberg and Hudi lifecycle races ### What problem does this PR solve? Issue Number: None Related PR: #66913 Problem Summary: Hudi synchronous listing cancellation could complete terminal accounting before publishing cancellation and return partial splits. Catalog-property replay could publish new properties before the runtime generation reset, and Hudi generation reads could observe that intermediate state. Iceberg frozen tables also dropped weak FileIOTracker keys while Doris independently closed SDK-owned REST, Glue, and S3 Tables FileIO. This change publishes cancellation before task callbacks, commits replay properties once through the synchronized runtime reset, reads the HMS generation under the same monitor, and retains SDK-tracked operations while leaving FileIO closure to each catalog tracker. ### Release note Fix Hudi cancellation, catalog replay generation fencing, and Iceberg FileIO tracker ownership. ### Check List (For Author) - Test: Unit Test - ExternalCatalogRuntimeStateTest - HudiBatchFsViewOwnerTest - CatalogMgrTest - IcebergTableCacheValueTest - IcebergExternalMetaCacheTest - Behavior changed: Yes (cancelled Hudi listing fails instead of returning partial splits, replay swaps properties with runtime state atomically, and catalog SDK trackers remain the sole per-table FileIO owners) - Does this need documentation: No --- .../apache/doris/datasource/CatalogMgr.java | 2 - .../datasource/hive/HMSExternalCatalog.java | 2 +- .../datasource/hudi/source/HudiScanNode.java | 9 +- .../iceberg/IcebergExternalMetaCache.java | 56 ++----------- .../iceberg/IcebergSnapshotCacheValue.java | 6 ++ .../doris/datasource/CatalogMgrTest.java | 31 +++++++ .../ExternalCatalogRuntimeStateTest.java | 49 +++++++++++ .../hudi/source/HudiBatchFsViewOwnerTest.java | 19 +++++ .../iceberg/IcebergExternalMetaCacheTest.java | 17 ++-- .../iceberg/IcebergTableCacheValueTest.java | 83 +++++++++++++++---- 10 files changed, 194 insertions(+), 80 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 776ca6392dea4b..fea26b32843b4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -740,8 +740,6 @@ private void alterExternalCatalogPropsFenced(ExternalCatalog externalCatalog, Ca throw new DdlException("Invalid catalog properties: " + validationException.getMessage(), validationException); } - } else { - externalCatalog.tryModifyCatalogProps(newProps); } if (newProps.containsKey(METADATA_REFRESH_INTERVAL_SEC)) { long catalogId = externalCatalog.getId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 48ecde29f4c697..7117b4434ffd8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -91,7 +91,7 @@ public class HMSExternalCatalog extends ExternalCatalog { private volatile AbstractHiveProperties hmsProperties; private AtomicLong runtimeGeneration = new AtomicLong(); - public long getRuntimeGeneration() { + public synchronized long getRuntimeGeneration() { return runtimeGeneration.get(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 2a2f439b30a9f7..1e6b522ebeb032 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -899,7 +899,7 @@ void submissionDone() { } void discardBeforeSubmission() { - close(); + stopping.compareAndSet(false, true); submissionDone(); } @@ -924,7 +924,7 @@ void awaitCompletion() { } catch (java.util.concurrent.ExecutionException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } - if (cancelled.isDone() && !tasksFinished.isDone()) { + if (cancelled.isDone()) { throw new CancellationException("Hudi split listing was cancelled"); } } @@ -932,8 +932,11 @@ void awaitCompletion() { @Override public void close() { if (stopping.compareAndSet(false, true)) { - tasks.forEach(TerminalTask::requestStop); + // Publish cancellation first. Cancelling a FutureTask before it starts invokes + // done() synchronously and may complete terminal accounting on this thread. + // awaitCompletion must still observe cancellation rather than return partial splits. cancelled.complete(null); + tasks.forEach(TerminalTask::requestStop); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index be0ca32e87e981..5f2a78d65da9a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -51,7 +51,6 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; @@ -228,8 +227,9 @@ WritableTableLease acquireWritableIcebergTable( } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } - try (TableResourceOwner owner = new TableResourceOwner( - tableCleanup(context.getCatalogType(), ops, table))) { + // REST, Glue and S3 Tables SDK trackers own their per-table FileIO. Doris only + // retains the catalog generation here so the tracker outlives this table lease. + try (TableResourceOwner owner = new TableResourceOwner(() -> { })) { ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, context.isEnableMappingVarbinary(), context.isEnableMappingTimestampTz()); owner.add(context.promote()::close); @@ -255,6 +255,8 @@ WritableTableLease acquireWritableIcebergTable( } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } + // The SDK tracker is the sole FileIO close owner; the cache value retains its + // catalog generation until every borrower releases the frozen table. try (TableResourceOwner owner = new TableResourceOwner(() -> { })) { ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, enableMappingVarbinary, enableMappingTimestampTz); @@ -494,8 +496,7 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } - try (TableResourceOwner owner = new TableResourceOwner( - tableCleanup(context.getCatalogType(), ops, table))) { + try (TableResourceOwner owner = new TableResourceOwner(() -> { })) { ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, enableMappingVarbinary, enableMappingTimestampTz); try (TableResourceOwner catalogOwner = new TableResourceOwner(context.promote()::close)) { @@ -613,51 +614,6 @@ private IcebergTableCacheValue.Lease borrow(NameMapping nameMapping) { } } - private Runnable tableCleanup(String catalogType, IcebergMetadataOps ops, Table table) { - FileIO catalogFileIO = IcebergExternalCatalog.ICEBERG_REST.equals(catalogType) ? catalogFileIO(ops) : null; - if (!shouldCloseTableFileIO(catalogType, table.io(), catalogFileIO)) { - return () -> { }; - } - FileIO tableFileIO = table.io(); - return () -> { - try { - tableFileIO.close(); - } catch (Exception e) { - LOG.warn("Failed to close Iceberg table FileIO", e); - } - }; - } - - static boolean shouldCloseTableFileIO(String catalogType, FileIO tableFileIO, FileIO catalogFileIO) { - if (IcebergExternalCatalog.ICEBERG_GLUE.equals(catalogType) - || IcebergExternalCatalog.ICEBERG_S3_TABLES.equals(catalogType)) { - return true; - } - return IcebergExternalCatalog.ICEBERG_REST.equals(catalogType) - && catalogFileIO != null && tableFileIO != catalogFileIO; - } - - @Nullable - private FileIO catalogFileIO(IcebergMetadataOps ops) { - Object catalog = ops.getCatalog(); - try { - if (catalog instanceof org.apache.iceberg.rest.RESTCatalog) { - Field sessionCatalogField = org.apache.iceberg.rest.RESTCatalog.class - .getDeclaredField("sessionCatalog"); - sessionCatalogField.setAccessible(true); - catalog = sessionCatalogField.get(catalog); - } - if (catalog instanceof org.apache.iceberg.rest.RESTSessionCatalog) { - Field ioField = org.apache.iceberg.rest.RESTSessionCatalog.class.getDeclaredField("io"); - ioField.setAccessible(true); - return (FileIO) ioField.get(catalog); - } - } catch (Exception e) { - LOG.warn("Failed to identify REST catalog FileIO; skip per-table close to protect shared IO", e); - } - return null; - } - private static ExecutionAuthenticator requireExecutionAuthenticator(CatalogIf catalog) { if (!(catalog instanceof ExternalCatalog)) { throw new RuntimeException("Iceberg metadata cache requires an external catalog"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 66b1b29c7371e0..df94a648221e48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -390,6 +390,11 @@ public LocationProvider locationProvider() { } private static class FrozenTableOperations implements TableOperations { + // REST, Glue and S3 Tables catalog FileIOTracker use weak TableOperations keys and close the + // associated FileIO when a key disappears. Retain the SDK operations for the whole frozen + // generation; the promoted catalog-generation guard keeps the tracker itself alive until + // all borrowers end. + private final TableOperations sdkTrackedOperations; private final TableMetadata metadata; private final FileIO fileIO; private final EncryptionManager encryptionManager; @@ -398,6 +403,7 @@ private static class FrozenTableOperations implements TableOperations { private FrozenTableOperations(TableOperations source, TableMetadata metadata, boolean nonGrowing) { + this.sdkTrackedOperations = source; this.metadata = metadata; this.fileIO = source.io(); this.encryptionManager = source.encryption(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index e17153e10f6593..a35716aa95d2e1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -192,6 +192,37 @@ void testReplayKeepsPersistedLegacyPaimonOptionLoadableButInactive() throws Exce Assertions.assertTrue(restoredProperties.getTableOptionsMap().isEmpty()); } + @Test + void testReplayPublishesPropertiesOnlyThroughTheFencedCommit() throws Exception { + CatalogMgr catalogMgr = new CatalogMgr(); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + long catalogId = 46L; + Mockito.when(catalog.getId()).thenReturn(catalogId); + addCatalog(catalogMgr, catalog); + Map oldProperties = ImmutableMap.of("s3.access_key", "old"); + Map newProperties = ImmutableMap.of("s3.access_key", "new"); + CatalogLog log = new CatalogLog(); + log.setCatalogId(catalogId); + log.setNewProps(newProperties); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any())) + .thenAnswer(invocation -> { + java.util.function.Supplier action = invocation.getArgument(1); + return action.get(); + }); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.replayAlterCatalogProps(log, oldProperties, true); + } + + Mockito.verify(catalog, Mockito.never()).tryModifyCatalogProps(Mockito.any()); + Mockito.verify(catalog).modifyCatalogProps(newProperties); + Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId); + } + private static class LatchingValidationCatalog extends ExternalCatalog { private final CountDownLatch validationStarted = new CountDownLatch(1); private final CountDownLatch initializationReadProperties = new CountDownLatch(1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogRuntimeStateTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogRuntimeStateTest.java index a2c5f229e8625c..f0bb7c334ca5ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogRuntimeStateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogRuntimeStateTest.java @@ -28,6 +28,13 @@ import java.lang.reflect.Field; import java.util.Collections; +import java.util.HashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; public class ExternalCatalogRuntimeStateTest { @@ -50,6 +57,48 @@ public void testHmsRuntimeStateRestoredAfterGsonReplay() throws Exception { assertTrackerCanRetainAndRelease(restored, HMSExternalCatalog.class, "icebergResourceTracker"); } + @Test + public void testHmsRuntimeGenerationCannotObservePropertyCommitMidReset() throws Exception { + CountDownLatch resetEntered = new CountDownLatch(1); + CountDownLatch allowResetToFinish = new CountDownLatch(1); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 3L, "hms", null, + new HashMap<>(Collections.singletonMap("s3.access_key", "old")), "") { + @Override + public synchronized void notifyPropertiesUpdated(java.util.Map updatedProps) { + resetEntered.countDown(); + try { + Assertions.assertTrue(allowResetToFinish.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + }; + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future modifier = executor.submit(() -> + catalog.modifyCatalogProps(Collections.singletonMap("s3.access_key", "new"))); + Assertions.assertTrue(resetEntered.await(5, TimeUnit.SECONDS)); + Assertions.assertEquals("new", catalog.getProperties().get("s3.access_key")); + + CountDownLatch readerStarted = new CountDownLatch(1); + Future reader = executor.submit(() -> { + readerStarted.countDown(); + return catalog.getRuntimeGeneration(); + }); + Assertions.assertTrue(readerStarted.await(5, TimeUnit.SECONDS)); + Assertions.assertThrows(TimeoutException.class, () -> reader.get(200, TimeUnit.MILLISECONDS)); + + allowResetToFinish.countDown(); + modifier.get(5, TimeUnit.SECONDS); + Assertions.assertEquals(1L, reader.get(5, TimeUnit.SECONDS)); + } finally { + allowResetToFinish.countDown(); + executor.shutdownNow(); + } + } + private static > T roundTrip(CatalogIf catalog, Class expectedType) { String json = GsonUtils.GSON.toJson(catalog, CatalogIf.class); CatalogIf restored = GsonUtils.GSON.fromJson(json, CatalogIf.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index ba5bbe4eec732a..77e2a861cff5dc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -221,4 +221,23 @@ void synchronousListingDiscardBeforeSubmissionReleasesLease() { Mockito.verify(lease).close(); Assertions.assertDoesNotThrow(owner::awaitCompletion); } + + @Test + void synchronousListingCancellationWinsAfterTerminalTasksCompleteInline() { + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease); + HudiScanNode.TerminalTask completed = new HudiScanNode.TerminalTask(() -> { }, () -> { }); + HudiScanNode.TerminalTask queued = new HudiScanNode.TerminalTask( + () -> Assertions.fail("cancelled task must not run"), () -> { }); + owner.track(completed); + owner.track(queued); + completed.run(); + owner.submissionDone(); + + owner.close(); + + Assertions.assertTrue(queued.isCancelled()); + Mockito.verify(lease).close(); + Assertions.assertThrows(CancellationException.class, owner::awaitCompletion); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 8632f4a2c29dba..66169e3a6ba38d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -516,7 +516,7 @@ protected CatalogIf getCatalog(long catalogId) { } @Test - public void testWritableTableLeaseClosesOwnedFileIoOnSuccessAndGenerationFailure() throws Exception { + public void testWritableTableLeaseLeavesSdkTrackedFileIoToCatalogTracker() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); IcebergMetadataOps firstOps = Mockito.mock(IcebergMetadataOps.class); @@ -526,13 +526,13 @@ public void testWritableTableLeaseClosesOwnedFileIoOnSuccessAndGenerationFailure new java.util.concurrent.atomic.AtomicReference<>(firstOps); Mockito.when(catalog.getMetadataOps()).thenAnswer(invocation -> currentOps.get()); Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); - Mockito.when(catalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + Mockito.when(catalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_S3_TABLES); IcebergExternalCatalog.TableLoadContext context = Mockito.mock(IcebergExternalCatalog.TableLoadContext.class); Mockito.when(catalog.beginTableLoad()).thenReturn(context); Mockito.when(context.getOps()).thenReturn(firstOps); Mockito.when(context.getAuthenticator()).thenReturn(authenticator); - Mockito.when(context.getCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + Mockito.when(context.getCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_S3_TABLES); IcebergCatalogResourceTracker.ResourceLease catalogLease = Mockito.mock(IcebergCatalogResourceTracker.ResourceLease.class); Mockito.when(context.promote()).thenReturn(catalogLease); @@ -564,7 +564,7 @@ protected CatalogIf getCatalog(long catalogId) { Assert.assertSame(successfulTable, lease.getTable()); Assert.assertEquals(0, successfulIo.getCloseCount()); } - Assert.assertEquals(1, successfulIo.getCloseCount()); + Assert.assertEquals(0, successfulIo.getCloseCount()); Mockito.verify(catalogLease).close(); try { @@ -573,7 +573,7 @@ protected CatalogIf getCatalog(long catalogId) { } catch (RuntimeException expected) { Assert.assertTrue(expected.getMessage().contains("please retry")); } - Assert.assertEquals(1, rejectedIo.getCloseCount()); + Assert.assertEquals(0, rejectedIo.getCloseCount()); Mockito.verify(context, Mockito.times(1)).promote(); } finally { cache.close(); @@ -879,13 +879,13 @@ public void testTableWrapperConstructionFailureReleasesTransferredResources() { Mockito.when(table.io()).thenReturn(fileIo); Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); - Mockito.when(catalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + Mockito.when(catalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_S3_TABLES); IcebergExternalCatalog.TableLoadContext context = Mockito.mock(IcebergExternalCatalog.TableLoadContext.class); Mockito.when(catalog.beginTableLoad()).thenReturn(context); Mockito.when(context.getOps()).thenReturn(metadataOps); Mockito.when(context.getAuthenticator()).thenReturn(authenticator); - Mockito.when(context.getCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_GLUE); + Mockito.when(context.getCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_S3_TABLES); try { Mockito.when(context.loadTable("remote_db", "remote_tbl")).thenReturn(table); } catch (Exception e) { @@ -910,7 +910,8 @@ protected CatalogIf getCatalog(long catalogId) { } catch (RuntimeException expected) { Assert.assertTrue(exceptionChainContains(expected, "metadata unavailable")); } - Assert.assertEquals(1, fileIo.getCloseCount()); + Assert.assertEquals(0, fileIo.getCloseCount()); + Mockito.verify(context).close(); Mockito.verify(catalogLease).close(); } finally { cache.close(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java index 75a4b40bbb847a..1e67912d492ffe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValueTest.java @@ -21,11 +21,16 @@ import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.nereids.StatementContext; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.FileIOTracker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.util.ArrayList; @@ -55,22 +60,68 @@ void leaseKeepsExecutorFromItsTableGeneration() { } @Test - void classifiesOnlyPerTableFileIOAsOwned() { - FileIO tableIo = newProxy(FileIO.class); - FileIO catalogIo = newProxy(FileIO.class); - - Assertions.assertTrue(IcebergExternalMetaCache.shouldCloseTableFileIO( - IcebergExternalCatalog.ICEBERG_GLUE, tableIo, null)); - Assertions.assertTrue(IcebergExternalMetaCache.shouldCloseTableFileIO( - IcebergExternalCatalog.ICEBERG_S3_TABLES, tableIo, null)); - Assertions.assertTrue(IcebergExternalMetaCache.shouldCloseTableFileIO( - IcebergExternalCatalog.ICEBERG_REST, tableIo, catalogIo)); - Assertions.assertFalse(IcebergExternalMetaCache.shouldCloseTableFileIO( - IcebergExternalCatalog.ICEBERG_REST, catalogIo, catalogIo)); - Assertions.assertFalse(IcebergExternalMetaCache.shouldCloseTableFileIO( - IcebergExternalCatalog.ICEBERG_REST, tableIo, null)); - Assertions.assertFalse(IcebergExternalMetaCache.shouldCloseTableFileIO( - IcebergExternalCatalog.ICEBERG_DLF, tableIo, null)); + void frozenGenerationRetainsSdkTrackedTableOperations() throws Exception { + TableOperations trackedOperations = Mockito.mock(TableOperations.class); + Table sdkTable = Mockito.mock(Table.class, + Mockito.withSettings().extraInterfaces(HasTableOperations.class)); + Mockito.when(((HasTableOperations) sdkTable).operations()).thenReturn(trackedOperations); + Mockito.when(sdkTable.name()).thenReturn("db.tbl"); + + Table retained = IcebergSnapshotCacheValue.retainTableGeneration(sdkTable); + TableOperations frozenOperations = ((HasTableOperations) retained).operations(); + Field sdkTrackedOperations = frozenOperations.getClass().getDeclaredField("sdkTrackedOperations"); + sdkTrackedOperations.setAccessible(true); + + Assertions.assertSame(trackedOperations, sdkTrackedOperations.get(frozenOperations)); + } + + @Test + void sdkTrackerCannotCloseFileIoBeforeDorisBorrowerEnds() { + FileIOTracker sdkTracker = new FileIOTracker(); + assertSdkTrackerCannotCloseBeforeBorrowerEnds(sdkTracker::track, sdkTracker::close); + } + + @Test + void s3TablesSdkTrackerCannotCloseFileIoBeforeDorisBorrowerEnds() { + software.amazon.s3tables.iceberg.imports.FileIOTracker sdkTracker = + new software.amazon.s3tables.iceberg.imports.FileIOTracker(); + assertSdkTrackerCannotCloseBeforeBorrowerEnds(sdkTracker::track, sdkTracker::close); + } + + private void assertSdkTrackerCannotCloseBeforeBorrowerEnds( + java.util.function.Consumer track, Runnable closeTracker) { + AtomicInteger fileIoCloses = new AtomicInteger(); + FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.doAnswer(invocation -> { + fileIoCloses.incrementAndGet(); + return null; + }).when(fileIO).close(); + TableOperations trackedOperations = Mockito.mock(TableOperations.class); + Mockito.when(trackedOperations.io()).thenReturn(fileIO); + track.accept(trackedOperations); + Table sdkTable = Mockito.mock(Table.class, + Mockito.withSettings().extraInterfaces(HasTableOperations.class)); + Mockito.when(((HasTableOperations) sdkTable).operations()).thenReturn(trackedOperations); + Mockito.when(sdkTable.name()).thenReturn("db.tbl"); + + IcebergCatalogResourceTracker catalogTracker = new IcebergCatalogResourceTracker(); + IcebergCatalogResourceTracker.LoadGuard guard = catalogTracker.beginLoad(); + IcebergCatalogResourceTracker.ResourceLease catalogLease = guard.promote(); + guard.close(); + Table retainedTable = IcebergSnapshotCacheValue.retainTableGeneration(sdkTable); + IcebergTableCacheValue value = new IcebergTableCacheValue( + retainedTable, null, () -> null, () -> { }, catalogLease::close); + IcebergTableCacheValue.Lease borrower = value.tryAcquire(); + Assertions.assertNotNull(borrower); + value.releaseLoaderReference(); + + catalogTracker.retireCurrent(closeTracker); + value.releaseCacheReference(); + Assertions.assertEquals(0, fileIoCloses.get()); + + borrower.close(); + Mockito.verify(fileIO, Mockito.timeout(3000)).close(); + Assertions.assertEquals(1, fileIoCloses.get()); } @Test From d86ecd5f351b9825f80a59641ba559171b56d7ab Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 31 Aug 2026 10:36:16 +0800 Subject: [PATCH 29/38] [fix](fe) Guard Iceberg operations during catalog reset ### What problem does this PR solve? Issue Number: None Related PR: #66913 Problem Summary: Iceberg catalog operations could outlive a catalog reset while their Catalog and FileIO generation was retired, and could combine an old catalog with new runtime properties or database name mappings. Retain the exact runtime generation for direct operations, freeze generation configuration, and resolve database identity under the catalog reset monitor for both dedicated Iceberg and HMS catalogs. ### Release note Fix Iceberg catalog operation lifetime during runtime reset. ### Check List (For Author) - Test: Unit Test - Iceberg lifecycle, metadata operation, cache, scan node, and runtime-state FE unit tests - ./build.sh --fe - Behavior changed: Yes (Iceberg catalog operations retain one consistent runtime generation until completion) - Does this need documentation: No --- .../datasource/hive/HMSExternalCatalog.java | 21 +- .../IcebergCatalogResourceTracker.java | 6 +- .../iceberg/IcebergExternalCatalog.java | 21 +- .../iceberg/IcebergExternalDatabase.java | 11 +- .../iceberg/IcebergMetadataOps.java | 214 ++++++++++++------ .../iceberg/IcebergMetadataOpTest.java | 114 +++++++++- .../IcebergMetadataOpsValidationTest.java | 3 + .../iceberg/source/IcebergScanNodeTest.java | 2 - 8 files changed, 292 insertions(+), 100 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 7117b4434ffd8c..0f14d887a60df5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -310,6 +310,25 @@ public synchronized IcebergMetadataOps getIcebergMetadataOps() { return icebergMetadataOps; } + /** Retains the exact HMS Iceberg runtime while a direct catalog operation is in progress. */ + public synchronized IcebergCatalogResourceTracker.LoadGuard beginIcebergCatalogOperation( + IcebergMetadataOps expectedOps) { + makeSureInitialized(); + if (icebergMetadataOps != expectedOps) { + throw new IllegalStateException("Iceberg catalog runtime changed before the operation started"); + } + return icebergResourceTracker.beginOperation(); + } + + /** Resolves a database only while the expected HMS Iceberg runtime is still current. */ + public synchronized ExternalDatabase getDbForIcebergCatalogOperation( + IcebergMetadataOps expectedOps, String dbName) { + if (icebergMetadataOps != expectedOps) { + throw new IllegalStateException("Iceberg catalog runtime changed before database resolution"); + } + return getDbNullable(dbName); + } + /** Retains the exact HMS Iceberg runtime while a table cache generation is being loaded or borrowed. */ public synchronized IcebergTableLoadContext beginIcebergTableLoad() { makeSureInitialized(); @@ -389,7 +408,7 @@ public boolean isEnableMappingTimestampTz() { } public Table loadTable(String dbName, String tableName) throws Exception { - return authenticator.execute(() -> ops.loadTable(dbName, tableName)); + return authenticator.execute(() -> ops.loadTableWithinCatalogGeneration(dbName, tableName)); } public IcebergCatalogResourceTracker.ResourceLease promote() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java index 8c63053807a181..4f92b46c59e6d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCatalogResourceTracker.java @@ -19,11 +19,15 @@ import java.util.concurrent.atomic.AtomicBoolean; -/** Keeps one catalog generation alive while tables loaded through it still have owners or borrowers. */ +/** Keeps one catalog generation alive while operations, loaded tables, or borrowers still use it. */ public final class IcebergCatalogResourceTracker { private Generation current = new Generation(); public synchronized LoadGuard beginLoad() { + return beginOperation(); + } + + public synchronized LoadGuard beginOperation() { current.retain(); return new LoadGuard(current); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index f79612e74c2775..815575da0409d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -27,8 +27,10 @@ import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalObjectLog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.metacache.CacheSpec; @@ -199,6 +201,23 @@ synchronized TableLoadContext beginTableLoad() { resourceTracker.beginLoad()); } + synchronized IcebergCatalogResourceTracker.LoadGuard beginCatalogOperation(IcebergMetadataOps expectedOps) { + makeSureInitialized(); + if (metadataOps != expectedOps) { + throw new IllegalStateException("Iceberg catalog runtime changed before the operation started"); + } + return resourceTracker.beginOperation(); + } + + synchronized ExternalDatabase getDbForCatalogOperation( + IcebergMetadataOps expectedOps, String dbName) { + makeSureInitialized(); + if (metadataOps != expectedOps) { + throw new IllegalStateException("Iceberg catalog runtime changed before database resolution"); + } + return getDbNullable(dbName); + } + public String getIcebergCatalogType() { makeSureInitialized(); return icebergCatalogType; @@ -289,7 +308,7 @@ IcebergMetadataOps getOps() { } Table loadTable(String dbName, String tableName) throws Exception { - return authenticator.execute(() -> ops.loadTable(dbName, tableName)); + return authenticator.execute(() -> ops.loadTableWithinCatalogGeneration(dbName, tableName)); } String getCatalogType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java index cfc50f3222b1f5..7de9eb5e9679c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java @@ -21,9 +21,6 @@ import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.InitDatabaseLog; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; - import java.util.Map; public class IcebergExternalDatabase extends ExternalDatabase { @@ -42,11 +39,9 @@ public IcebergExternalTable buildTableInternal(String remoteTableName, String lo public String getLocation() { try { - return extCatalog.getExecutionAuthenticator().execute(() -> { - Map props = ((SupportsNamespaces) ((IcebergExternalCatalog) getCatalog()).getCatalog()) - .loadNamespaceMetadata(Namespace.of(name)); - return props.getOrDefault("location", ""); - }); + IcebergMetadataOps ops = (IcebergMetadataOps) extCatalog.getMetadataOps(); + Map props = ops.loadNamespaceMetadata(name); + return props.getOrDefault("location", ""); } catch (Exception e) { throw new RuntimeException("Failed to get location for Iceberg database: " + name, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 9ed866d57ca27b..e67753083be50d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -39,6 +39,7 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.property.metastore.IcebergRestProperties; import org.apache.doris.datasource.property.metastore.MetastoreProperties; @@ -87,12 +88,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.Callable; import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -106,6 +109,12 @@ public class IcebergMetadataOps implements ExternalMetadataOps { protected SupportsNamespaces nsCatalog; private ExecutionAuthenticator executionAuthenticator; private final ThreadPoolExecutor threadPoolWithPreAuth; + private final boolean viewCatalogEnabled; + private final boolean nestedNamespaceEnabled; + private final boolean enableMappingVarbinary; + private final boolean enableMappingTimestampTz; + private final String icebergCatalogType; + private final Map catalogProperties; // Generally, there should be only two levels under the catalog, namely .
, // but the REST type catalog is obtained from an external server, // and the level provided by the external server may be three levels, ..
. @@ -119,11 +128,17 @@ public IcebergMetadataOps(ExternalCatalog dorisCatalog, Catalog catalog) { nsCatalog = (SupportsNamespaces) catalog; this.executionAuthenticator = dorisCatalog.getExecutionAuthenticator(); this.threadPoolWithPreAuth = dorisCatalog.getThreadPoolWithPreAuth(); + this.catalogProperties = Collections.unmodifiableMap(new HashMap<>(dorisCatalog.getProperties())); + this.icebergCatalogType = catalogProperties.get(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE); + this.enableMappingVarbinary = dorisCatalog.getEnableMappingVarbinary(); + this.enableMappingTimestampTz = dorisCatalog.getEnableMappingTimestampTz(); - if (dorisCatalog.getProperties().containsKey(IcebergExternalCatalog.EXTERNAL_CATALOG_NAME)) { + if (catalogProperties.containsKey(IcebergExternalCatalog.EXTERNAL_CATALOG_NAME)) { externalCatalogName = - Optional.of(dorisCatalog.getProperties().get(IcebergExternalCatalog.EXTERNAL_CATALOG_NAME)); + Optional.of(catalogProperties.get(IcebergExternalCatalog.EXTERNAL_CATALOG_NAME)); } + viewCatalogEnabled = determineViewCatalogEnabled(); + nestedNamespaceEnabled = determineNestedNamespaceEnabled(); } public Catalog getCatalog() { @@ -144,7 +159,7 @@ public void close() { @Override public boolean tableExist(String dbName, String tblName) { try { - return executionAuthenticator.execute(() -> catalog.tableExists(getTableIdentifier(dbName, tblName))); + return executeCatalogOperation(() -> tableExistsInternal(dbName, tblName)); } catch (Exception e) { throw new RuntimeException("Failed to check table exist, error message is:" + e.getMessage(), e); } @@ -152,7 +167,7 @@ public boolean tableExist(String dbName, String tblName) { public boolean databaseExist(String dbName) { try { - return executionAuthenticator.execute(() -> nsCatalog.namespaceExists(getNamespace(dbName))); + return executeCatalogOperation(() -> databaseExistsInternal(dbName)); } catch (Exception e) { throw new RuntimeException("Failed to check database exist, error message is:" + e.getMessage(), e); } @@ -160,7 +175,7 @@ public boolean databaseExist(String dbName) { public List listDatabaseNames() { try { - return executionAuthenticator.execute(() -> listNestedNamespaces(getNamespace())); + return executeCatalogOperation(() -> listNestedNamespaces(getNamespace())); } catch (Exception e) { LOG.warn("failed to list database names in catalog {}, root cause: {}", dorisCatalog.getName(), Util.getRootCauseMessage(e), e); @@ -172,18 +187,13 @@ public List listDatabaseNames() { private List listNestedNamespaces(Namespace parentNs) { // Handle nested namespaces for Iceberg REST catalog, // only if "iceberg.rest.nested-namespace-enabled" is true. - if (dorisCatalog instanceof IcebergRestExternalCatalog) { - IcebergRestExternalCatalog restCatalog = (IcebergRestExternalCatalog) dorisCatalog; - MetastoreProperties metaProps = restCatalog.getCatalogProperty().getMetastoreProperties(); - if (metaProps instanceof IcebergRestProperties - && ((IcebergRestProperties) metaProps).isIcebergRestNestedNamespaceEnabled()) { - return nsCatalog.listNamespaces(parentNs) - .stream() - .flatMap(childNs -> Stream.concat( - Stream.of(childNs.toString()), - listNestedNamespaces(childNs).stream() - )).collect(Collectors.toList()); - } + if (nestedNamespaceEnabled) { + return nsCatalog.listNamespaces(parentNs) + .stream() + .flatMap(childNs -> Stream.concat( + Stream.of(childNs.toString()), + listNestedNamespaces(childNs).stream() + )).collect(Collectors.toList()); } return nsCatalog.listNamespaces(parentNs) @@ -195,27 +205,7 @@ private List listNestedNamespaces(Namespace parentNs) { @Override public List listTableNames(String dbName) { try { - return executionAuthenticator.execute(() -> { - List tableIdentifiers = catalog.listTables(getNamespace(dbName)); - List views; - // Our original intention was simply to clearly define the responsibilities of ViewCatalog and Catalog. - // IcebergMetadataOps handles listTableNames and listViewNames separately. - // listTableNames should only focus on the table type, - // but in reality, Iceberg's return includes views. Therefore, we added a filter to exclude views. - if (isViewCatalogEnabled()) { - views = ((ViewCatalog) catalog).listViews(getNamespace(dbName)) - .stream().map(TableIdentifier::name).collect(Collectors.toList()); - } else { - views = Collections.emptyList(); - } - if (views.isEmpty()) { - return tableIdentifiers.stream().map(TableIdentifier::name).collect(Collectors.toList()); - } else { - return tableIdentifiers.stream() - .map(TableIdentifier::name) - .filter(name -> !views.contains(name)).collect(Collectors.toList()); - } - }); + return executeCatalogOperation(() -> listTableNamesInternal(dbName)); } catch (RuntimeException e) { // We want to catch real exception like NoSuchNamespaceException and throw it directly throw e; @@ -228,7 +218,7 @@ public List listTableNames(String dbName) { public boolean createDbImpl(String dbName, boolean ifNotExists, Map properties) throws DdlException { try { - return executionAuthenticator.execute(() -> performCreateDb(dbName, ifNotExists, properties)); + return executeCatalogOperation(() -> performCreateDb(dbName, ifNotExists, properties)); } catch (Exception e) { throw new DdlException("Failed to create database: " + dbName + ": " + Util.getRootCauseMessage(e), e); @@ -243,7 +233,7 @@ public void afterCreateDb() { private boolean performCreateDb(String dbName, boolean ifNotExists, Map properties) throws DdlException { SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog; - if (databaseExist(dbName)) { + if (databaseExistsInternal(dbName)) { if (ifNotExists) { LOG.info("create database[{}] which already exists", dbName); return true; @@ -252,7 +242,6 @@ private boolean performCreateDb(String dbName, boolean ifNotExists, Map @Override public void dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException { try { - executionAuthenticator.execute(() -> { + executeCatalogOperation(() -> { performDropDb(dbName, ifExists, force); return null; }); @@ -295,7 +284,7 @@ public void dropDbImpl(String dbName, boolean ifExists, boolean force) throws Dd } private void performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { - ExternalDatabase dorisDb = dorisCatalog.getDbNullable(dbName); + ExternalDatabase dorisDb = getDatabaseWithinCatalogGeneration(dbName); if (dorisDb == null) { if (ifExists) { LOG.info("drop database[{}] which does not exist", dbName); @@ -307,7 +296,7 @@ private void performDropDb(String dbName, boolean ifExists, boolean force) throw if (force) { try { // try to drop all tables in the database - List remoteTableNames = listTableNames(dorisDb.getRemoteName()); + List remoteTableNames = listTableNamesInternal(dorisDb.getRemoteName()); for (String remoteTableName : remoteTableNames) { performDropTable(dorisDb.getRemoteName(), remoteTableName, true); } @@ -315,7 +304,7 @@ private void performDropDb(String dbName, boolean ifExists, boolean force) throw LOG.info("drop database[{}] with force, drop all tables, num: {}", dbName, remoteTableNames.size()); } // try to drop all views in the database - List remoteViewNames = listViewNames(dorisDb.getRemoteName()); + List remoteViewNames = listViewNamesInternal(dorisDb.getRemoteName()); for (String remoteViewName : remoteViewNames) { performDropView(dorisDb.getRemoteName(), remoteViewName); } @@ -339,7 +328,7 @@ public void afterDropDb(String dbName) { @Override public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserException { try { - return executionAuthenticator.execute(() -> performCreateTable(createTableInfo)); + return executeCatalogOperation(() -> performCreateTable(createTableInfo)); } catch (Exception e) { throw new DdlException( "Failed to create table: " + createTableInfo.getTableName() + ", error message is:" + e.getMessage(), @@ -347,15 +336,15 @@ public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserExcep } } - public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserException { + private boolean performCreateTable(CreateTableInfo createTableInfo) throws UserException { String dbName = createTableInfo.getDbName(); - ExternalDatabase db = dorisCatalog.getDbNullable(dbName); + ExternalDatabase db = getDatabaseWithinCatalogGeneration(dbName); if (db == null) { throw new UserException("Failed to get database: '" + dbName + "' in catalog: " + dorisCatalog.getName()); } String tableName = createTableInfo.getTableName(); // 1. first, check if table exist in remote - if (tableExist(db.getRemoteName(), tableName)) { + if (tableExistsInternal(db.getRemoteName(), tableName)) { if (createTableInfo.isIfNotExists()) { LOG.info("create table[{}] which already exists", tableName); return true; @@ -393,7 +382,6 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx Schema schema = new Schema(visit.asNestedType().asStructType().fields()); Map properties = createTableInfo.getProperties(); properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE); - Map catalogProperties = dorisCatalog.getProperties(); if (!properties.containsKey(TableProperties.FORMAT_VERSION) && !IcebergUtils.hasIcebergCatalogFormatVersion(catalogProperties)) { properties.put(TableProperties.FORMAT_VERSION, "2"); @@ -437,9 +425,8 @@ public void afterCreateTable(String dbName, String tblName) { @Override public void dropTableImpl(ExternalTable dorisTable, boolean ifExists) throws DdlException { try { - executionAuthenticator.execute(() -> { - if (getExternalCatalog().getMetadataOps() - .viewExists(dorisTable.getRemoteDbName(), dorisTable.getRemoteName())) { + executeCatalogOperation(() -> { + if (viewExistsInternal(dorisTable.getRemoteDbName(), dorisTable.getRemoteName())) { performDropView(dorisTable.getRemoteDbName(), dorisTable.getRemoteName()); } else { performDropTable(dorisTable.getRemoteDbName(), dorisTable.getRemoteName(), ifExists); @@ -463,7 +450,7 @@ public void afterDropTable(String dbName, String tblName) { } private void performDropTable(String remoteDbName, String remoteTblName, boolean ifExists) throws DdlException { - if (!tableExist(remoteDbName, remoteTblName)) { + if (!tableExistsInternal(remoteDbName, remoteTblName)) { if (ifExists) { LOG.info("drop table[{}] which does not exist", remoteTblName); return; @@ -476,7 +463,7 @@ private void performDropTable(String remoteDbName, String remoteTblName, boolean public void renameTableImpl(String dbName, String tblName, String newTblName) throws DdlException { try { - executionAuthenticator.execute(() -> { + executeCatalogOperation(() -> { catalog.renameTable(getTableIdentifier(dbName, tblName), getTableIdentifier(dbName, newTblName)); return null; }); @@ -1620,7 +1607,7 @@ private void validateCommonColumnMetadata(Column column, boolean rejectKey) thro private org.apache.doris.catalog.Type mappedDorisType(org.apache.iceberg.types.Type icebergType) { return IcebergUtils.icebergTypeToDorisType(icebergType, - dorisCatalog.getEnableMappingVarbinary(), dorisCatalog.getEnableMappingTimestampTz()); + enableMappingVarbinary, enableMappingTimestampTz); } private boolean isSameMappedDorisType(org.apache.doris.catalog.Type mappedType, @@ -1845,20 +1832,21 @@ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFiel @Override public Table loadTable(String dbName, String tblName) { try { - return executionAuthenticator.execute(() -> catalog.loadTable(getTableIdentifier(dbName, tblName))); + return executeCatalogOperation(() -> loadTableWithinCatalogGeneration(dbName, tblName)); } catch (Exception e) { throw new RuntimeException("Failed to load table, error message is:" + e.getMessage(), e); } } + /** Loads a table while the caller holds this ops instance's exact catalog-generation guard. */ + public Table loadTableWithinCatalogGeneration(String dbName, String tblName) { + return catalog.loadTable(getTableIdentifier(dbName, tblName)); + } + @Override public boolean viewExists(String remoteDbName, String remoteViewName) { - if (!isViewCatalogEnabled()) { - return false; - } try { - return executionAuthenticator.execute(() -> - ((ViewCatalog) catalog).viewExists(getTableIdentifier(remoteDbName, remoteViewName))); + return executeCatalogOperation(() -> viewExistsInternal(remoteDbName, remoteViewName)); } catch (Exception e) { throw new RuntimeException("Failed to check view exist, error message is:" + e.getMessage(), e); @@ -1867,13 +1855,8 @@ public boolean viewExists(String remoteDbName, String remoteViewName) { @Override public View loadView(String dbName, String tblName) { - if (!isViewCatalogEnabled()) { - return null; - } try { - ViewCatalog viewCatalog = (ViewCatalog) catalog; - return executionAuthenticator.execute( - () -> viewCatalog.loadView(TableIdentifier.of(getNamespace(dbName), tblName))); + return executeCatalogOperation(() -> loadViewInternal(dbName, tblName)); } catch (Exception e) { throw new RuntimeException("Failed to load view, error message is:" + e.getMessage(), e); } @@ -1881,13 +1864,8 @@ public View loadView(String dbName, String tblName) { @Override public List listViewNames(String db) { - if (!isViewCatalogEnabled()) { - return Collections.emptyList(); - } try { - return executionAuthenticator.execute(() -> - ((ViewCatalog) catalog).listViews(getNamespace(db)) - .stream().map(TableIdentifier::name).collect(Collectors.toList())); + return executeCatalogOperation(() -> listViewNamesInternal(db)); } catch (RuntimeException e) { // We want to catch real exception like NoSuchNamespaceException and throw it directly throw e; @@ -1919,7 +1897,7 @@ private Namespace getNamespace() { return externalCatalogName.map(Namespace::of).orElseGet(() -> Namespace.empty()); } - private boolean isViewCatalogEnabled() { + private boolean determineViewCatalogEnabled() { if (!(catalog instanceof ViewCatalog)) { return false; } @@ -1932,10 +1910,96 @@ private boolean isViewCatalogEnabled() { return true; } + private boolean determineNestedNamespaceEnabled() { + if (dorisCatalog instanceof IcebergRestExternalCatalog) { + MetastoreProperties metaProps = dorisCatalog.getCatalogProperty().getMetastoreProperties(); + return metaProps instanceof IcebergRestProperties + && ((IcebergRestProperties) metaProps).isIcebergRestNestedNamespaceEnabled(); + } + return false; + } + + private boolean isViewCatalogEnabled() { + return viewCatalogEnabled; + } + public ThreadPoolExecutor getThreadPoolWithPreAuth() { return threadPoolWithPreAuth; } + public Map loadNamespaceMetadata(String dbName) { + try { + return executeCatalogOperation(() -> nsCatalog.loadNamespaceMetadata(getNamespace(dbName))); + } catch (Exception e) { + throw new RuntimeException("Failed to load namespace metadata, error message is:" + e.getMessage(), e); + } + } + + private T executeCatalogOperation(Callable operation) throws Exception { + if (dorisCatalog instanceof IcebergExternalCatalog) { + try (IcebergCatalogResourceTracker.LoadGuard ignored = + ((IcebergExternalCatalog) dorisCatalog).beginCatalogOperation(this)) { + return executionAuthenticator.execute(operation); + } + } + if (dorisCatalog instanceof HMSExternalCatalog) { + try (IcebergCatalogResourceTracker.LoadGuard ignored = + ((HMSExternalCatalog) dorisCatalog).beginIcebergCatalogOperation(this)) { + return executionAuthenticator.execute(operation); + } + } + return executionAuthenticator.execute(operation); + } + + private ExternalDatabase getDatabaseWithinCatalogGeneration(String dbName) { + if (dorisCatalog instanceof IcebergExternalCatalog) { + return ((IcebergExternalCatalog) dorisCatalog).getDbForCatalogOperation(this, dbName); + } + if (dorisCatalog instanceof HMSExternalCatalog) { + return ((HMSExternalCatalog) dorisCatalog).getDbForIcebergCatalogOperation(this, dbName); + } + throw new IllegalStateException("Unsupported Iceberg catalog type: " + dorisCatalog.getClass().getName()); + } + + private boolean tableExistsInternal(String dbName, String tblName) { + return catalog.tableExists(getTableIdentifier(dbName, tblName)); + } + + private boolean databaseExistsInternal(String dbName) { + return nsCatalog.namespaceExists(getNamespace(dbName)); + } + + private List listTableNamesInternal(String dbName) { + List tableIdentifiers = catalog.listTables(getNamespace(dbName)); + List views = listViewNamesInternal(dbName); + if (views.isEmpty()) { + return tableIdentifiers.stream().map(TableIdentifier::name).collect(Collectors.toList()); + } + return tableIdentifiers.stream() + .map(TableIdentifier::name) + .filter(name -> !views.contains(name)).collect(Collectors.toList()); + } + + private boolean viewExistsInternal(String remoteDbName, String remoteViewName) { + return isViewCatalogEnabled() + && ((ViewCatalog) catalog).viewExists(getTableIdentifier(remoteDbName, remoteViewName)); + } + + private View loadViewInternal(String dbName, String tblName) { + if (!isViewCatalogEnabled()) { + return null; + } + return ((ViewCatalog) catalog).loadView(TableIdentifier.of(getNamespace(dbName), tblName)); + } + + private List listViewNamesInternal(String dbName) { + if (!isViewCatalogEnabled()) { + return Collections.emptyList(); + } + return ((ViewCatalog) catalog).listViews(getNamespace(dbName)) + .stream().map(TableIdentifier::name).collect(Collectors.toList()); + } + private void performDropView(String remoteDbName, String remoteViewName) throws DdlException { if (!isViewCatalogEnabled()) { throw new DdlException("Drop Iceberg view is not supported with not view catalog."); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java index ccb43dc8740853..d57739bd8ddf4d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java @@ -23,6 +23,7 @@ import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; import org.apache.iceberg.CatalogProperties; @@ -45,9 +46,94 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class IcebergMetadataOpTest { + @Test + public void testCatalogOperationDelaysGenerationRetirement() throws Exception { + IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class); + Catalog icebergCatalog = Mockito.mock(Catalog.class, + Mockito.withSettings().extraInterfaces(SupportsNamespaces.class)); + IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); + CountDownLatch operationStarted = new CountDownLatch(1); + CountDownLatch allowOperationToFinish = new CountDownLatch(1); + AtomicInteger cleanupCalls = new AtomicInteger(); + + Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + }); + Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); + Mockito.when(dorisCatalog.beginCatalogOperation(Mockito.any())) + .thenAnswer(invocation -> tracker.beginOperation()); + Mockito.when(icebergCatalog.tableExists(TableIdentifier.of("db", "tbl"))).thenAnswer(invocation -> { + operationStarted.countDown(); + Assert.assertTrue(allowOperationToFinish.await(5, TimeUnit.SECONDS)); + return true; + }); + + IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future operation = executor.submit(() -> ops.tableExist("db", "tbl")); + Assert.assertTrue(operationStarted.await(5, TimeUnit.SECONDS)); + + tracker.retireCurrent(cleanupCalls::incrementAndGet); + Assert.assertEquals(0, cleanupCalls.get()); + + allowOperationToFinish.countDown(); + Assert.assertTrue(operation.get(5, TimeUnit.SECONDS)); + Assert.assertEquals(1, cleanupCalls.get()); + } finally { + allowOperationToFinish.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void testHmsCatalogOperationDelaysGenerationRetirement() throws Exception { + HMSExternalCatalog dorisCatalog = Mockito.mock(HMSExternalCatalog.class); + Catalog icebergCatalog = Mockito.mock(Catalog.class, + Mockito.withSettings().extraInterfaces(SupportsNamespaces.class)); + IcebergCatalogResourceTracker tracker = new IcebergCatalogResourceTracker(); + CountDownLatch operationStarted = new CountDownLatch(1); + CountDownLatch allowOperationToFinish = new CountDownLatch(1); + AtomicInteger cleanupCalls = new AtomicInteger(); + + Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + }); + Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); + Mockito.when(dorisCatalog.beginIcebergCatalogOperation(Mockito.any())) + .thenAnswer(invocation -> tracker.beginOperation()); + Mockito.when(icebergCatalog.tableExists(TableIdentifier.of("db", "tbl"))).thenAnswer(invocation -> { + operationStarted.countDown(); + Assert.assertTrue(allowOperationToFinish.await(5, TimeUnit.SECONDS)); + return true; + }); + + IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future operation = executor.submit(() -> ops.tableExist("db", "tbl")); + Assert.assertTrue(operationStarted.await(5, TimeUnit.SECONDS)); + + tracker.retireCurrent(cleanupCalls::incrementAndGet); + Assert.assertEquals(0, cleanupCalls.get()); + + allowOperationToFinish.countDown(); + Assert.assertTrue(operation.get(5, TimeUnit.SECONDS)); + Assert.assertEquals(1, cleanupCalls.get()); + Mockito.verify(dorisCatalog).beginIcebergCatalogOperation(ops); + } finally { + allowOperationToFinish.countDown(); + executor.shutdownNow(); + } + } + @Test public void testGetNamespaces() { Namespace ns = IcebergMetadataOps.getNamespace(Optional.empty(), "db1"); @@ -129,15 +215,18 @@ public void testListTableNamesFiltersViewsWhenRestViewEnabled() { public void testPerformCreateTableRespectsCatalogDefaultFormatVersion() throws Exception { Map catalogProps = new HashMap<>(); catalogProps.put(CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.FORMAT_VERSION, "3"); + catalogProps.put(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE, IcebergExternalCatalog.ICEBERG_HMS); IcebergExternalCatalog dorisCatalog = mockHmsCatalog(catalogProps); Catalog icebergCatalog = Mockito.mock(Catalog.class, Mockito.withSettings().extraInterfaces(SupportsNamespaces.class)); IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); + Mockito.verify(dorisCatalog, Mockito.never()).getIcebergCatalogType(); + catalogProps.put(CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.FORMAT_VERSION, "1"); ExternalDatabase dorisDb = Mockito.mock(ExternalDatabase.class); Mockito.when(dorisDb.getRemoteName()).thenReturn("db"); Mockito.when(dorisDb.getTableNullable("tbl")).thenReturn(null); - Mockito.doReturn(dorisDb).when(dorisCatalog).getDbNullable("db"); + Mockito.doReturn(dorisDb).when(dorisCatalog).getDbForCatalogOperation(ops, "db"); Mockito.when(dorisCatalog.getName()).thenReturn("iceberg_catalog"); Mockito.when(icebergCatalog.tableExists(TableIdentifier.of("db", "tbl"))).thenReturn(false); @@ -150,15 +239,17 @@ public void testPerformCreateTableRespectsCatalogDefaultFormatVersion() throws E new Column("id", Type.INT, true))); Mockito.when(createTableInfo.getProperties()).thenReturn(tableProps); - ops.performCreateTable(createTableInfo); + ops.createTableImpl(createTableInfo); + Mockito.verify(dorisCatalog, Mockito.never()).getDbNullable(Mockito.anyString()); Mockito.verify(createTableInfo).validateIcebergRowLineageColumns(3); ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); Mockito.verify(icebergCatalog).createTable(Mockito.eq(TableIdentifier.of("db", "tbl")), Mockito.any(Schema.class), Mockito.any(PartitionSpec.class), propsCaptor.capture()); Assert.assertFalse(propsCaptor.getValue().containsKey(TableProperties.FORMAT_VERSION)); Assert.assertEquals(3, IcebergUtils.getEffectiveIcebergFormatVersion( - propsCaptor.getValue(), catalogProps)); + propsCaptor.getValue(), Collections.singletonMap( + CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.FORMAT_VERSION, "3"))); } @Test @@ -174,8 +265,8 @@ public void testCreateDatabaseWithPropertiesForSupportedCatalogs() throws Except SupportsNamespaces namespaceCatalog = (SupportsNamespaces) icebergCatalog; IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class); Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); - Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); - Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(catalogType); + Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.singletonMap( + IcebergExternalCatalog.ICEBERG_CATALOG_TYPE, catalogType)); Mockito.when(namespaceCatalog.namespaceExists(Namespace.of(dbName))).thenReturn(false); IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); Map properties = Collections.singletonMap("owner", "doris"); @@ -198,8 +289,8 @@ public void testCreateDatabaseWithLocationForSupportedCatalogs() throws Exceptio SupportsNamespaces namespaceCatalog = (SupportsNamespaces) icebergCatalog; IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class); Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); - Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); - Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(catalogType); + Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.singletonMap( + IcebergExternalCatalog.ICEBERG_CATALOG_TYPE, catalogType)); Mockito.when(namespaceCatalog.namespaceExists(Namespace.of(dbName))).thenReturn(false); IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); Map properties = Collections.singletonMap( @@ -219,8 +310,8 @@ public void testCreateDatabaseWithLocationForJdbcCatalogIsRejected() { SupportsNamespaces namespaceCatalog = (SupportsNamespaces) icebergCatalog; IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class); Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); - Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); - Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_JDBC); + Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.singletonMap( + IcebergExternalCatalog.ICEBERG_CATALOG_TYPE, IcebergExternalCatalog.ICEBERG_JDBC)); Mockito.when(namespaceCatalog.namespaceExists(Namespace.of(dbName))).thenReturn(false); IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); Map properties = Collections.singletonMap( @@ -250,8 +341,8 @@ public void testCreateDatabaseWithPropertiesForUnsupportedCatalogs() { SupportsNamespaces namespaceCatalog = (SupportsNamespaces) icebergCatalog; IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class); Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); - Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); - Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(catalogType); + Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.singletonMap( + IcebergExternalCatalog.ICEBERG_CATALOG_TYPE, catalogType)); Mockito.when(namespaceCatalog.namespaceExists(Namespace.of(dbName))).thenReturn(false); IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); Map properties = Collections.singletonMap("owner", "doris"); @@ -271,7 +362,6 @@ private IcebergExternalCatalog mockHmsCatalog(Map catalogPropert Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); Mockito.when(dorisCatalog.getProperties()).thenReturn(catalogProperties); - Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_HMS); Mockito.when(dorisCatalog.getCatalogProperty()).thenReturn(new CatalogProperty(null, Collections.emptyMap())); return dorisCatalog; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index a0c998d5faed22..115b019f1f010c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -664,6 +664,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + ops = new IcebergMetadataOps(dorisCatalog, ops.getCatalog()); try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { @@ -694,6 +695,7 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + ops = new IcebergMetadataOps(dorisCatalog, ops.getCatalog()); try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { @@ -722,6 +724,7 @@ public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdate Mockito.when(icebergTable.schema()).thenReturn(schema); Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + ops = new IcebergMetadataOps(dorisCatalog, ops.getCatalog()); try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index bb5c4ec419ad35..941f30c4dd7019 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -129,8 +129,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; From 2088184c4901c6b4afd31335eb8a1692e5c07166 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 31 Aug 2026 12:30:39 +0800 Subject: [PATCH 30/38] [fix](fe) Complete Iceberg and Hudi generation fencing ### What problem does this PR solve? Issue Number: close #66912 Related PR: #66913 Problem Summary: Iceberg create-table tests constructed a second metadata-operations object outside the catalog-owned runtime and were correctly rejected by the exact-generation guard. Hudi scan planning could also resolve the current filesystem-view cache after capturing an older catalog runtime, run filesystem-view synchronization outside the captured authenticator, or deadlock catalog reset if a cold load and remote sync were kept under the cache lifecycle monitor. In addition, mixed-case Iceberg catalog type values were frozen without normalization and later rejected by lowercase-only database validation. Use the catalog-owned Iceberg metadata operations in tests and normalize the frozen Iceberg catalog type. Capture the exact Hudi fs-view cache generation and authenticator with the HMS runtime, load outside lifecycle monitors, atomically validate and pin the exact view, and run remote synchronization under the captured authenticator after the pin. A reset that wins during a cold load retires the unpublished view and cannot redirect the scan to a replacement generation. ### Release note Fix Iceberg and Hudi metadata resource handling across catalog reset and property changes. ### Check List (For Author) - Test: Unit Test - CreateIcebergTableTest - IcebergMetadataOpTest - HudiExternalMetaCacheTest - HudiFsViewCacheValueTest - HudiScanNodeTest - ./build.sh --fe - FE Checkstyle - Behavior changed: Yes. Hudi filesystem-view access remains bound to the captured catalog generation and authentication context; mixed-case Iceberg catalog types are normalized. - Does this need documentation: No --- .../datasource/hive/HMSExternalCatalog.java | 33 +++++ .../hudi/HudiExternalMetaCache.java | 65 ++++++++- .../datasource/hudi/HudiFsViewCacheValue.java | 15 +-- .../datasource/hudi/source/HudiScanNode.java | 20 ++- .../iceberg/IcebergMetadataOps.java | 4 +- .../hudi/HudiExternalMetaCacheTest.java | 125 ++++++++++++++++++ .../hudi/HudiFsViewCacheValueTest.java | 42 +----- .../iceberg/CreateIcebergTableTest.java | 2 +- .../iceberg/IcebergMetadataOpTest.java | 3 + 9 files changed, 242 insertions(+), 67 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 0f14d887a60df5..09dd26a7df44e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -95,6 +95,14 @@ public synchronized long getRuntimeGeneration() { return runtimeGeneration.get(); } + /** Captures the authentication context and generation used by one Hudi scan. */ + public synchronized HudiScanRuntimeContext getHudiScanRuntimeContext() { + makeSureInitialized(); + HudiExternalMetaCache hudiCache = Env.getCurrentEnv().getExtMetaCacheMgr().hudi(getId()); + return new HudiScanRuntimeContext(runtimeGeneration.get(), executionAuthenticator, + hudiCache.captureFsViewGeneration(getId())); + } + @Override public synchronized void modifyCatalogProps(Map props) { // Fence scans before the mutable CatalogProperty is changed. super invokes resetToUninitialized while @@ -420,4 +428,29 @@ public void close() { guard.close(); } } + + public static final class HudiScanRuntimeContext { + private final long generation; + private final ExecutionAuthenticator authenticator; + private final HudiExternalMetaCache.FsViewGeneration fsViewGeneration; + + private HudiScanRuntimeContext(long generation, ExecutionAuthenticator authenticator, + HudiExternalMetaCache.FsViewGeneration fsViewGeneration) { + this.generation = generation; + this.authenticator = authenticator; + this.fsViewGeneration = fsViewGeneration; + } + + public long getGeneration() { + return generation; + } + + public ExecutionAuthenticator getAuthenticator() { + return authenticator; + } + + public HudiExternalMetaCache.FsViewGeneration getFsViewGeneration() { + return fsViewGeneration; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index 865da42299c0d5..b88ac86d6b4dbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.hudi; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.ExternalCatalog; @@ -29,6 +30,7 @@ import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -111,17 +113,70 @@ public HoodieTableMetaClient getHoodieTableMetaClient(NameMapping nameMapping) { return metaClientEntry.get(nameMapping.getCtlId()).get(HudiMetaClientCacheKey.of(nameMapping)); } - public HudiFsViewCacheValue.Lease getFsView(NameMapping nameMapping) { + public synchronized FsViewGeneration captureFsViewGeneration(long catalogId) { + return new FsViewGeneration(catalogId, fsViewEntry.get(catalogId)); + } + + private HudiFsViewCacheValue.Lease getFsView( + FsViewGeneration generation, NameMapping nameMapping, ExecutionAuthenticator authenticator) { HudiFsViewCacheKey key = HudiFsViewCacheKey.of(nameMapping); while (true) { - HudiFsViewCacheValue value = fsViewEntry.get(nameMapping.getCtlId()).get(key); - HudiFsViewCacheValue.Lease lease = value.tryAcquire(); - if (lease != null) { + synchronized (this) { + if (fsViewEntry.getIfInitialized(generation.catalogId) != generation.entry) { + throw new IllegalStateException( + "Hudi catalog runtime changed before filesystem-view acquisition"); + } + } + HudiFsViewCacheValue value = generation.entry.get(key); + HudiFsViewCacheValue.Lease lease; + boolean staleGeneration; + synchronized (this) { + staleGeneration = fsViewEntry.getIfInitialized(generation.catalogId) != generation.entry; + lease = staleGeneration ? null : value.tryAcquire(); + } + if (staleGeneration) { + value.evict(); + throw new IllegalStateException( + "Hudi catalog runtime changed before filesystem-view acquisition"); + } + if (lease == null) { + continue; + } + try { + // The lease pins the exact view after the cache-generation handoff, so reset may + // retire the entry while remote timeline I/O runs outside every lifecycle monitor. + authenticator.execute(() -> { + lease.get().sync(); + return null; + }); return lease; + } catch (Exception e) { + lease.close(); + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException("Failed to synchronize Hudi filesystem view", e); } } } + /** A reference to one exact catalog cache generation. */ + public final class FsViewGeneration { + private final long catalogId; + private final MetaCacheEntry entry; + + private FsViewGeneration(long catalogId, + MetaCacheEntry entry) { + this.catalogId = catalogId; + this.entry = entry; + } + + public HudiFsViewCacheValue.Lease getFsView( + NameMapping nameMapping, ExecutionAuthenticator authenticator) { + return HudiExternalMetaCache.this.getFsView(this, nameMapping, authenticator); + } + } + public HudiSchemaCacheValue getHudiSchemaCacheValue(NameMapping nameMapping, long timestamp) { SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()) .get(new HudiSchemaCacheKey(nameMapping, timestamp)); @@ -153,7 +208,7 @@ public TablePartitionValues getPartitionValues(HMSExternalTable table, boolean u HudiPartitionCacheKey.of(table.getOrBuildNameMapping(), lastTimestamp, useHiveSyncPartition)); } - private HudiFsViewCacheValue createFsView(HudiFsViewCacheKey key) { + protected HudiFsViewCacheValue createFsView(HudiFsViewCacheKey key) { HoodieTableMetaClient tableMetaClient = metaClientEntry.get(key.getNameMapping().getCtlId()) .get(HudiMetaClientCacheKey.of(key.getNameMapping())); HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java index 94048a4dff4d93..652ca0cdc0395c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValue.java @@ -39,28 +39,17 @@ public HudiFsViewCacheValue(HoodieTableFileSystemView fsView) { } public Lease tryAcquire() { - Lease lease; synchronized (this) { if (loaderReferenceAvailable) { loaderReferenceAvailable = false; - lease = new Lease(this, fsView); + return new Lease(this, fsView); } else if (evicted) { return null; } else { refCount++; - lease = new Lease(this, fsView); + return new Lease(this, fsView); } } - try { - // The cache uses expire-after-access without detached refresh. Sync every foreground generation handoff - // so a continuously hot key still observes newly completed commits. The exact lease keeps the view alive; - // do not hold the generation-owner monitor across remote timeline I/O. - fsView.sync(); - return lease; - } catch (RuntimeException e) { - lease.close(); - throw e; - } } public synchronized void evict() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 1e6b522ebeb032..8fbe16fb9c436e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.common.util.FileFormatUtils; import org.apache.doris.common.util.LocationPath; @@ -40,6 +41,7 @@ import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hive.source.HiveScanNode; +import org.apache.doris.datasource.hudi.HudiExternalMetaCache; import org.apache.doris.datasource.hudi.HudiFsViewCacheValue; import org.apache.doris.datasource.hudi.HudiPartitionUtils; import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; @@ -122,6 +124,8 @@ public class HudiScanNode extends HiveScanNode { private List partitionColumnNames; private String storagePropertiesFingerprint; private final long hmsRuntimeGeneration; + private final ExecutionAuthenticator executionAuthenticator; + private final HudiExternalMetaCache.FsViewGeneration fsViewGeneration; private boolean partitionInit = false; private HoodieTimeline timeline; @@ -157,7 +161,11 @@ public HudiScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumn SessionVariable sv, DirectoryLister directoryLister, ScanContext scanContext) { super(id, desc, "HUDI_SCAN_NODE", StatisticalType.HUDI_SCAN_NODE, needCheckColumnPriv, sv, directoryLister, scanContext); - hmsRuntimeGeneration = ((HMSExternalCatalog) hmsTable.getCatalog()).getRuntimeGeneration(); + HMSExternalCatalog.HudiScanRuntimeContext runtimeContext = + ((HMSExternalCatalog) hmsTable.getCatalog()).getHudiScanRuntimeContext(); + hmsRuntimeGeneration = runtimeContext.getGeneration(); + executionAuthenticator = runtimeContext.getAuthenticator(); + fsViewGeneration = runtimeContext.getFsViewGeneration(); isCowTable = hmsTable.isHoodieCowTable(); if (LOG.isDebugEnabled()) { if (isCowTable) { @@ -390,10 +398,8 @@ private synchronized void acquireFsView() { if (fsViewReleased.get()) { throw new IllegalStateException("Hudi filesystem-view lease has already been released"); } - fsViewLease = Env.getCurrentEnv() - .getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()) - .getFsView(hmsTable.getOrBuildNameMapping()); + fsViewLease = fsViewGeneration.getFsView( + hmsTable.getOrBuildNameMapping(), executionAuthenticator); fsView = fsViewLease.get(); } @@ -605,7 +611,7 @@ public List getSplits(int numBackends) throws UserException { } acquireFsView(); List splits = Collections.synchronizedList(new ArrayList<>()); - hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> { + executionAuthenticator.execute(() -> { getPartitionsSplits(prunedPartitions, splits); return null; }); @@ -625,7 +631,7 @@ private void initPrunedPartitions() throws UserException { } long startTime = System.currentTimeMillis(); try { - prunedPartitions = hmsTable.getCatalog().getExecutionAuthenticator().execute(() + prunedPartitions = executionAuthenticator.execute(() -> getPrunedPartitions(hudiClient)); if (getSummaryProfile() != null) { getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index e67753083be50d..d065e7283ea62d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -90,6 +90,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -129,7 +130,8 @@ public IcebergMetadataOps(ExternalCatalog dorisCatalog, Catalog catalog) { this.executionAuthenticator = dorisCatalog.getExecutionAuthenticator(); this.threadPoolWithPreAuth = dorisCatalog.getThreadPoolWithPreAuth(); this.catalogProperties = Collections.unmodifiableMap(new HashMap<>(dorisCatalog.getProperties())); - this.icebergCatalogType = catalogProperties.get(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE); + String catalogType = catalogProperties.get(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE); + this.icebergCatalogType = catalogType == null ? null : catalogType.toLowerCase(Locale.ROOT); this.enableMappingVarbinary = dorisCatalog.getEnableMappingVarbinary(); this.enableMappingTimestampTz = dorisCatalog.getEnableMappingTimestampTz(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiExternalMetaCacheTest.java index 3932294033d23f..2b5801ad3befa2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiExternalMetaCacheTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.hudi; import org.apache.doris.common.Config; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; @@ -26,16 +27,26 @@ import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; public class HudiExternalMetaCacheTest { + private static final ExecutionAuthenticator AUTHENTICATOR = new ExecutionAuthenticator() { }; + @Test public void testEntryAccessAfterExplicitInit() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -174,6 +185,120 @@ public void testDefaultSpecsFollowConfig() { } } + @Test + public void testFsViewGenerationDoesNotCrossCatalogReset() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + HudiExternalMetaCache cache = new HudiExternalMetaCache(executor); + long catalogId = 1L; + NameMapping nameMapping = nameMapping(catalogId, "db1", "tbl1"); + HudiFsViewCacheKey key = HudiFsViewCacheKey.of(nameMapping); + + cache.initCatalog(catalogId, Collections.emptyMap()); + HoodieTableFileSystemView oldView = Mockito.mock(HoodieTableFileSystemView.class); + cache.entry(catalogId, HudiExternalMetaCache.ENTRY_FS_VIEW, + HudiFsViewCacheKey.class, HudiFsViewCacheValue.class) + .put(key, new HudiFsViewCacheValue(oldView)); + HudiExternalMetaCache.FsViewGeneration oldGeneration = + cache.captureFsViewGeneration(catalogId); + + cache.invalidateCatalog(catalogId); + cache.initCatalog(catalogId, Collections.emptyMap()); + HoodieTableFileSystemView newView = Mockito.mock(HoodieTableFileSystemView.class); + cache.entry(catalogId, HudiExternalMetaCache.ENTRY_FS_VIEW, + HudiFsViewCacheKey.class, HudiFsViewCacheValue.class) + .put(key, new HudiFsViewCacheValue(newView)); + + try { + oldGeneration.getFsView(nameMapping, AUTHENTICATOR); + Assert.fail("stale generation must not acquire the replacement cache entry"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("runtime changed")); + } + Mockito.verify(oldView).close(); + Mockito.verify(oldView, Mockito.never()).sync(); + Mockito.verify(newView, Mockito.never()).sync(); + + HudiExternalMetaCache.FsViewGeneration newGeneration = + cache.captureFsViewGeneration(catalogId); + AtomicBoolean authenticated = new AtomicBoolean(); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + authenticated.set(true); + try { + return task.call(); + } finally { + authenticated.set(false); + } + } + }; + Mockito.doAnswer(invocation -> { + Assert.assertTrue(authenticated.get()); + return null; + }).when(newView).sync(); + try (HudiFsViewCacheValue.Lease lease = newGeneration.getFsView(nameMapping, authenticator)) { + Assert.assertSame(newView, lease.get()); + } + Assert.assertFalse(authenticated.get()); + Mockito.verify(newView).sync(); + cache.invalidateCatalog(catalogId); + Mockito.verify(newView).close(); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testColdLoadDoesNotBlockCatalogRetirement() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newFixedThreadPool(2); + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch allowLoad = new CountDownLatch(1); + HoodieTableFileSystemView view = Mockito.mock(HoodieTableFileSystemView.class); + try { + HudiExternalMetaCache cache = new HudiExternalMetaCache(refreshExecutor) { + @Override + protected HudiFsViewCacheValue createFsView(HudiFsViewCacheKey key) { + loadStarted.countDown(); + try { + Assert.assertTrue(allowLoad.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + return new HudiFsViewCacheValue(view); + } + }; + long catalogId = 1L; + NameMapping nameMapping = nameMapping(catalogId, "db1", "tbl1"); + cache.initCatalog(catalogId, Collections.emptyMap()); + HudiExternalMetaCache.FsViewGeneration generation = + cache.captureFsViewGeneration(catalogId); + + Future acquisition = workers.submit( + () -> generation.getFsView(nameMapping, AUTHENTICATOR)); + Assert.assertTrue(loadStarted.await(3L, TimeUnit.SECONDS)); + + Future retirement = workers.submit(() -> cache.invalidateCatalog(catalogId)); + retirement.get(3L, TimeUnit.SECONDS); + allowLoad.countDown(); + + try { + acquisition.get(3L, TimeUnit.SECONDS); + Assert.fail("load from the retired generation must not be handed to the scan"); + } catch (ExecutionException expected) { + Assert.assertTrue(expected.getCause() instanceof IllegalStateException); + } + Mockito.verify(view).close(); + Mockito.verify(view, Mockito.never()).sync(); + } finally { + allowLoad.countDown(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + private NameMapping nameMapping(long catalogId, String dbName, String tableName) { return new NameMapping(catalogId, dbName, tableName, "remote_" + dbName, "remote_" + tableName); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java index af2a10d3b3d0ad..368f16a53f2984 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiFsViewCacheValueTest.java @@ -22,12 +22,6 @@ import org.junit.Test; import org.mockito.Mockito; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - public class HudiFsViewCacheValueTest { @Test @@ -38,7 +32,6 @@ public void testEvictionClosesAfterExactLeaseRelease() { Assert.assertNotNull(lease); Assert.assertSame(view, lease.get()); - Mockito.verify(view).sync(); value.evict(); Mockito.verify(view, Mockito.never()).close(); Assert.assertNull(value.tryAcquire()); @@ -61,7 +54,7 @@ public void testEvictionBeforeLoaderReferenceHandoff() { } @Test - public void testLeaseSynchronizesHotCachedView() { + public void testRepeatedLeaseAcquisition() { HoodieTableFileSystemView view = Mockito.mock(HoodieTableFileSystemView.class); HudiFsViewCacheValue value = new HudiFsViewCacheValue(view); HudiFsViewCacheValue.Lease firstLease = value.tryAcquire(); @@ -72,37 +65,6 @@ public void testLeaseSynchronizesHotCachedView() { Assert.assertNotNull(secondLease); secondLease.close(); - Mockito.verify(view, Mockito.times(2)).sync(); - } - - @Test - public void testEvictionDoesNotWaitForBlockedSync() throws Exception { - HoodieTableFileSystemView view = Mockito.mock(HoodieTableFileSystemView.class); - HudiFsViewCacheValue value = new HudiFsViewCacheValue(view); - CountDownLatch syncStarted = new CountDownLatch(1); - CountDownLatch allowSync = new CountDownLatch(1); - Mockito.doAnswer(invocation -> { - syncStarted.countDown(); - Assert.assertTrue(allowSync.await(3L, TimeUnit.SECONDS)); - return null; - }).when(view).sync(); - ExecutorService executor = Executors.newFixedThreadPool(2); - try { - Future acquisition = executor.submit(value::tryAcquire); - Assert.assertTrue(syncStarted.await(3L, TimeUnit.SECONDS)); - - Future eviction = executor.submit(value::evict); - eviction.get(3L, TimeUnit.SECONDS); - Mockito.verify(view, Mockito.never()).close(); - - allowSync.countDown(); - HudiFsViewCacheValue.Lease lease = acquisition.get(3L, TimeUnit.SECONDS); - Mockito.verify(view, Mockito.never()).close(); - lease.close(); - Mockito.verify(view).close(); - } finally { - allowSync.countDown(); - executor.shutdownNow(); - } + Mockito.verify(view, Mockito.never()).sync(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 6962f911538f81..6ca1adbbb137d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -71,7 +71,7 @@ public static void beforeClass() throws Throwable { icebergCatalog = (IcebergHadoopExternalCatalog) CatalogFactory.createFromCommand(1, createCatalogCommand); icebergCatalog.makeSureInitialized(); // create db - ops = new IcebergMetadataOps(icebergCatalog, icebergCatalog.getCatalog()); + ops = (IcebergMetadataOps) icebergCatalog.getMetadataOps(); ops.createDb(dbName, true, Maps.newHashMap()); icebergCatalog.makeSureInitialized(); IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java index d57739bd8ddf4d..f15e5169ffc6f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java @@ -44,6 +44,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; @@ -256,6 +257,7 @@ public void testPerformCreateTableRespectsCatalogDefaultFormatVersion() throws E public void testCreateDatabaseWithPropertiesForSupportedCatalogs() throws Exception { List supportedCatalogTypes = Arrays.asList( IcebergExternalCatalog.ICEBERG_HMS, + IcebergExternalCatalog.ICEBERG_HMS.toUpperCase(Locale.ROOT), IcebergExternalCatalog.ICEBERG_JDBC, IcebergExternalCatalog.ICEBERG_GLUE); for (String catalogType : supportedCatalogTypes) { @@ -281,6 +283,7 @@ public void testCreateDatabaseWithPropertiesForSupportedCatalogs() throws Except public void testCreateDatabaseWithLocationForSupportedCatalogs() throws Exception { List supportedCatalogTypes = Arrays.asList( IcebergExternalCatalog.ICEBERG_HMS, + IcebergExternalCatalog.ICEBERG_HMS.toUpperCase(Locale.ROOT), IcebergExternalCatalog.ICEBERG_GLUE); for (String catalogType : supportedCatalogTypes) { String dbName = catalogType + "_location_db"; From fac7113c831ce119b2424395ad26977e23b5a1d6 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 31 Aug 2026 19:55:50 +0800 Subject: [PATCH 31/38] [fix](fe) Retain Iceberg mutations and fence Hudi schema publication ### What problem does this PR solve? Issue Number: close #66913 Related PR: #66913 Problem Summary: Iceberg metadata mutations could release the writable catalog generation before branch, tag, schema, or partition commits completed, allowing a concurrent catalog reset to retire resources still used by the commit. Hudi native-reader schema resolution could similarly cross an HMS runtime reset and publish schema information from the retired generation. Hold the exact writable Iceberg table lease and its captured authenticator through every metadata commit, and revalidate the HMS runtime generation after Hudi schema resolution before publishing schema history or range descriptors. ### Release note Fix Iceberg and Hudi external catalog resource lifecycle races during metadata mutation and scan planning. ### Check List (For Author) - Test: Unit Test - IcebergMetadataOpsValidationTest - IcebergExternalTableBranchAndTagTest - HudiScanNodeTest - Full FE build - Behavior changed: No - Does this need documentation: No --- .../datasource/hudi/source/HudiScanNode.java | 19 +- .../iceberg/IcebergMetadataOps.java | 885 ++++++++++-------- .../hudi/source/HudiScanNodeTest.java | 48 + .../IcebergExternalTableBranchAndTagTest.java | 7 + .../IcebergMetadataOpsValidationTest.java | 123 ++- 5 files changed, 622 insertions(+), 460 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 8fbe16fb9c436e..d75086059f1b60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -330,6 +330,7 @@ private void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) { TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); tableFormatFileDesc.setTableFormatType(hudiSplit.getTableFormatType().value()); THudiFileDesc fileDesc = new THudiFileDesc(); + InternalSchema internalSchema = null; if (rangeDesc.getFormatType() == TFileFormatType.FORMAT_JNI) { fileDesc.setInstantTime(hudiSplit.getInstantTime()); fileDesc.setSerde(hudiSplit.getSerde()); @@ -347,23 +348,17 @@ private void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) { if (hudiSchemaCacheValue.isEnableSchemaEvolution()) { long commitInstantTime = Long.parseLong(FSUtils.getCommitTime( new File(hudiSplit.getPath().getNormalizedLocation()).getName())); - InternalSchema internalSchema = hudiSchemaCacheValue + internalSchema = hudiSchemaCacheValue .getCommitInstantInternalSchema(hudiClient, commitInstantTime); - putHistorySchemaInfo(internalSchema); //for schema change. (native reader) - fileDesc.setSchemaId(internalSchema.schemaId()); } else { try { TableSchemaResolver schemaUtil = new TableSchemaResolver(hudiClient); - InternalSchema internalSchema = - AvroInternalSchemaConverter.convert(schemaUtil.getTableAvroSchema(true)); - putHistorySchemaInfo(internalSchema); //Handle column name case for BE - fileDesc.setSchemaId(internalSchema.schemaId()); + internalSchema = AvroInternalSchemaConverter.convert(schemaUtil.getTableAvroSchema(true)); } catch (Exception e) { throw new RuntimeException("Cannot get hudi table schema.", e); } } } - tableFormatFileDesc.setHudiParams(fileDesc); Map partitionValues = hudiSplit.getHudiPartitionValues(); if (partitionValues != null) { List formPathKeys = new ArrayList<>(); @@ -378,6 +373,14 @@ private void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) { rangeDesc.setColumnsFromPath(parsedColumnsFromPath.getValues()); rangeDesc.setColumnsFromPathIsNull(parsedColumnsFromPath.getIsNull()); } + // Schema resolution may access the remote timeline after the initial scan-generation check. + // Reject the stale result before it is copied into shared query parameters or the range descriptor. + ensureHmsRuntimeGeneration(); + if (internalSchema != null) { + putHistorySchemaInfo(internalSchema); + fileDesc.setSchemaId(internalSchema.schemaId()); + } + tableFormatFileDesc.setHudiParams(fileDesc); rangeDesc.setTableFormatParams(tableFormatFileDesc); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index d065e7283ea62d..e5995bc176c00e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -495,75 +495,79 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) @Override public void createOrReplaceBranchImpl(ExternalTable dorisTable, CreateOrReplaceBranchInfo branchInfo) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - BranchOptions branchOptions = branchInfo.getBranchOptions(); - - Long snapshotId = branchOptions.getSnapshotId() - .orElse( - // use current snapshot - Optional.ofNullable(icebergTable.currentSnapshot()).map(Snapshot::snapshotId).orElse(null)); - - ManageSnapshots manageSnapshots; - try { - manageSnapshots = executionAuthenticator.execute(icebergTable::manageSnapshots); - } catch (Exception e) { - throw new RuntimeException( - "Failed to create ManageSnapshots for table: " + icebergTable.name() - + ", error message is: {} " + ExceptionUtils.getRootCauseMessage(e), e); - } - String branchName = branchInfo.getBranchName(); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + ExecutionAuthenticator authenticator = lease.getAuthenticator(); + BranchOptions branchOptions = branchInfo.getBranchOptions(); - if (branchName == null || branchName.trim().isEmpty()) { - throw new UserException("Branch name cannot be empty"); - } + Long snapshotId = branchOptions.getSnapshotId() + .orElse( + // use current snapshot + Optional.ofNullable(icebergTable.currentSnapshot()).map(Snapshot::snapshotId).orElse(null)); - boolean refExists = null != icebergTable.refs().get(branchName); - boolean create = branchInfo.getCreate(); - boolean replace = branchInfo.getReplace(); - boolean ifNotExists = branchInfo.getIfNotExists(); - Runnable safeCreateBranch = () -> { + ManageSnapshots manageSnapshots; try { - executionAuthenticator.execute(() -> { - if (snapshotId == null) { - manageSnapshots.createBranch(branchName); - } else { - manageSnapshots.createBranch(branchName, snapshotId); - } - }); + manageSnapshots = authenticator.execute(icebergTable::manageSnapshots); } catch (Exception e) { throw new RuntimeException( - "Failed to create branch: " + branchName + " in table: " + icebergTable.name() + "Failed to create ManageSnapshots for table: " + icebergTable.name() + ", error message is: {} " + ExceptionUtils.getRootCauseMessage(e), e); } - }; + String branchName = branchInfo.getBranchName(); - if (create && replace && !refExists) { - safeCreateBranch.run(); - } else if (replace) { - if (snapshotId == null) { - // Cannot perform a replace operation on an empty table - throw new UserException( - "Cannot complete replace branch operation on " + icebergTable.name() - + " , main has no snapshot"); + if (branchName == null || branchName.trim().isEmpty()) { + throw new UserException("Branch name cannot be empty"); } - manageSnapshots.replaceBranch(branchName, snapshotId); - } else { - if (refExists && ifNotExists) { - return; + + boolean refExists = null != icebergTable.refs().get(branchName); + boolean create = branchInfo.getCreate(); + boolean replace = branchInfo.getReplace(); + boolean ifNotExists = branchInfo.getIfNotExists(); + Runnable safeCreateBranch = () -> { + try { + authenticator.execute(() -> { + if (snapshotId == null) { + manageSnapshots.createBranch(branchName); + } else { + manageSnapshots.createBranch(branchName, snapshotId); + } + }); + } catch (Exception e) { + throw new RuntimeException( + "Failed to create branch: " + branchName + " in table: " + icebergTable.name() + + ", error message is: {} " + ExceptionUtils.getRootCauseMessage(e), e); + } + }; + + if (create && replace && !refExists) { + safeCreateBranch.run(); + } else if (replace) { + if (snapshotId == null) { + // Cannot perform a replace operation on an empty table + throw new UserException( + "Cannot complete replace branch operation on " + icebergTable.name() + + " , main has no snapshot"); + } + manageSnapshots.replaceBranch(branchName, snapshotId); + } else { + if (refExists && ifNotExists) { + return; + } + safeCreateBranch.run(); } - safeCreateBranch.run(); - } - branchOptions.getRetain().ifPresent(n -> manageSnapshots.setMaxSnapshotAgeMs(branchName, n)); - branchOptions.getNumSnapshots().ifPresent(n -> manageSnapshots.setMinSnapshotsToKeep(branchName, n)); - branchOptions.getRetention().ifPresent(n -> manageSnapshots.setMaxRefAgeMs(branchName, n)); + branchOptions.getRetain().ifPresent(n -> manageSnapshots.setMaxSnapshotAgeMs(branchName, n)); + branchOptions.getNumSnapshots().ifPresent(n -> manageSnapshots.setMinSnapshotsToKeep(branchName, n)); + branchOptions.getRetention().ifPresent(n -> manageSnapshots.setMaxRefAgeMs(branchName, n)); - try { - executionAuthenticator.execute(manageSnapshots::commit); - } catch (Exception e) { - throw new RuntimeException( - "Failed to create or replace branch: " + branchName + " in table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); + try { + authenticator.execute(manageSnapshots::commit); + } catch (Exception e) { + throw new RuntimeException( + "Failed to create or replace branch: " + branchName + " in table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } } } @@ -583,51 +587,56 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { @Override public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagInfo tagInfo) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - TagOptions tagOptions = tagInfo.getTagOptions(); - Long snapshotId = tagOptions.getSnapshotId() - .orElse( - // use current snapshot - Optional.ofNullable(icebergTable.currentSnapshot()).map(Snapshot::snapshotId).orElse(null)); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + ExecutionAuthenticator authenticator = lease.getAuthenticator(); + TagOptions tagOptions = tagInfo.getTagOptions(); + Long snapshotId = tagOptions.getSnapshotId() + .orElse( + // use current snapshot + Optional.ofNullable(icebergTable.currentSnapshot()).map(Snapshot::snapshotId).orElse(null)); - if (snapshotId == null) { - // Creating tag for empty tables is not allowed - throw new UserException( - "Cannot complete replace branch operation on " + icebergTable.name() + " , main has no snapshot"); - } + if (snapshotId == null) { + // Creating tag for empty tables is not allowed + throw new UserException( + "Cannot complete replace branch operation on " + icebergTable.name() + + " , main has no snapshot"); + } - String tagName = tagInfo.getTagName(); + String tagName = tagInfo.getTagName(); - if (tagName == null || tagName.trim().isEmpty()) { - throw new UserException("Tag name cannot be empty"); - } + if (tagName == null || tagName.trim().isEmpty()) { + throw new UserException("Tag name cannot be empty"); + } - boolean create = tagInfo.getCreate(); - boolean replace = tagInfo.getReplace(); - boolean ifNotExists = tagInfo.getIfNotExists(); - boolean refExists = null != icebergTable.refs().get(tagName); + boolean create = tagInfo.getCreate(); + boolean replace = tagInfo.getReplace(); + boolean ifNotExists = tagInfo.getIfNotExists(); + boolean refExists = null != icebergTable.refs().get(tagName); - try { - executionAuthenticator.execute(() -> { - ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); - if (create && replace && !refExists) { - manageSnapshots.createTag(tagName, snapshotId); - } else if (replace) { - manageSnapshots.replaceTag(tagName, snapshotId); - } else { - if (refExists && ifNotExists) { - return; + try { + authenticator.execute(() -> { + ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); + if (create && replace && !refExists) { + manageSnapshots.createTag(tagName, snapshotId); + } else if (replace) { + manageSnapshots.replaceTag(tagName, snapshotId); + } else { + if (refExists && ifNotExists) { + return; + } + manageSnapshots.createTag(tagName, snapshotId); } - manageSnapshots.createTag(tagName, snapshotId); - } - tagOptions.getRetain().ifPresent(n -> manageSnapshots.setMaxRefAgeMs(tagName, n)); - manageSnapshots.commit(); - }); - } catch (Exception e) { - throw new RuntimeException( - "Failed to create or replace tag: " + tagName + " in table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); + tagOptions.getRetain().ifPresent(n -> manageSnapshots.setMaxRefAgeMs(tagName, n)); + manageSnapshots.commit(); + }); + } catch (Exception e) { + throw new RuntimeException( + "Failed to create or replace tag: " + tagName + " in table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } } } @@ -635,19 +644,22 @@ public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagI public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws UserException { String tagName = tagInfo.getTagName(); boolean ifExists = tagInfo.getIfExists(); - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - SnapshotRef snapshotRef = icebergTable.refs().get(tagName); - - if (snapshotRef != null || !ifExists) { - try { - executionAuthenticator.execute(() -> { - ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); - manageSnapshots.removeTag(tagName).commit(); - }); - } catch (Exception e) { - throw new RuntimeException( - "Failed to drop tag: " + tagName + " in table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + SnapshotRef snapshotRef = icebergTable.refs().get(tagName); + + if (snapshotRef != null || !ifExists) { + try { + lease.getAuthenticator().execute(() -> { + ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); + manageSnapshots.removeTag(tagName).commit(); + }); + } catch (Exception e) { + throw new RuntimeException( + "Failed to drop tag: " + tagName + " in table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } } } } @@ -656,19 +668,22 @@ public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws Us public void dropBranchImpl(ExternalTable dorisTable, DropBranchInfo branchInfo) throws UserException { String branchName = branchInfo.getBranchName(); boolean ifExists = branchInfo.getIfExists(); - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - SnapshotRef snapshotRef = icebergTable.refs().get(branchName); - - if (snapshotRef != null || !ifExists) { - try { - executionAuthenticator.execute(() -> { - ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); - manageSnapshots.removeBranch(branchName).commit(); - }); - } catch (Exception e) { - throw new RuntimeException( - "Failed to drop branch: " + branchName + " in table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + SnapshotRef snapshotRef = icebergTable.refs().get(branchName); + + if (snapshotRef != null || !ifExists) { + try { + lease.getAuthenticator().execute(() -> { + ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); + manageSnapshots.removeBranch(branchName).commit(); + }); + } catch (Exception e) { + throw new RuntimeException( + "Failed to drop branch: " + branchName + " in table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } } } } @@ -759,22 +774,25 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateAddColumnMetadata(column, true); - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); - validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); - Schema schema = icebergTable.schema(); - validateNoCaseInsensitiveSiblingCollision( - schema.asStruct(), "", column.getName(), null, "add"); - UpdateSchema updateSchema = icebergTable.updateSchema(); - addOneColumn(updateSchema, column); - if (position != null) { - applyPosition(updateSchema, position, ColumnPath.of(column.getName()), schema, "add"); - } - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to add column: " + column.getName() + " to table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); + validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); + Schema schema = icebergTable.schema(); + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", column.getName(), null, "add"); + UpdateSchema updateSchema = icebergTable.updateSchema(); + addOneColumn(updateSchema, column); + if (position != null) { + applyPosition(updateSchema, position, ColumnPath.of(column.getName()), schema, "add"); + } + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to add column: " + column.getName() + " to table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -790,69 +808,78 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); - ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); - if (!parentPath.getType().isStructType()) { - throw new UserException("Parent column path '" + columnPath.getParentPathString() - + "' is not a struct in Iceberg table: " + icebergTable.name()); - } - validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), - parentPath.getColumnPath(), columnPath.getLeafName(), null, "add"); - - UpdateSchema updateSchema = icebergTable.updateSchema(); - org.apache.iceberg.types.Type dorisType = - toIcebergTypeForSchemaChange(column.getType(), columnPath.getFullPath()); - updateSchema.addColumn(parentPath.getFullPath(), columnPath.getLeafName(), dorisType, - column.getComment()); - if (position != null) { - applyPosition(updateSchema, position, childPath(parentPath.getColumnPath(), columnPath.getLeafName()), - icebergTable.schema(), "add"); - } - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to add nested column: " + columnPath.getFullPath() + " to table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); + ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); + if (!parentPath.getType().isStructType()) { + throw new UserException("Parent column path '" + columnPath.getParentPathString() + + "' is not a struct in Iceberg table: " + icebergTable.name()); + } + validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), + parentPath.getColumnPath(), columnPath.getLeafName(), null, "add"); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + org.apache.iceberg.types.Type dorisType = + toIcebergTypeForSchemaChange(column.getType(), columnPath.getFullPath()); + updateSchema.addColumn(parentPath.getFullPath(), columnPath.getLeafName(), dorisType, + column.getComment()); + if (position != null) { + applyPosition(updateSchema, position, childPath(parentPath.getColumnPath(), columnPath.getLeafName()), + icebergTable.schema(), "add"); + } + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to add nested column: " + columnPath.getFullPath() + " to table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - for (Column column : columns) { - validateAddColumnMetadata(column, true); - validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); - validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); - } - validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + for (Column column : columns) { + validateAddColumnMetadata(column, true); + validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); + validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); + } + validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); - UpdateSchema updateSchema = icebergTable.updateSchema(); - for (Column column : columns) { - addOneColumn(updateSchema, column); - } - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to add columns to table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); + UpdateSchema updateSchema = icebergTable.updateSchema(); + for (Column column : columns) { + addOneColumn(updateSchema, column); + } + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to add columns to table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - validateRowLineageColumnMutation(icebergTable, columnName, "drop"); - ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); - UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.deleteColumn(columnPath.getFullPath()); - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to drop column: " + columnName + " from table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + validateRowLineageColumnMutation(icebergTable, columnName, "drop"); + ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.deleteColumn(columnPath.getFullPath()); + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to drop column: " + columnName + " from table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -863,16 +890,19 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); return; } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); - UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.deleteColumn(resolvedPath.getFullPath()); - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to drop nested column: " + columnPath.getFullPath() + " from table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.deleteColumn(resolvedPath.getFullPath()); + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to drop nested column: " + columnPath.getFullPath() + " from table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -898,20 +928,23 @@ public void updateTableProperties(ExternalTable dorisTable, Map @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - validateRowLineageColumnMutation(icebergTable, oldName, "rename"); - validateRowLineageColumnMutation(icebergTable, newName, "rename to"); - Schema schema = icebergTable.schema(); - ResolvedColumnPath oldPath = resolveColumnPath(schema, ColumnPath.of(oldName), "rename"); - validateNoCaseInsensitiveSiblingCollision( - schema.asStruct(), "", newName, oldPath.getField(), "rename"); - UpdateSchema updateSchema = icebergTable.updateSchema(); - applyRenameColumn(schema, updateSchema, oldPath, newName); - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to rename column: " + oldName + " to " + newName - + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + validateRowLineageColumnMutation(icebergTable, oldName, "rename"); + validateRowLineageColumnMutation(icebergTable, newName, "rename to"); + Schema schema = icebergTable.schema(); + ResolvedColumnPath oldPath = resolveColumnPath(schema, ColumnPath.of(oldName), "rename"); + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", newName, oldPath.getField(), "rename"); + UpdateSchema updateSchema = icebergTable.updateSchema(); + applyRenameColumn(schema, updateSchema, oldPath, newName); + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to rename column: " + oldName + " to " + newName + + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -923,19 +956,24 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); return; } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); - ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); - validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), - parentPath.getColumnPath(), newName, resolvedPath.getField(), "rename"); - - UpdateSchema updateSchema = icebergTable.updateSchema(); - applyRenameColumn(icebergTable.schema(), updateSchema, resolvedPath, newName); - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to rename nested column: " + columnPath.getFullPath() + " to " + newName - + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + ResolvedColumnPath resolvedPath = validateNestedStructFieldPath( + icebergTable.schema(), columnPath, "rename"); + ResolvedColumnPath parentPath = resolveColumnPath( + icebergTable.schema(), columnPath.getParentPath(), "rename"); + validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), + parentPath.getColumnPath(), newName, resolvedPath.getField(), "rename"); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + applyRenameColumn(icebergTable.schema(), updateSchema, resolvedPath, newName); + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to rename nested column: " + columnPath.getFullPath() + " to " + newName + + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -985,63 +1023,66 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); - NestedField currentCol = icebergTable.schema().asStruct() - .caseInsensitiveField(columnPath.getTopLevelName()); - if (currentCol == null) { - throw new UserException("Column " + columnPath.getTopLevelName() + " does not exist"); - } - ResolvedColumnPath resolvedPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), - currentCol.type(), currentCol); - - validateModifyColumnMetadata(column, resolvedPath.getFullPath(), true); - boolean variantModify = validateVariantTypeChange( - currentCol.type(), column.getType(), resolvedPath.getFullPath()); - // A same-type VARIANT MODIFY only changes metadata. Existing v3 ORC tables can legally - // contain VARIANT columns even though Doris cannot write VARIANT values to ORC files. - validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), false, - !variantModify); - org.apache.iceberg.types.Type targetType; - if (variantModify) { - validateForModifyVariantColumn(column, currentCol, resolvedPath.getFullPath()); - targetType = currentCol.type(); - } else if (column.getType().isComplexType()) { - validateForModifyComplexColumn(column, currentCol); - targetType = currentCol.type(); - } else { - validateForModifyColumn(column, currentCol); - targetType = resolvePrimitiveTypeForModify( - currentCol.type(), column.getType(), resolvedPath.getFullPath()); - } + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); + NestedField currentCol = icebergTable.schema().asStruct() + .caseInsensitiveField(columnPath.getTopLevelName()); + if (currentCol == null) { + throw new UserException("Column " + columnPath.getTopLevelName() + " does not exist"); + } + ResolvedColumnPath resolvedPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), + currentCol.type(), currentCol); - UpdateSchema updateSchema = icebergTable.updateSchema(); - // Preserve the Iceberg doc when MODIFY COLUMN omits COMMENT; only an explicit COMMENT may change it. - String targetComment = resolveTargetComment(currentCol, column); - if (variantModify) { - if (!Objects.equals(currentCol.doc(), targetComment)) { - updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); + validateModifyColumnMetadata(column, resolvedPath.getFullPath(), true); + boolean variantModify = validateVariantTypeChange( + currentCol.type(), column.getType(), resolvedPath.getFullPath()); + // A same-type VARIANT MODIFY only changes metadata. Existing v3 ORC tables can legally + // contain VARIANT columns even though Doris cannot write VARIANT values to ORC files. + validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), false, + !variantModify); + org.apache.iceberg.types.Type targetType; + if (variantModify) { + validateForModifyVariantColumn(column, currentCol, resolvedPath.getFullPath()); + targetType = currentCol.type(); + } else if (column.getType().isComplexType()) { + validateForModifyComplexColumn(column, currentCol); + targetType = currentCol.type(); + } else { + validateForModifyColumn(column, currentCol); + targetType = resolvePrimitiveTypeForModify( + currentCol.type(), column.getType(), resolvedPath.getFullPath()); } - } else if (column.getType().isComplexType()) { - applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), - column.getType()); - if (!Objects.equals(currentCol.doc(), targetComment)) { - updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + // Preserve the Iceberg doc when MODIFY COLUMN omits COMMENT; only an explicit COMMENT may change it. + String targetComment = resolveTargetComment(currentCol, column); + if (variantModify) { + if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); + } + } else if (column.getType().isComplexType()) { + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), + column.getType()); + if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); + } + } else { + applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, + targetType.asPrimitiveType(), targetComment); } - } else { - applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, - targetType.asPrimitiveType(), targetComment); - } - applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); - if (position != null) { - applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); - } - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to modify column: " + column.getName() + " in table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); + if (position != null) { + applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); + } + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to modify column: " + column.getName() + " in table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -1054,50 +1095,53 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column return; } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); - NestedField currentCol = resolvedPath.getField(); - validateCollectionPseudoFieldComment( - icebergTable.schema(), resolvedPath, column.getComment(), column.isCommentSpecified()); - if (position != null) { - validatePositionTarget(icebergTable.schema(), resolvedPath.getColumnPath(), "modify"); - } + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); + NestedField currentCol = resolvedPath.getField(); + validateCollectionPseudoFieldComment( + icebergTable.schema(), resolvedPath, column.getComment(), column.isCommentSpecified()); + if (position != null) { + validatePositionTarget(icebergTable.schema(), resolvedPath.getColumnPath(), "modify"); + } - validateNestedModifyColumnMetadata(column, resolvedPath.getFullPath()); - validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); - org.apache.iceberg.types.Type targetType; - if (column.getType().isComplexType()) { - validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); - targetType = currentCol.type(); - } else { - validateForModifyColumn(column, currentCol, columnPath.getFullPath()); - targetType = resolvePrimitiveTypeForModify( - currentCol.type(), column.getType(), resolvedPath.getFullPath()); - } + validateNestedModifyColumnMetadata(column, resolvedPath.getFullPath()); + validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); + org.apache.iceberg.types.Type targetType; + if (column.getType().isComplexType()) { + validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); + targetType = currentCol.type(); + } else { + validateForModifyColumn(column, currentCol, columnPath.getFullPath()); + targetType = resolvePrimitiveTypeForModify( + currentCol.type(), column.getType(), resolvedPath.getFullPath()); + } - UpdateSchema updateSchema = icebergTable.updateSchema(); - String targetComment = resolveTargetComment(currentCol, column); - if (column.getType().isComplexType()) { - applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), - column.getType()); - if (!Objects.equals(currentCol.doc(), targetComment)) { - updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); + UpdateSchema updateSchema = icebergTable.updateSchema(); + String targetComment = resolveTargetComment(currentCol, column); + if (column.getType().isComplexType()) { + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), + column.getType()); + if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); + } + } else { + applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, + targetType.asPrimitiveType(), targetComment); } - } else { - applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, - targetType.asPrimitiveType(), targetComment); - } - applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); - if (position != null) { - applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); - } + if (position != null) { + applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); + } - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to modify nested column: " + columnPath.getFullPath() + " in table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to modify nested column: " + columnPath.getFullPath() + " in table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -1105,21 +1149,24 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column @Override public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - if (!columnPath.isNested()) { - validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); - } - ResolvedColumnPath resolvedPath = resolveColumnPath( - icebergTable.schema(), columnPath, "modify comment"); - validateCollectionPseudoFieldComment(icebergTable.schema(), resolvedPath, comment, true); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + if (!columnPath.isNested()) { + validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); + } + ResolvedColumnPath resolvedPath = resolveColumnPath( + icebergTable.schema(), columnPath, "modify comment"); + validateCollectionPseudoFieldComment(icebergTable.schema(), resolvedPath, comment, true); - UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.updateColumnDoc(resolvedPath.getFullPath(), StringUtils.defaultString(comment)); - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to modify column comment: " + columnPath.getFullPath() + " in table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), StringUtils.defaultString(comment)); + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to modify column comment: " + columnPath.getFullPath() + " in table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } } refreshTable(dorisTable, updateTime); } @@ -1672,28 +1719,31 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - List canonicalOrder = new ArrayList<>(newOrder.size()); - Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - for (String columnName : newOrder) { - validateRowLineageColumnMutation(icebergTable, columnName, "reorder"); - String canonicalName = resolveColumnPath( - icebergTable.schema(), ColumnPath.of(columnName), "reorder").getFullPath(); - if (!canonicalNames.add(canonicalName)) { - throw new UserException("Duplicate column in reorder columns: " + columnName); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + List canonicalOrder = new ArrayList<>(newOrder.size()); + Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String columnName : newOrder) { + validateRowLineageColumnMutation(icebergTable, columnName, "reorder"); + String canonicalName = resolveColumnPath( + icebergTable.schema(), ColumnPath.of(columnName), "reorder").getFullPath(); + if (!canonicalNames.add(canonicalName)) { + throw new UserException("Duplicate column in reorder columns: " + columnName); + } + canonicalOrder.add(canonicalName); + } + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.moveFirst(canonicalOrder.get(0)); + for (int i = 1; i < canonicalOrder.size(); i++) { + updateSchema.moveAfter(canonicalOrder.get(i), canonicalOrder.get(i - 1)); + } + try { + lease.getAuthenticator().execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to reorder columns in table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); } - canonicalOrder.add(canonicalName); - } - UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.moveFirst(canonicalOrder.get(0)); - for (int i = 1; i < canonicalOrder.size(); i++) { - updateSchema.moveAfter(canonicalOrder.get(i), canonicalOrder.get(i - 1)); - } - try { - executionAuthenticator.execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to reorder columns in table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); } refreshTable(dorisTable, updateTime); } @@ -1739,26 +1789,29 @@ private Term getTransform(String transformName, String columnName, Integer trans */ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); - String transformName = clause.getTransformName(); - Integer transformArg = clause.getTransformArg(); - String columnName = clause.getColumnName(); - String partitionFieldName = clause.getPartitionFieldName(); - Term transform = getTransform(transformName, columnName, transformArg); + String transformName = clause.getTransformName(); + Integer transformArg = clause.getTransformArg(); + String columnName = clause.getColumnName(); + String partitionFieldName = clause.getPartitionFieldName(); + Term transform = getTransform(transformName, columnName, transformArg); - if (partitionFieldName != null) { - updateSpec.addField(partitionFieldName, transform); - } else { - updateSpec.addField(transform); - } + if (partitionFieldName != null) { + updateSpec.addField(partitionFieldName, transform); + } else { + updateSpec.addField(transform); + } - try { - executionAuthenticator.execute(() -> updateSpec.commit()); - } catch (Exception e) { - throw new UserException("Failed to add partition field to table: " + icebergTable.name() - + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); + try { + lease.getAuthenticator().execute(() -> updateSpec.commit()); + } catch (Exception e) { + throw new UserException("Failed to add partition field to table: " + icebergTable.name() + + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); + } } refreshTable(dorisTable, updateTime); } @@ -1768,24 +1821,27 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause */ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); - if (clause.getPartitionFieldName() != null) { - updateSpec.removeField(clause.getPartitionFieldName()); - } else { - String transformName = clause.getTransformName(); - Integer transformArg = clause.getTransformArg(); - String columnName = clause.getColumnName(); - Term transform = getTransform(transformName, columnName, transformArg); - updateSpec.removeField(transform); - } + if (clause.getPartitionFieldName() != null) { + updateSpec.removeField(clause.getPartitionFieldName()); + } else { + String transformName = clause.getTransformName(); + Integer transformArg = clause.getTransformArg(); + String columnName = clause.getColumnName(); + Term transform = getTransform(transformName, columnName, transformArg); + updateSpec.removeField(transform); + } - try { - executionAuthenticator.execute(() -> updateSpec.commit()); - } catch (Exception e) { - throw new UserException("Failed to drop partition field from table: " + icebergTable.name() - + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); + try { + lease.getAuthenticator().execute(() -> updateSpec.commit()); + } catch (Exception e) { + throw new UserException("Failed to drop partition field from table: " + icebergTable.name() + + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); + } } refreshTable(dorisTable, updateTime); } @@ -1795,38 +1851,41 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus */ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); - UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); + try (IcebergExternalMetaCache.WritableTableLease lease = + IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + Table icebergTable = lease.getTable(); + UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); - // remove old partition field - if (clause.getOldPartitionFieldName() != null) { - updateSpec.removeField(clause.getOldPartitionFieldName()); - } else { - String oldTransformName = clause.getOldTransformName(); - Integer oldTransformArg = clause.getOldTransformArg(); - String oldColumnName = clause.getOldColumnName(); - Term oldTransform = getTransform(oldTransformName, oldColumnName, oldTransformArg); - updateSpec.removeField(oldTransform); - } - - // add new partition field - String newPartitionFieldName = clause.getNewPartitionFieldName(); - String newTransformName = clause.getNewTransformName(); - Integer newTransformArg = clause.getNewTransformArg(); - String newColumnName = clause.getNewColumnName(); - Term newTransform = getTransform(newTransformName, newColumnName, newTransformArg); - - if (newPartitionFieldName != null) { - updateSpec.addField(newPartitionFieldName, newTransform); - } else { - updateSpec.addField(newTransform); - } + // remove old partition field + if (clause.getOldPartitionFieldName() != null) { + updateSpec.removeField(clause.getOldPartitionFieldName()); + } else { + String oldTransformName = clause.getOldTransformName(); + Integer oldTransformArg = clause.getOldTransformArg(); + String oldColumnName = clause.getOldColumnName(); + Term oldTransform = getTransform(oldTransformName, oldColumnName, oldTransformArg); + updateSpec.removeField(oldTransform); + } - try { - executionAuthenticator.execute(() -> updateSpec.commit()); - } catch (Exception e) { - throw new UserException("Failed to replace partition field in table: " + icebergTable.name() - + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); + // add new partition field + String newPartitionFieldName = clause.getNewPartitionFieldName(); + String newTransformName = clause.getNewTransformName(); + Integer newTransformArg = clause.getNewTransformArg(); + String newColumnName = clause.getNewColumnName(); + Term newTransform = getTransform(newTransformName, newColumnName, newTransformArg); + + if (newPartitionFieldName != null) { + updateSpec.addField(newPartitionFieldName, newTransform); + } else { + updateSpec.addField(newTransform); + } + + try { + lease.getAuthenticator().execute(() -> updateSpec.commit()); + } catch (Exception e) { + throw new UserException("Failed to replace partition field in table: " + icebergTable.name() + + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); + } } refreshTable(dorisTable, updateTime); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java index 570a42749eeca3..95bf23fed72fbe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiScanNodeTest.java @@ -26,23 +26,30 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hive.source.HiveScanNode; +import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; +import org.apache.doris.datasource.hudi.HudiUtils; import org.apache.doris.nereids.StatementContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileRangeDesc; import com.google.common.collect.ImmutableMap; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.Option; +import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.storage.StoragePath; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Answers; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; @@ -312,6 +319,47 @@ public void testEmptyFullScanDoesNotAcquireUnusedFsView() throws Exception { Assertions.assertNull(getField(node, HudiScanNode.class, "fsViewLease")); } + @Test + public void testSchemaResolutionRejectsRetiredHmsGenerationBeforeDescriptorPublication() throws Exception { + HudiScanNode node = Mockito.mock(HudiScanNode.class, Answers.CALLS_REAL_METHODS); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + AtomicLong runtimeGeneration = new AtomicLong(1L); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getRuntimeGeneration()).thenAnswer(invocation -> runtimeGeneration.get()); + setField(node, HiveScanNode.class, "hmsTable", table); + setField(node, HudiScanNode.class, "hmsRuntimeGeneration", 1L); + setField(node, HudiScanNode.class, "queryInstant", "20260831120000"); + setField(node, HudiScanNode.class, "hudiClient", Mockito.mock(HoodieTableMetaClient.class)); + + HudiSplit split = new HudiSplit( + LocationPath.of("file:///table/fileid_1-0-1_20260831120000.parquet"), + 0, 10, 10, new String[0], Collections.emptyList()); + split.setTableFormatType(TableFormatType.HUDI); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET); + HudiSchemaCacheValue schemaValue = Mockito.mock(HudiSchemaCacheValue.class); + Mockito.when(schemaValue.isEnableSchemaEvolution()).thenReturn(true); + Mockito.when(schemaValue.getCommitInstantInternalSchema(Mockito.any(), Mockito.anyLong())) + .thenAnswer(invocation -> { + runtimeGeneration.incrementAndGet(); + return Mockito.mock(InternalSchema.class); + }); + + try (MockedStatic mockedHudiUtils = Mockito.mockStatic(HudiUtils.class)) { + mockedHudiUtils.when(() -> HudiUtils.getSchemaCacheValue(table, "20260831120000")) + .thenReturn(schemaValue); + Method method = HudiScanNode.class.getDeclaredMethod( + "setHudiParams", TFileRangeDesc.class, HudiSplit.class); + method.setAccessible(true); + InvocationTargetException exception = Assertions.assertThrows( + InvocationTargetException.class, () -> method.invoke(node, rangeDesc, split)); + Assertions.assertInstanceOf(IllegalStateException.class, exception.getCause()); + } + Mockito.verify(schemaValue).getCommitInstantInternalSchema(Mockito.any(), Mockito.anyLong()); + Assertions.assertFalse(rangeDesc.isSetTableFormatParams()); + } + private static HudiScanNode partitionScanNode( StatementContext.ExternalScanTaskCache cache, HoodieTableFileSystemView fsView, String queryInstant, boolean nativeReader, boolean runtimePrune) throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java index 5a3590602a7a17..ed14e15fc55397 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.RefreshManager; import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; @@ -98,6 +99,12 @@ public void setUp() throws IOException { .thenReturn(icebergTable); mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.any(), Mockito.any())) .thenReturn(icebergTable); + IcebergExternalMetaCache.WritableTableLease lease = + Mockito.mock(IcebergExternalMetaCache.WritableTableLease.class); + Mockito.when(lease.getTable()).thenReturn(icebergTable); + Mockito.when(lease.getAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + mockedIcebergUtils.when(() -> IcebergUtils.acquireWritableIcebergTable(Mockito.any(), Mockito.any())) + .thenReturn(lease); // mock Env.getCurrentEnv().getEditLog().logBranchOrTag(info) to do nothing Env mockEnv = Mockito.mock(Env.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 115b019f1f010c..9d075e8f3283d0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -139,8 +139,7 @@ public void testTopLevelVariantModifyOnlyUpdatesMetadataOnOrcTable() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())) - .thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("payload"), column, ColumnPosition.FIRST, 1L); } @@ -167,8 +166,7 @@ public void testTopLevelVariantModifyRejectsTypeConversions() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())) - .thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("variant_col"), new Column("variant_col", Type.STRING, true), null, 1L), @@ -242,6 +240,36 @@ public void testUpdateTablePropertiesDoesNotRefreshAfterCommitFailure() { Mockito.verify(dorisCatalog, Mockito.never()).getDbForReplay(Mockito.anyString()); } + @Test + public void testSchemaMutationRetainsWritableGenerationThroughCommit() throws Exception { + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(icebergTable.schema()).thenReturn(new Schema()); + Mockito.when(icebergTable.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + AtomicBoolean leaseClosed = new AtomicBoolean(); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + IcebergExternalMetaCache.WritableTableLease lease = + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); + Mockito.doAnswer(invocation -> { + leaseClosed.set(true); + return null; + }).when(lease).close(); + Mockito.doAnswer(invocation -> { + Assert.assertFalse("writable generation closed before schema commit", leaseClosed.get()); + return null; + }).when(updateSchema).commit(); + + ops.addColumn(dorisTable, new Column("new_col", Type.INT, true), null, 123L); + + Assert.assertTrue(leaseClosed.get()); + } + } + @Test public void testValidateForModifyColumnRejectsComplexToPrimitive() { Column column = new Column("struct_col", Type.INT, true); @@ -358,7 +386,7 @@ public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), new Column("new_field", Type.LARGEINT, true), null, 1L), @@ -392,7 +420,7 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); @@ -427,7 +455,7 @@ public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwab try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); } @@ -455,7 +483,7 @@ public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComm try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -497,7 +525,7 @@ public void testFullStructModifyPreservesOmittedChildComments() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), new Column("payload", payloadType, true), null, 1L); @@ -527,7 +555,7 @@ public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -553,7 +581,7 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("id"), new Column("id", Type.BIGINT, true), null, 1L); @@ -579,7 +607,7 @@ public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L), @@ -601,7 +629,7 @@ public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L); @@ -632,7 +660,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws T try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, @@ -668,7 +696,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); @@ -699,7 +727,7 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, @@ -728,7 +756,7 @@ public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdate try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, @@ -759,7 +787,7 @@ public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); // Iceberg schema columns are represented as keys in Doris, so the legacy API must not // interpret isKey as an explicit KEY clause. @@ -790,7 +818,7 @@ public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, column, null, 1L); } @@ -824,7 +852,7 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); @@ -877,7 +905,8 @@ public void execute(Runnable task) { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(staleTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, staleTable, conflictOps, + conflictDorisCatalog.getExecutionAuthenticator()); try { conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), @@ -928,7 +957,7 @@ public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); icebergTable.refresh(); @@ -978,7 +1007,7 @@ public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.renameColumn(dorisTable, "a", "renamed", 1L); icebergTable.refresh(); @@ -1007,7 +1036,7 @@ public void testNestedColumnOperationsRejectDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), nestedAddDefaultColumn, null, 1L), @@ -1038,7 +1067,7 @@ public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: id"); @@ -1069,7 +1098,7 @@ public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), @@ -1102,7 +1131,7 @@ public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); @@ -1149,7 +1178,7 @@ public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, new Column("info", infoType, true), null, 1L), @@ -1225,7 +1254,7 @@ public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.addColumn( dorisTable, new Column("id", Type.STRING, true), null, 1L), @@ -1261,7 +1290,7 @@ public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); } @@ -1283,7 +1312,7 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), null, 1L); @@ -1306,7 +1335,7 @@ public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), @@ -1337,7 +1366,7 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), "struct comment", 1L); @@ -1363,7 +1392,7 @@ public void testRejectsCommentsOnDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), @@ -1403,7 +1432,7 @@ public void testRejectsTopLevelRowLineageMutationsForV3Tables() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); + mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable); assertUserException(() -> ops.addColumn(dorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L), @@ -1459,10 +1488,8 @@ public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(v3DorisTable), Mockito.any())) - .thenReturn(v3IcebergTable); - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(v2DorisTable), Mockito.any())) - .thenReturn(v2IcebergTable); + mockWritableTableLease(mockedIcebergUtils, v3DorisTable, v3IcebergTable); + mockWritableTableLease(mockedIcebergUtils, v2DorisTable, v2IcebergTable); ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), new Column("_last_updated_sequence_number", Type.BIGINT, true), null, 1L); @@ -1553,6 +1580,24 @@ private void invokeValidationMethod(Method method, Column column, NestedField cu } } + private IcebergExternalMetaCache.WritableTableLease mockWritableTableLease( + MockedStatic mockedIcebergUtils, ExternalTable dorisTable, Table icebergTable) { + return mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable, ops, + dorisCatalog.getExecutionAuthenticator()); + } + + private IcebergExternalMetaCache.WritableTableLease mockWritableTableLease( + MockedStatic mockedIcebergUtils, ExternalTable dorisTable, Table icebergTable, + IcebergMetadataOps expectedOps, ExecutionAuthenticator authenticator) { + IcebergExternalMetaCache.WritableTableLease lease = + Mockito.mock(IcebergExternalMetaCache.WritableTableLease.class); + Mockito.when(lease.getTable()).thenReturn(icebergTable); + Mockito.when(lease.getAuthenticator()).thenReturn(authenticator); + mockedIcebergUtils.when(() -> IcebergUtils.acquireWritableIcebergTable(dorisTable, expectedOps)) + .thenReturn(lease); + return lease; + } + private void assertUserException(ThrowingRunnable runnable, String expectedMessage) { try { runnable.run(); From fe0b502cade4ea4ffbf0e669a8b7a2380b7d6705 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 1 Sep 2026 11:29:38 +0800 Subject: [PATCH 32/38] [fix](fe) Keep Iceberg and Hudi resources generation-scoped ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: Iceberg table handles, snapshots, actions, transactions, and asynchronous rewrite tasks could outlive the cache or catalog generation that created their FileIO, authenticator, and planning executor. Hudi runtime capture could also invert the lifecycle-stripe and catalog-monitor lock order. Cache invalidation or catalog reset could therefore close resources still used by an operation, leak cancelled rewrite tasks through the transient-task map, or deadlock a first Hudi scan with ALTER. This change introduces an immutable Iceberg runtime context and a reference-counted writable-table lease shared by the action, transaction, and each asynchronous task. It binds reads and mutations to one exact catalog generation, releases task borrowers on every success, failure, cancellation, timeout, and pre-start path, removes cancelled tasks from the scheduler map before cancellation, and enforces lifecycle-stripe then catalog-monitor ordering for Hudi. ### Release note Fix Iceberg FileIO and Hudi filesystem-view lifecycle handling during cache eviction and catalog reset. ### Check List (For Author) - Test: Unit Test and FE build - Focused Iceberg/Hudi lifecycle, cache, action, transaction, scan, and rewrite unit tests - ./build.sh --fe - git diff --check - Behavior changed: Yes. Active Iceberg and Hudi operations retain their exact runtime generation until completion. - Does this need documentation: No --- .../datasource/hive/HMSExternalCatalog.java | 28 ++- .../datasource/hive/HMSExternalTable.java | 4 +- .../datasource/hive/IcebergDlaTable.java | 15 +- .../iceberg/IcebergExternalMetaCache.java | 190 ++++++++++++++++-- .../iceberg/IcebergExternalTable.java | 4 - .../iceberg/IcebergRuntimeContext.java | 60 ++++++ .../iceberg/IcebergSnapshotCacheValue.java | 16 ++ .../iceberg/IcebergTableCacheValue.java | 11 + .../iceberg/IcebergTransaction.java | 131 ++++++++---- .../datasource/iceberg/IcebergUtils.java | 23 ++- .../iceberg/action/BaseIcebergAction.java | 29 +++ .../IcebergCherrypickSnapshotAction.java | 4 +- .../action/IcebergExpireSnapshotsAction.java | 5 +- .../action/IcebergFastForwardAction.java | 5 +- .../action/IcebergPublishChangesAction.java | 4 +- .../action/IcebergRewriteDataFilesAction.java | 10 +- .../action/IcebergRewriteManifestsAction.java | 4 +- .../IcebergRollbackToSnapshotAction.java | 4 +- .../IcebergRollbackToTimestampAction.java | 5 +- .../IcebergSetCurrentSnapshotAction.java | 5 +- .../rewrite/RewriteDataFileExecutor.java | 168 +++++++++------- .../iceberg/rewrite/RewriteGroupTask.java | 39 +++- .../iceberg/source/IcebergScanNode.java | 41 ++-- .../hive/HMSExternalCatalogLifecycleTest.java | 98 +++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 47 +++-- .../iceberg/IcebergTransactionTest.java | 162 ++++++++++----- .../iceberg/action/BaseIcebergActionTest.java | 78 +++++++ .../rewrite/RewriteDataFileExecutorTest.java | 25 +++ .../iceberg/rewrite/RewriteGroupTaskTest.java | 17 ++ 29 files changed, 940 insertions(+), 292 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRuntimeContext.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalCatalogLifecycleTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/BaseIcebergActionTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 09dd26a7df44e6..89b1ac6a3077fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -96,11 +96,20 @@ public synchronized long getRuntimeGeneration() { } /** Captures the authentication context and generation used by one Hudi scan. */ - public synchronized HudiScanRuntimeContext getHudiScanRuntimeContext() { - makeSureInitialized(); - HudiExternalMetaCache hudiCache = Env.getCurrentEnv().getExtMetaCacheMgr().hudi(getId()); - return new HudiScanRuntimeContext(runtimeGeneration.get(), executionAuthenticator, - hudiCache.captureFsViewGeneration(getId())); + public HudiScanRuntimeContext getHudiScanRuntimeContext() { + ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + return cacheMgr.withCatalogLifecycleLock(getId(), () -> { + long generation; + ExecutionAuthenticator authenticator; + synchronized (this) { + makeSureInitialized(); + generation = runtimeGeneration.get(); + authenticator = executionAuthenticator; + } + HudiExternalMetaCache hudiCache = cacheMgr.hudi(getId()); + return new HudiScanRuntimeContext(generation, authenticator, + hudiCache.captureFsViewGeneration(getId())); + }); } @Override @@ -349,9 +358,14 @@ public synchronized IcebergTableLoadContext beginIcebergTableLoad() { } @Override - public synchronized void resetToUninitialized(boolean invalidCache) { + public void resetToUninitialized(boolean invalidCache) { ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - resetCatalogRuntime(cacheMgr, invalidCache); + cacheMgr.withCatalogLifecycleLock(getId(), () -> { + synchronized (this) { + resetCatalogRuntime(cacheMgr, invalidCache); + } + return null; + }); } private void resetCatalogRuntime(ExternalMetaCacheMgr cacheMgr, boolean invalidCache) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index a727d4386f774a..5cc00497e2bb28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -897,8 +897,8 @@ public Optional getColumnStatistic(String colName) { return getHiveColumnStats(colName); case ICEBERG: if (GlobalVariable.enableFetchIcebergStats) { - return StatisticsUtil.getIcebergColumnStats(colName, - IcebergUtils.getIcebergTable(this)); + return IcebergUtils.withIcebergTable(this, + table -> StatisticsUtil.getIcebergColumnStats(colName, table)); } else { break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java index 4868e0a58410b0..2d57c1e7683886 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java @@ -114,17 +114,19 @@ protected boolean isValidRelatedTable() { if (isValidRelatedTableCached) { return isValidRelatedTable; } - isValidRelatedTable = false; + isValidRelatedTable = IcebergUtils.withIcebergTable(hmsTable, this::isValidRelatedTable); + isValidRelatedTableCached = true; + return isValidRelatedTable; + } + + private boolean isValidRelatedTable(Table table) { Set allFields = Sets.newHashSet(); - Table table = IcebergUtils.getIcebergTable(hmsTable); for (PartitionSpec spec : table.specs().values()) { if (spec == null) { - isValidRelatedTableCached = true; return false; } List fields = spec.fields(); if (fields.size() != 1) { - isValidRelatedTableCached = true; return false; } PartitionField partitionField = spec.fields().get(0); @@ -133,13 +135,10 @@ protected boolean isValidRelatedTable() { && !IcebergUtils.MONTH.equals(transformName) && !IcebergUtils.DAY.equals(transformName) && !IcebergUtils.HOUR.equals(transformName)) { - isValidRelatedTableCached = true; return false; } allFields.add(table.schema().findColumnName(partitionField.sourceId())); } - isValidRelatedTableCached = true; - isValidRelatedTable = allFields.size() == 1; - return isValidRelatedTable; + return allFields.size() == 1; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 5f2a78d65da9a0..023e6841ff1359 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -39,6 +39,8 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.qe.ConnectContext; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFiles; @@ -60,6 +62,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.Nullable; @@ -148,9 +151,7 @@ public Table getIcebergTable(ExternalTable dorisTable) { if (lease != null) { return lease.getIcebergTable(); } - // Background callers have no deterministic statement boundary. Use a live catalog load - // instead of returning a cache generation that can be evicted immediately after lookup. - return getWritableIcebergTable(dorisTable); + throw new IllegalStateException("Iceberg table access outside a statement must use a scoped borrower"); } ThreadPoolExecutor getIcebergTableExecutor(ExternalTable dorisTable) { @@ -172,7 +173,8 @@ T withIcebergTable(ExternalTable dorisTable, Function action) { } } - public Table getWritableIcebergTable(ExternalTable dorisTable) { + @VisibleForTesting + Table getWritableIcebergTable(ExternalTable dorisTable) { return getWritableIcebergTable(dorisTable, null); } @@ -183,7 +185,8 @@ public Table getWritableIcebergTable(ExternalTable dorisTable) { * so the caller's later updates - executed through its retained ops and authenticator - * can never operate a newer generation's handle. */ - public Table getWritableIcebergTable(ExternalTable dorisTable, @Nullable IcebergMetadataOps expectedOps) { + @VisibleForTesting + Table getWritableIcebergTable(ExternalTable dorisTable, @Nullable IcebergMetadataOps expectedOps) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { @@ -206,7 +209,11 @@ public Table getWritableIcebergTable(ExternalTable dorisTable, @Nullable Iceberg return table; } - WritableTableLease acquireWritableIcebergTable( + public WritableTableLease acquireWritableIcebergTable(ExternalTable dorisTable) { + return acquireWritableIcebergTable(dorisTable, null); + } + + public WritableTableLease acquireWritableIcebergTable( ExternalTable dorisTable, @Nullable IcebergMetadataOps expectedOps) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); CatalogIf catalog = getCatalog(nameMapping.getCtlId()); @@ -233,8 +240,12 @@ WritableTableLease acquireWritableIcebergTable( ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, context.isEnableMappingVarbinary(), context.isEnableMappingTimestampTz()); owner.add(context.promote()::close); + IcebergRuntimeContext runtimeContext = new IcebergRuntimeContext( + context.getAuthenticator(), ops.getThreadPoolWithPreAuth(), + context.getMetastoreProperties(), context.getStorageProperties()); WritableTableLease lease = new WritableTableLease( - table, context.getAuthenticator(), owner.cleanup()); + table, ops, runtimeContext, context.isEnableMappingVarbinary(), + context.isEnableMappingTimestampTz(), owner.cleanup()); owner.transfer(); return lease; } @@ -261,8 +272,12 @@ WritableTableLease acquireWritableIcebergTable( ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, enableMappingVarbinary, enableMappingTimestampTz); owner.add(context.promote()::close); + IcebergRuntimeContext runtimeContext = new IcebergRuntimeContext( + context.getAuthenticator(), context.getExecutor(), + context.getMetastoreProperties(), context.getStorageProperties()); WritableTableLease lease = new WritableTableLease( - table, context.getAuthenticator(), owner.cleanup()); + table, ops, runtimeContext, enableMappingVarbinary, + enableMappingTimestampTz, owner.cleanup()); owner.transfer(); return lease; } @@ -302,7 +317,34 @@ private Table createQueryTable( public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - IcebergTableCacheValue tableValue = statementValue(nameMapping); + IcebergTableCacheValue.Lease statementLease = statementLease(nameMapping); + if (statementLease != null) { + return getSnapshotCache(dorisTable, nameMapping, statementLease.getValue()); + } + try (IcebergTableCacheValue.Lease operationLease = borrow(nameMapping)) { + return getSnapshotCache(dorisTable, nameMapping, operationLease.getValue()) + .withoutRetainedTable(); + } + } + + /** Build a snapshot bound to an action's retained writable generation without another lookup. */ + public IcebergSnapshotCacheValue getSnapshotForWritableLease( + ExternalTable dorisTable, WritableTableLease lease) { + Table table = lease.getTable(); + return execute(lease.getAuthenticator(), () -> loadSnapshotProjection( + dorisTable, table, table, IcebergSnapshotCacheValue.retainCurrentSnapshotJson(table), + true, lease.getAuthenticator(), lease.isEnableMappingVarbinary(), + lease.isEnableMappingTimestampTz(), + dorisTable instanceof IcebergExternalTable + ? ((IcebergExternalTable) dorisTable).isValidRelatedTable(table) : null) + .bindCapturedAuthenticator(lease.getAuthenticator()) + .bindRuntimeContext(lease.getRuntimeContext()) + .bindSchemaMappingOptions(lease.isEnableMappingVarbinary(), + lease.isEnableMappingTimestampTz())); + } + + private IcebergSnapshotCacheValue getSnapshotCache( + ExternalTable dorisTable, NameMapping nameMapping, IcebergTableCacheValue tableValue) { Table retainedTable = tableValue.getRetainedIcebergTable(); java.util.Optional optionalKey = IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); @@ -318,6 +360,7 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { authenticator, tableValue.isEnableMappingVarbinary(), tableValue.isEnableMappingTimestampTz()) .bindCapturedAuthenticator(authenticator) + .bindRuntimeContext(tableValue.getRuntimeContext()) .bindSchemaMappingOptions(tableValue.isEnableMappingVarbinary(), tableValue.isEnableMappingTimestampTz())); } @@ -337,6 +380,7 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { authenticator, tableValue.isEnableMappingVarbinary(), tableValue.isEnableMappingTimestampTz()) .bindCapturedAuthenticator(authenticator) + .bindRuntimeContext(tableValue.getRuntimeContext()) .bindSchemaMappingOptions(tableValue.isEnableMappingVarbinary(), tableValue.isEnableMappingTimestampTz()); if (entry.isWeightAccounting()) { @@ -374,6 +418,14 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { return snapshotValue; } + @VisibleForTesting + IcebergSnapshotCacheValue getSnapshotCacheWithRetainedTableForTest(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + try (IcebergTableCacheValue.Lease operationLease = borrow(nameMapping)) { + return getSnapshotCache(dorisTable, nameMapping, operationLease.getValue()); + } + } + public List getSnapshotList(ExternalTable dorisTable) { Table icebergTable = getQueryScopedIcebergTable(dorisTable); List snapshots = com.google.common.collect.Lists.newArrayList(); @@ -387,7 +439,17 @@ public View getIcebergView(ExternalTable dorisTable) { } public IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId) { - IcebergTableCacheValue tableValue = statementValue(nameMapping); + IcebergTableCacheValue.Lease statementLease = statementLease(nameMapping); + if (statementLease != null) { + return getIcebergSchemaCacheValue(nameMapping, schemaId, statementLease.getValue()); + } + try (IcebergTableCacheValue.Lease operationLease = borrow(nameMapping)) { + return getIcebergSchemaCacheValue(nameMapping, schemaId, operationLease.getValue()); + } + } + + private IcebergSchemaCacheValue getIcebergSchemaCacheValue( + NameMapping nameMapping, long schemaId, IcebergTableCacheValue tableValue) { return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable(), tableValue.getAuthenticator(), tableValue.isEnableMappingVarbinary(), tableValue.isEnableMappingTimestampTz()); @@ -502,6 +564,7 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { try (TableResourceOwner catalogOwner = new TableResourceOwner(context.promote()::close)) { IcebergTableCacheValue value = execute(context.getAuthenticator(), () -> createLoadedTableValue( nameMapping, table, ops.getThreadPoolWithPreAuth(), context.getAuthenticator(), + context.getMetastoreProperties(), context.getStorageProperties(), enableMappingVarbinary, enableMappingTimestampTz, owner, catalogOwner.cleanup())); owner.transfer(); @@ -526,6 +589,7 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { try (TableResourceOwner catalogOwner = new TableResourceOwner(context.promote()::close)) { IcebergTableCacheValue value = execute(context.getAuthenticator(), () -> createLoadedTableValue( nameMapping, table, context.getExecutor(), context.getAuthenticator(), + context.getMetastoreProperties(), context.getStorageProperties(), enableMappingVarbinary, enableMappingTimestampTz, owner, catalogOwner.cleanup())); owner.transfer(); @@ -541,12 +605,17 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { private IcebergTableCacheValue createLoadedTableValue(NameMapping nameMapping, Table table, ThreadPoolExecutor planningExecutor, ExecutionAuthenticator authenticator, + org.apache.doris.datasource.property.metastore.MetastoreProperties metastoreProperties, + Map storageProperties, boolean enableMappingVarbinary, boolean enableMappingTimestampTz, TableResourceOwner tableOwner, Runnable cleanup) { IcebergTableCacheValue loaded = new IcebergTableCacheValue( table, planningExecutor, () -> null, tableOwner.cleanup(), cleanup); try { loaded.bindAuthenticator(authenticator); + loaded.bindRuntimeContext(new IcebergRuntimeContext( + authenticator, planningExecutor, metastoreProperties, storageProperties)); loaded.bindSchemaMappingOptions(enableMappingVarbinary, enableMappingTimestampTz); MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); @@ -580,9 +649,7 @@ private IcebergTableCacheValue statementValue(NameMapping nameMapping) { if (lease != null) { return lease.getValue(); } - IcebergTableCacheValue value = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); - value.releaseLoaderReference(); - return value; + throw new IllegalStateException("Iceberg table generation access requires a statement scope"); } @Nullable @@ -614,6 +681,11 @@ private IcebergTableCacheValue.Lease borrow(NameMapping nameMapping) { } } + @VisibleForTesting + IcebergTableCacheValue.Lease borrowForTest(NameMapping nameMapping) { + return borrow(nameMapping); + } + private static ExecutionAuthenticator requireExecutionAuthenticator(CatalogIf catalog) { if (!(catalog instanceof ExternalCatalog)) { throw new RuntimeException("Iceberg metadata cache requires an external catalog"); @@ -812,29 +884,90 @@ private static boolean sharesOperationalResources( && currentValue.isEnableMappingTimestampTz() == projection.isEnableMappingTimestampTz(); } - static final class WritableTableLease implements AutoCloseable { + public static final class WritableTableLease implements AutoCloseable { private final Table table; - private final ExecutionAuthenticator authenticator; - private final Runnable cleanup; + private final IcebergMetadataOps ops; + private final IcebergRuntimeContext runtimeContext; + private final boolean enableMappingVarbinary; + private final boolean enableMappingTimestampTz; + private final ReferenceCountedCleanup cleanup; private final AtomicBoolean closed = new AtomicBoolean(); - private WritableTableLease(Table table, ExecutionAuthenticator authenticator, Runnable cleanup) { + WritableTableLease(Table table, IcebergMetadataOps ops, IcebergRuntimeContext runtimeContext, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz, Runnable cleanup) { + this(table, ops, runtimeContext, enableMappingVarbinary, enableMappingTimestampTz, + new ReferenceCountedCleanup(cleanup)); + } + + private WritableTableLease(Table table, IcebergMetadataOps ops, IcebergRuntimeContext runtimeContext, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz, + ReferenceCountedCleanup cleanup) { this.table = table; - this.authenticator = authenticator; + this.ops = ops; + this.runtimeContext = runtimeContext; + this.enableMappingVarbinary = enableMappingVarbinary; + this.enableMappingTimestampTz = enableMappingTimestampTz; this.cleanup = cleanup; } - Table getTable() { + /** Retain the exact catalog generation for asynchronous work owned by the caller. */ + public WritableTableLease retain() { + Preconditions.checkState(!closed.get(), "Writable Iceberg table lease is already closed"); + cleanup.retain(); + return new WritableTableLease(table, ops, runtimeContext, + enableMappingVarbinary, enableMappingTimestampTz, cleanup); + } + + public Table getTable() { return table; } - ExecutionAuthenticator getAuthenticator() { - return authenticator; + public IcebergMetadataOps getOps() { + return ops; + } + + public ExecutionAuthenticator getAuthenticator() { + return runtimeContext.getAuthenticator(); + } + + public IcebergRuntimeContext getRuntimeContext() { + return runtimeContext; + } + + boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; } @Override public void close() { if (closed.compareAndSet(false, true)) { + cleanup.release(); + } + } + } + + private static final class ReferenceCountedCleanup { + private final AtomicInteger references = new AtomicInteger(1); + private final Runnable cleanup; + + private ReferenceCountedCleanup(Runnable cleanup) { + this.cleanup = cleanup; + } + + private void retain() { + int current; + do { + current = references.get(); + Preconditions.checkState(current > 0, "Writable Iceberg table lease is already closed"); + } while (!references.compareAndSet(current, current + 1)); + } + + private void release() { + if (references.decrementAndGet() == 0) { cleanup.run(); } } @@ -895,6 +1028,17 @@ private IcebergSnapshotCacheValue loadSnapshotProjection( String retainedCurrentSnapshotJson, boolean isolateForQueries, ExecutionAuthenticator authenticator, boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { + return loadSnapshotProjection(dorisTable, projectionTable, retainedTable, + retainedCurrentSnapshotJson, isolateForQueries, authenticator, + enableMappingVarbinary, enableMappingTimestampTz, null); + } + + private IcebergSnapshotCacheValue loadSnapshotProjection( + ExternalTable dorisTable, Table projectionTable, Table retainedTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries, + ExecutionAuthenticator authenticator, + boolean enableMappingVarbinary, boolean enableMappingTimestampTz, + @Nullable Boolean validRelatedTableOverride) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); @@ -903,7 +1047,9 @@ private IcebergSnapshotCacheValue loadSnapshotProjection( MTMVRelatedTableIf table = (MTMVRelatedTableIf) dorisTable; IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(projectionTable); IcebergPartitionInfo icebergPartitionInfo; - if (!table.isValidRelatedTable()) { + boolean validRelatedTable = validRelatedTableOverride == null + ? table.isValidRelatedTable() : validRelatedTableOverride; + if (!validRelatedTable) { icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, projectionTable, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 81d08123ec8315..d3284942637ba9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -148,10 +148,6 @@ public Table getIcebergTable() { return IcebergUtils.getIcebergTable(this); } - public Table getWritableIcebergTable() { - return IcebergUtils.getWritableIcebergTable(this); - } - @Override public String getComment() { return properties().getOrDefault(TABLE_COMMENT_PROP, ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRuntimeContext.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRuntimeContext.java new file mode 100644 index 00000000000000..4ee0ffbef557a0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRuntimeContext.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; + +/** Immutable execution state retained together with one Iceberg table generation. */ +public final class IcebergRuntimeContext { + private final ExecutionAuthenticator authenticator; + private final ThreadPoolExecutor planningExecutor; + private final MetastoreProperties metastoreProperties; + private final Map storageProperties; + + IcebergRuntimeContext(ExecutionAuthenticator authenticator, ThreadPoolExecutor planningExecutor, + MetastoreProperties metastoreProperties, + Map storageProperties) { + this.authenticator = authenticator; + this.planningExecutor = planningExecutor; + this.metastoreProperties = metastoreProperties; + this.storageProperties = ImmutableMap.copyOf(storageProperties); + } + + public ExecutionAuthenticator getAuthenticator() { + return authenticator; + } + + public ThreadPoolExecutor getPlanningExecutor() { + return planningExecutor; + } + + public MetastoreProperties getMetastoreProperties() { + return metastoreProperties; + } + + public Map getStorageProperties() { + return storageProperties; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index df94a648221e48..429f0ad2852107 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -63,6 +63,8 @@ public class IcebergSnapshotCacheValue { */ @Nullable private transient volatile ExecutionAuthenticator capturedAuthenticator; + @Nullable + private transient volatile IcebergRuntimeContext runtimeContext; private transient volatile boolean enableMappingVarbinary; private transient volatile boolean enableMappingTimestampTz; @@ -138,6 +140,11 @@ public IcebergSnapshotCacheValue bindCapturedAuthenticator(@Nullable ExecutionAu return this; } + public IcebergSnapshotCacheValue bindRuntimeContext(@Nullable IcebergRuntimeContext runtimeContext) { + this.runtimeContext = runtimeContext; + return this; + } + public IcebergSnapshotCacheValue bindSchemaMappingOptions( boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { this.enableMappingVarbinary = enableMappingVarbinary; @@ -150,6 +157,11 @@ public ExecutionAuthenticator getCapturedAuthenticator() { return capturedAuthenticator; } + @Nullable + public IcebergRuntimeContext getRuntimeContext() { + return runtimeContext; + } + public boolean isEnableMappingVarbinary() { return enableMappingVarbinary; } @@ -178,6 +190,10 @@ public Optional>> getNameMapping() { return nameMapping; } + IcebergSnapshotCacheValue withoutRetainedTable() { + return new IcebergSnapshotCacheValue(partitionInfo, snapshot, nameMapping); + } + public Optional
getIcebergTable() { return queryIsolationPrepared ? icebergTable.map(table -> createQueryScopedTable( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index 3129693ba0f426..cb225930c72237 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -49,6 +49,8 @@ public class IcebergTableCacheValue { // counterpart for the concurrent catalog-reset rationale. @Nullable private volatile org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator; + @Nullable + private volatile IcebergRuntimeContext runtimeContext; private volatile boolean enableMappingVarbinary; private volatile boolean enableMappingTimestampTz; private String retainedCurrentSnapshotJson; @@ -140,6 +142,10 @@ void bindAuthenticator( this.authenticator = authenticator; } + void bindRuntimeContext(IcebergRuntimeContext runtimeContext) { + this.runtimeContext = runtimeContext; + } + void bindSchemaMappingOptions(boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { this.enableMappingVarbinary = enableMappingVarbinary; this.enableMappingTimestampTz = enableMappingTimestampTz; @@ -165,6 +171,11 @@ org.apache.doris.common.security.authentication.ExecutionAuthenticator getAuthen return authenticator; } + @Nullable + IcebergRuntimeContext getRuntimeContext() { + return runtimeContext; + } + boolean isEnableMappingVarbinary() { return enableMappingVarbinary; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 240f7ba6411ad2..435d8af7d5d4d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -21,6 +21,7 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper; @@ -87,6 +88,7 @@ public class IcebergTransaction implements Transaction { private final IcebergMetadataOps ops; private Table table; + private IcebergExternalMetaCache.WritableTableLease writableTableLease; private org.apache.iceberg.Transaction transaction; private final List commitDataList = Lists.newArrayList(); @@ -148,7 +150,8 @@ public void beginInsert(ExternalTable dorisTable, Table targetTable, this.writeSchemaContext = insertCtx == null ? Optional.empty() : insertCtx.getWriteSchemaContext(); try { - ops.getExecutionAuthenticator().execute(() -> { + acquireWritableTableLease(dorisTable); + writableTableLease.getAuthenticator().execute(() -> { // Planning, BE serialization, and commit must share one Iceberg metadata // generation even if the catalog refreshes between those phases. this.table = createTransactionTable(dorisTable, targetTable); @@ -176,6 +179,7 @@ public void beginInsert(ExternalTable dorisTable, Table targetTable, this.rewrittenDeleteFilesByReferencedDataFile = Collections.emptyMap(); }); } catch (Exception e) { + releaseWritableTableLease(); throw new UserException("Failed to begin insert for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } @@ -184,11 +188,17 @@ public void beginInsert(ExternalTable dorisTable, Table targetTable, /** Begin an insert when no statement-retained table is available. */ public void beginInsert(ExternalTable dorisTable, Optional ctx) throws UserException { - beginInsert(dorisTable, IcebergUtils.getWritableIcebergTable(dorisTable), ctx); + beginInsert(dorisTable, null, ctx); } /** Begin a rewrite against the same retained table used by every rewrite task. */ public void beginRewrite(ExternalTable dorisTable, Table targetTable) throws UserException { + beginRewrite(dorisTable, targetTable, null); + } + + /** Begin a rewrite using the generation already retained by the enclosing action. */ + public void beginRewrite(ExternalTable dorisTable, Table targetTable, + IcebergExternalMetaCache.WritableTableLease retainedLease) throws UserException { // For rewrite operations, we work directly on the main table this.branchName = null; this.isRewriteMode = true; @@ -196,7 +206,16 @@ public void beginRewrite(ExternalTable dorisTable, Table targetTable) throws Use this.writeSchemaContext = Optional.empty(); try { - ops.getExecutionAuthenticator().execute(() -> { + if (retainedLease == null) { + acquireWritableTableLease(dorisTable); + } else { + Preconditions.checkState(writableTableLease == null, + "Writable Iceberg table lease is already acquired for %s", dorisTable.getName()); + Preconditions.checkState(retainedLease.getOps() == ops, + "Iceberg catalog runtime changed before rewrite transaction started"); + writableTableLease = retainedLease; + } + writableTableLease.getAuthenticator().execute(() -> { // A rewrite group must not silently switch to refreshed metadata mid-transaction. this.table = targetTable; this.baseSnapshotId = null; @@ -213,6 +232,7 @@ public void beginRewrite(ExternalTable dorisTable, Table targetTable) throws Use return null; }); } catch (Exception e) { + releaseWritableTableLease(); throw new UserException("Failed to begin rewrite for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } @@ -223,16 +243,13 @@ public void beginRewrite(ExternalTable dorisTable, Table targetTable) throws Use * API */ public void finishRewrite() { - // TODO: refactor IcebergTransaction to make code cleaner - convertCommitDataListToDataFilesToAdd(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Finishing rewrite with {} files to delete and {} files to add", - filesToDelete.size(), filesToAdd.size()); - } - try { - ops.getExecutionAuthenticator().execute(() -> { + writableAuthenticator().execute(() -> { + convertCommitDataListToDataFilesToAdd(); + if (LOG.isDebugEnabled()) { + LOG.debug("Finishing rewrite with {} files to delete and {} files to add", + filesToDelete.size(), filesToAdd.size()); + } updateManifestAfterRewrite(); return null; }); @@ -304,7 +321,8 @@ public void beginDelete(ExternalTable dorisTable, Table targetTable) throws User this.isRewriteMode = false; this.hasStagedUpdates = false; try { - ops.getExecutionAuthenticator().execute(() -> { + acquireWritableTableLease(dorisTable); + writableTableLease.getAuthenticator().execute(() -> { // RowDelta's validation base must match the generation used to select row IDs; // reloading the live table here could silently include a concurrent commit. this.table = createTransactionTable(dorisTable, targetTable); @@ -322,16 +340,22 @@ public void beginDelete(ExternalTable dorisTable, Table targetTable) throws User LOG.info("Started delete transaction for table: {}", dorisTable.getName()); }); } catch (Exception e) { + releaseWritableTableLease(); throw new UserException("Failed to begin delete for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } } public void beginDelete(ExternalTable dorisTable) throws UserException { - beginDelete(dorisTable, IcebergUtils.getWritableIcebergTable(dorisTable)); + beginDelete(dorisTable, null); } private Table createTransactionTable(ExternalTable dorisTable, Table retainedTable) { + Preconditions.checkState(writableTableLease != null, + "Writable Iceberg table lease is not acquired for %s", dorisTable.getName()); + if (retainedTable == null) { + return writableTableLease.getTable(); + } if (!IcebergSnapshotCacheValue.isRetainedGeneration(retainedTable)) { return retainedTable; } @@ -342,7 +366,26 @@ private Table createTransactionTable(ExternalTable dorisTable, Table retainedTab // generation's table here would splice it with the retained ops/authenticator this // transaction keeps using for begin/update/commit. return IcebergSnapshotCacheValue.createWritableTable( - retainedTable, IcebergUtils.getWritableIcebergTable(dorisTable, ops)); + retainedTable, writableTableLease.getTable()); + } + + private void acquireWritableTableLease(ExternalTable dorisTable) { + Preconditions.checkState(writableTableLease == null, + "Writable Iceberg table lease is already acquired for %s", dorisTable.getName()); + writableTableLease = IcebergUtils.acquireWritableIcebergTable(dorisTable, ops); + } + + private void releaseWritableTableLease() { + if (writableTableLease != null) { + writableTableLease.close(); + writableTableLease = null; + } + } + + private ExecutionAuthenticator writableAuthenticator() { + Preconditions.checkState(writableTableLease != null, + "Writable Iceberg table lease is not acquired"); + return writableTableLease.getAuthenticator(); } /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ @@ -354,12 +397,12 @@ public void beginMerge(ExternalTable dorisTable, Table targetTable) throws UserE * Begin merge operation for Iceberg UPDATE (single scan RowDelta). */ public void beginMerge(ExternalTable dorisTable) throws UserException { - beginMerge(dorisTable, IcebergUtils.getWritableIcebergTable(dorisTable), Optional.empty()); + beginMerge(dorisTable, null, Optional.empty()); } /** Begin a merge transaction after validating its statement-pinned write schema. */ public void beginMerge(ExternalTable dorisTable, Optional ctx) throws UserException { - beginMerge(dorisTable, IcebergUtils.getWritableIcebergTable(dorisTable), ctx); + beginMerge(dorisTable, null, ctx); } /** Begin a merge against retained metadata and a statement-pinned write schema. */ @@ -369,7 +412,8 @@ public void beginMerge(ExternalTable dorisTable, Table targetTable, this.writeSchemaContext = insertCtx == null ? Optional.empty() : insertCtx.getWriteSchemaContext(); try { - ops.getExecutionAuthenticator().execute(() -> { + acquireWritableTableLease(dorisTable); + writableTableLease.getAuthenticator().execute(() -> { this.branchName = null; this.isRewriteMode = false; // Keep row binding, partition routing, writer schema, and commit on one generation. @@ -393,6 +437,7 @@ public void beginMerge(ExternalTable dorisTable, Table targetTable, return null; }); } catch (Exception e) { + releaseWritableTableLease(); throw new UserException("Failed to begin merge for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } @@ -406,7 +451,7 @@ public void finishDelete(NameMapping nameMapping) { LOG.debug("iceberg table {} delete operation finished!", nameMapping.getFullLocalName()); } try { - ops.getExecutionAuthenticator().execute(() -> { + writableAuthenticator().execute(() -> { updateManifestAfterDelete(); }); } catch (Exception e) { @@ -423,7 +468,7 @@ public void finishMerge(NameMapping nameMapping) { LOG.debug("iceberg table {} merge operation finished!", nameMapping.getFullLocalName()); } try { - ops.getExecutionAuthenticator().execute(() -> { + writableAuthenticator().execute(() -> { updateManifestAfterMerge(); }); } catch (Exception e) { @@ -583,7 +628,7 @@ public void finishInsert(NameMapping nameMapping) { LOG.info("iceberg table {} insert table finished!", nameMapping.getFullLocalName()); } try { - ops.getExecutionAuthenticator().execute(() -> { + writableAuthenticator().execute(() -> { //create and start the iceberg transaction TUpdateMode updateMode = TUpdateMode.APPEND; if (insertCtx != null) { @@ -633,12 +678,24 @@ private WriteResult convertToWriterResult(List dataCommitDat @Override public void commit() throws UserException { - // Empty overwrites may intentionally stage no Iceberg update, so validate once more even - // when commitTransaction() has no metadata CAS to invoke the atomic operations fence. - if (!hasStagedUpdates) { - validatePinnedWriterMetadata(); + try { + Preconditions.checkState(writableTableLease != null, + "Writable Iceberg table lease was released before commit"); + writableTableLease.getAuthenticator().execute(() -> { + // Empty overwrites may intentionally stage no Iceberg update, so validate once more even + // when commitTransaction() has no metadata CAS to invoke the atomic operations fence. + if (!hasStagedUpdates) { + validatePinnedWriterMetadata(); + } + transaction.commitTransaction(); + }); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new UserException("Failed to commit Iceberg transaction: " + e.getMessage(), e); + } finally { + releaseWritableTableLease(); } - transaction.commitTransaction(); } /** @@ -770,17 +827,21 @@ public boolean requireStrictCleanup() { @Override public void rollback() { - if (isRewriteMode) { - // Clear the collected files for rewrite mode - synchronized (filesToDelete) { - filesToDelete.clear(); - } - synchronized (filesToAdd) { - filesToAdd.clear(); + try { + if (isRewriteMode) { + // Clear the collected files for rewrite mode + synchronized (filesToDelete) { + filesToDelete.clear(); + } + synchronized (filesToAdd) { + filesToAdd.clear(); + } + LOG.info("Rewrite transaction rolled back"); } - LOG.info("Rewrite transaction rolled back"); + // For insert mode, do nothing as original implementation + } finally { + releaseWritableTableLease(); } - // For insert mode, do nothing as original implementation } public long getUpdateCnt() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index e6e935578d23c7..e7e3a5806d62b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1146,26 +1146,38 @@ public static Table getQueryScopedIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getQueryScopedIcebergTable(dorisTable); } - public static Table getWritableIcebergTable(ExternalTable dorisTable) { + @VisibleForTesting + static Table getWritableIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); } /** Writable acquisition anchored to the caller's retained catalog generation. */ - public static Table getWritableIcebergTable(ExternalTable dorisTable, IcebergMetadataOps expectedOps) { + @VisibleForTesting + static Table getWritableIcebergTable(ExternalTable dorisTable, IcebergMetadataOps expectedOps) { return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable, expectedOps); } - static IcebergExternalMetaCache.WritableTableLease acquireWritableIcebergTable( + public static IcebergExternalMetaCache.WritableTableLease acquireWritableIcebergTable( + ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).acquireWritableIcebergTable(dorisTable); + } + + public static IcebergExternalMetaCache.WritableTableLease acquireWritableIcebergTable( ExternalTable dorisTable, IcebergMetadataOps expectedOps) { return icebergExternalMetaCache(dorisTable).acquireWritableIcebergTable(dorisTable, expectedOps); } + public static IcebergSnapshotCacheValue getSnapshotForWritableLease( + ExternalTable dorisTable, IcebergExternalMetaCache.WritableTableLease lease) { + return icebergExternalMetaCache(dorisTable).getSnapshotForWritableLease(dorisTable, lease); + } + public static ThreadPoolExecutor getIcebergTableExecutor(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTableExecutor(dorisTable); } /** The action must return derived metadata rather than retain the supplied table. */ - static T withIcebergTable(ExternalTable dorisTable, Function action) { + public static T withIcebergTable(ExternalTable dorisTable, Function action) { return icebergExternalMetaCache(dorisTable).withIcebergTable(dorisTable, action); } @@ -2278,7 +2290,7 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( /** * An explicit VERSION/TIME or branch/tag relation retains its query-scoped table exactly like * a latest projection retains the frozen generation, so the value must carry the generation's - * captured execution context for the planning-time fence in IcebergScanNode. + * captured execution context so planning never mixes this generation with live catalog state. */ static IcebergSnapshotCacheValue newExplicitSnapshotValue( IcebergTableQueryInfo info, Table queryScopedTable, IcebergTableCacheValue generation) { @@ -2287,6 +2299,7 @@ static IcebergSnapshotCacheValue newExplicitSnapshotValue( new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), getNameMapping(queryScopedTable), queryScopedTable) .bindCapturedAuthenticator(generation.getAuthenticator()) + .bindRuntimeContext(generation.getRuntimeContext()) .bindSchemaMappingOptions(generation.isEnableMappingVarbinary(), generation.isEnableMappingTimestampTz()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/BaseIcebergAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/BaseIcebergAction.java index b54d7509f0056b..e537cbd91564cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/BaseIcebergAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/BaseIcebergAction.java @@ -19,11 +19,16 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache.WritableTableLease; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.commands.execute.BaseExecuteAction; +import org.apache.iceberg.Table; + +import java.util.List; import java.util.Map; import java.util.Optional; @@ -71,4 +76,28 @@ protected void validateIcebergAction() throws UserException { // Default implementation does nothing. } + @Override + protected final List executeAction(TableIf table) throws UserException { + try (WritableTableLease lease = IcebergUtils.acquireWritableIcebergTable((IcebergExternalTable) table)) { + return lease.getAuthenticator().execute( + () -> executeIcebergAction(table, lease)); + } catch (UserException e) { + throw e; + } catch (Exception e) { + throw new UserException("Failed to execute Iceberg action: " + e.getMessage(), e); + } + } + + /** Override when an action needs to pass the exact writable generation into nested work. */ + protected List executeIcebergAction(TableIf table, WritableTableLease lease) + throws UserException { + return executeIcebergAction(table, lease.getTable()); + } + + /** Execute an action while the writable table's exact catalog generation remains retained. */ + protected List executeIcebergAction(TableIf table, Table icebergTable) + throws UserException { + throw new UnsupportedOperationException("Iceberg action must implement an execution hook"); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java index 94924514a58a48..d6343eefd68d11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java @@ -24,7 +24,6 @@ import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -69,8 +68,7 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { Long sourceSnapshotId = namedArguments.getLong(SNAPSHOT_ID); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java index 82a93022354067..c5ceb590f24f41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java @@ -25,7 +25,6 @@ import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -148,9 +147,7 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); - + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { // Parse parameters String olderThan = namedArguments.getString(OLDER_THAN); Integer retainLast = namedArguments.getInt(RETAIN_LAST); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java index cd746a7dbe6959..d2108a50d400c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java @@ -24,7 +24,6 @@ import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -69,9 +68,7 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); - + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { String sourceBranch = namedArguments.getString(BRANCH); String desBranch = namedArguments.getString(TO); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java index bf3f116d1cba81..bf54b2ef23bd13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java @@ -24,7 +24,6 @@ import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -65,8 +64,7 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { String targetWapId = namedArguments.getString(WAP_ID); // Find the target WAP snapshot diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteDataFilesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteDataFilesAction.java index a22397a146b379..ad9db487dad8be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteDataFilesAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteDataFilesAction.java @@ -22,8 +22,8 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache.WritableTableLease; import org.apache.doris.datasource.iceberg.IcebergExternalTable; -import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.rewrite.RewriteDataFileExecutor; import org.apache.doris.datasource.iceberg.rewrite.RewriteDataFilePlanner; import org.apache.doris.datasource.iceberg.rewrite.RewriteDataGroup; @@ -168,10 +168,9 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { + protected List executeIcebergAction(TableIf table, WritableTableLease lease) throws UserException { + Table icebergTable = lease.getTable(); try { - Table icebergTable = IcebergUtils.getIcebergTable((IcebergExternalTable) table); - if (icebergTable.currentSnapshot() == null) { LOG.info("Table {} has no data, skipping rewrite", table.getName()); // return empty result @@ -196,7 +195,8 @@ protected List executeAction(TableIf table) throws UserException { RewriteDataFileExecutor executor = new RewriteDataFileExecutor( (IcebergExternalTable) table, connectContext); long targetFileSizeBytes = namedArguments.getLong(TARGET_FILE_SIZE_BYTES); - RewriteResult totalResult = executor.executeGroupsConcurrently(groupsList, targetFileSizeBytes); + RewriteResult totalResult = executor.executeGroupsConcurrently( + groupsList, targetFileSizeBytes, lease); return totalResult.toStringList(); } catch (Exception e) { LOG.warn("Failed to rewrite data files for table: " + table.getName(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java index dce45c2729693b..8303180331dadd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java @@ -23,7 +23,6 @@ import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.rewrite.RewriteManifestExecutor; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -66,9 +65,8 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { try { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot current = icebergTable.currentSnapshot(); if (current == null) { // No current snapshot means the table is empty, no manifests to rewrite diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java index a5609f83439d45..cee5b8916657ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java @@ -24,7 +24,6 @@ import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -67,8 +66,7 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { Long targetSnapshotId = namedArguments.getLong(SNAPSHOT_ID); Snapshot targetSnapshot = icebergTable.snapshot(targetSnapshotId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java index de7e2a680791c2..0c2406fe2fd885 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java @@ -24,7 +24,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -95,9 +94,7 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); - + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { String timestampStr = namedArguments.getString(TIMESTAMP); Snapshot previousSnapshot = icebergTable.currentSnapshot(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java index 5b2c5bd220eb7f..036599a34b7b76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java @@ -25,7 +25,6 @@ import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -86,9 +85,7 @@ protected void validateIcebergAction() throws UserException { } @Override - protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); - + protected List executeIcebergAction(TableIf table, Table icebergTable) throws UserException { Snapshot previousSnapshot = icebergTable.currentSnapshot(); Long previousSnapshotId = previousSnapshot != null ? previousSnapshot.snapshotId() : null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java index 095bad988849ef..c99b44b9b3dabd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java @@ -19,9 +19,11 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache.WritableTableLease; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.qe.ConnectContext; import org.apache.doris.resource.computegroup.ComputeGroup; @@ -31,12 +33,10 @@ import com.google.common.collect.Lists; // Keep third-party imports lexical to preserve the repository's CustomImportOrder invariant. -import org.apache.iceberg.Table; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -59,82 +59,91 @@ public RewriteDataFileExecutor(IcebergExternalTable dorisTable, /** * Execute rewrite for multiple groups concurrently */ - public RewriteResult executeGroupsConcurrently(List groups, long targetFileSizeBytes) + public RewriteResult executeGroupsConcurrently(List groups, long targetFileSizeBytes, + WritableTableLease writableTableLease) throws UserException { // Begin transaction long transactionId = dorisTable.getCatalog().getTransactionManager().begin(); IcebergTransaction transaction = (IcebergTransaction) dorisTable.getCatalog().getTransactionManager() .getTransaction(transactionId); - MvccSnapshot targetSnapshot = dorisTable.loadSnapshot(Optional.empty(), Optional.empty()); - Table targetIcebergTable = ((IcebergMvccSnapshot) targetSnapshot).getSnapshotCacheValue() - .getIcebergTable().orElseThrow( - () -> new UserException("Iceberg rewrite target metadata is not available")); - transaction.beginRewrite(dorisTable, targetIcebergTable); - - // Register files to delete - for (RewriteDataGroup group : groups) { - transaction.updateRewriteFiles(Lists.newArrayList(group.getDataFiles())); - } - - // Create result collector and tasks + MvccSnapshot targetSnapshot = new IcebergMvccSnapshot( + IcebergUtils.getSnapshotForWritableLease(dorisTable, writableTableLease)); List tasks = Lists.newArrayList(); RewriteResultCollector resultCollector = new RewriteResultCollector(groups.size(), tasks); - - // Get available BE count once before creating tasks - // This avoids calling getBackendsNumber() in each task during multi-threaded execution. - // Use compute group from connect context to align with actual BE selection for queries. - int availableBeCount = getAvailableBeCount(); - - // Create tasks with callbacks - for (RewriteDataGroup group : groups) { - RewriteGroupTask task = new RewriteGroupTask( - group, - transactionId, - dorisTable, - targetSnapshot, - connectContext, - targetFileSizeBytes, - availableBeCount, - new RewriteGroupTask.RewriteResultCallback() { - @Override - public void onTaskCompleted(Long taskId) { - resultCollector.onTaskCompleted(taskId); - } - - @Override - public void onTaskFailed(Long taskId, Exception error) { - resultCollector.onTaskFailed(taskId, error); - } - }); - tasks.add(task); - } - - // Submit tasks to TransientTaskManager + boolean committed = false; try { - for (TransientTaskExecutor task : tasks) { - Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(task); - } - } catch (JobException e) { - throw new UserException("Failed to submit rewrite tasks: " + e.getMessage(), e); - } - - // Wait for all tasks to complete - waitForTasksCompletion(resultCollector, groups.size()); + transaction.beginRewrite(dorisTable, writableTableLease.getTable(), writableTableLease); - // Finish rewrite operation - transaction.finishRewrite(); + // Register files to delete + for (RewriteDataGroup group : groups) { + transaction.updateRewriteFiles(Lists.newArrayList(group.getDataFiles())); + } - // Collect statistics from transaction after all tasks are completed - int rewrittenDataFilesCount = groups.stream().mapToInt(group -> group.getDataFiles().size()).sum(); - // this should after finishRewrite - int addedDataFilesCount = transaction.getFilesToAddCount(); - long rewrittenBytesCount = groups.stream().mapToLong(group -> group.getTotalSize()).sum(); - int removedDeleteFilesCount = groups.stream().mapToInt(group -> group.getDeleteFileCount()).sum(); + // Create result collector and tasks + // Get available BE count once before creating tasks + // This avoids calling getBackendsNumber() in each task during multi-threaded execution. + // Use compute group from connect context to align with actual BE selection for queries. + int availableBeCount = getAvailableBeCount(); + + // Create tasks with callbacks + for (RewriteDataGroup group : groups) { + RewriteGroupTask task = new RewriteGroupTask( + group, + transactionId, + dorisTable, + targetSnapshot, + writableTableLease.retain(), + connectContext, + targetFileSizeBytes, + availableBeCount, + new RewriteGroupTask.RewriteResultCallback() { + @Override + public void onTaskCompleted(Long taskId) { + resultCollector.onTaskCompleted(taskId); + } + + @Override + public void onTaskFailed(Long taskId, Exception error) { + resultCollector.onTaskFailed(taskId, error); + } + }); + tasks.add(task); + } - commitAndInvalidate(transaction); + // Submit tasks to TransientTaskManager + try { + for (TransientTaskExecutor task : tasks) { + Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(task); + } + } catch (JobException e) { + throw new UserException("Failed to submit rewrite tasks: " + e.getMessage(), e); + } - return new RewriteResult(rewrittenDataFilesCount, addedDataFilesCount, - rewrittenBytesCount, removedDeleteFilesCount); + // Wait for all tasks to complete + waitForTasksCompletion(resultCollector, groups.size()); + + // Finish rewrite operation + transaction.finishRewrite(); + + // Collect statistics from transaction after all tasks are completed + int rewrittenDataFilesCount = groups.stream().mapToInt(group -> group.getDataFiles().size()).sum(); + // this should after finishRewrite + int addedDataFilesCount = transaction.getFilesToAddCount(); + long rewrittenBytesCount = groups.stream().mapToLong(group -> group.getTotalSize()).sum(); + int removedDeleteFilesCount = groups.stream().mapToInt(group -> group.getDeleteFileCount()).sum(); + + transaction.commit(); + committed = true; + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable); + + return new RewriteResult(rewrittenDataFilesCount, addedDataFilesCount, + rewrittenBytesCount, removedDeleteFilesCount); + } finally { + if (!committed) { + resultCollector.cancelAllTasks(); + transaction.rollback(); + } + } } void commitAndInvalidate(IcebergTransaction transaction) throws UserException { @@ -191,7 +200,7 @@ private int getAvailableBeCount() throws UserException { /** * Result collector for concurrent rewrite tasks */ - private static class RewriteResultCollector { + static class RewriteResultCollector { private final int expectedTasks; private final AtomicInteger completedTasks = new AtomicInteger(0); private final AtomicInteger failedTasks = new AtomicInteger(0); @@ -227,16 +236,29 @@ public synchronized void onTaskFailed(Long taskId, Exception error) { private void cancelAllOtherTasks(Long failedTaskId) { for (RewriteGroupTask task : allTasks) { if (!task.getId().equals(failedTaskId)) { - try { - task.cancel(); - LOG.info("Cancelled task {}", task.getId()); - } catch (Exception e) { - LOG.warn("Failed to cancel task {}: {}", task.getId(), e.getMessage()); - } + cancelTask(task); } } } + void cancelAllTasks() { + for (RewriteGroupTask task : allTasks) { + cancelTask(task); + } + } + + private void cancelTask(RewriteGroupTask task) { + try { + // addMemoryTask registers before publishing. Remove first so a failed or + // suppressed publish cannot retain a table-bearing snapshot in the manager map. + Env.getCurrentEnv().getTransientTaskManager().removeMemoryTask(task.getId()); + task.cancel(); + LOG.info("Cancelled task {}", task.getId()); + } catch (Exception e) { + LOG.warn("Failed to cancel task {}: {}", task.getId(), e.getMessage()); + } + } + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { return completionLatch.await(timeout, unit); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java index 6476f922bdc970..f89fb5d410e7be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Env; import org.apache.doris.common.Status; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache.WritableTableLease; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccTableInfo; @@ -61,21 +62,24 @@ public class RewriteGroupTask implements TransientTaskExecutor { private final long transactionId; private final IcebergExternalTable dorisTable; private final MvccSnapshot targetSnapshot; + private final WritableTableLease writableTableLease; private final ConnectContext connectContext; private final long targetFileSizeBytes; private final RewriteResultCallback resultCallback; private final Long taskId; private final AtomicBoolean isCanceled; private final AtomicBoolean isFinished; + private final AtomicBoolean executionClaimed; private final int availableBeCount; // for canceling the task - private StmtExecutor stmtExecutor; + private volatile StmtExecutor stmtExecutor; public RewriteGroupTask(RewriteDataGroup group, long transactionId, IcebergExternalTable dorisTable, MvccSnapshot targetSnapshot, + WritableTableLease writableTableLease, ConnectContext connectContext, long targetFileSizeBytes, int availableBeCount, @@ -84,6 +88,7 @@ public RewriteGroupTask(RewriteDataGroup group, this.transactionId = transactionId; this.dorisTable = dorisTable; this.targetSnapshot = targetSnapshot; + this.writableTableLease = writableTableLease; this.connectContext = connectContext; this.targetFileSizeBytes = targetFileSizeBytes; this.availableBeCount = availableBeCount; @@ -91,6 +96,7 @@ public RewriteGroupTask(RewriteDataGroup group, this.taskId = UUID.randomUUID().getMostSignificantBits(); this.isCanceled = new AtomicBoolean(false); this.isFinished = new AtomicBoolean(false); + this.executionClaimed = new AtomicBoolean(false); } // Tests that only exercise scheduling strategy do not create an Iceberg metadata snapshot. @@ -101,7 +107,7 @@ public RewriteGroupTask(RewriteDataGroup group, long targetFileSizeBytes, int availableBeCount, RewriteResultCallback resultCallback) { - this(group, transactionId, dorisTable, null, connectContext, targetFileSizeBytes, + this(group, transactionId, dorisTable, null, null, connectContext, targetFileSizeBytes, availableBeCount, resultCallback); } @@ -115,18 +121,16 @@ public void execute() throws JobException { LOG.debug("[Rewrite Task] taskId: {} starting execution for group with {} tasks", taskId, group.getTaskCount()); - if (isCanceled.get()) { - LOG.debug("[Rewrite Task] taskId: {} was already canceled before execution", taskId); - throw new JobException("Rewrite task has been canceled, task id: " + taskId); - } - - if (isFinished.get()) { - LOG.debug("[Rewrite Task] taskId: {} was already finished", taskId); + if (!executionClaimed.compareAndSet(false, true)) { + LOG.debug("[Rewrite Task] taskId: {} was canceled before execution", taskId); return; } ConnectContext taskConnectContext = null; try { + if (isCanceled.get()) { + throw new JobException("Rewrite task has been canceled, task id: " + taskId); + } // Step 1: Create and customize a new ConnectContext for this task taskConnectContext = buildConnectContext(); // Set target file size for Iceberg write @@ -168,6 +172,9 @@ public void execute() throws JobException { } finally { ConnectContext.remove(); isFinished.set(true); + if (writableTableLease != null) { + writableTableLease.close(); + } } } } @@ -180,6 +187,17 @@ public void cancel() throws JobException { } isCanceled.set(true); + if (executionClaimed.compareAndSet(false, true)) { + JobException error = new JobException("Rewrite task has been canceled, task id: " + taskId); + if (resultCallback != null) { + resultCallback.onTaskFailed(taskId, error); + } + isFinished.set(true); + if (writableTableLease != null) { + writableTableLease.close(); + } + return; + } if (stmtExecutor != null) { stmtExecutor.cancel(new Status(TStatusCode.CANCELLED, "rewrite task cancelled")); } @@ -194,6 +212,9 @@ private void executeGroup(ConnectContext taskConnectContext, StatementBase taskParsedStmt) throws Exception { // Step 1: Create stmt executor stmtExecutor = new StmtExecutor(taskConnectContext, taskParsedStmt); + if (isCanceled.get()) { + throw new JobException("Rewrite task has been canceled, task id: " + taskId); + } // Step 2: Create insert executor AbstractInsertExecutor insertExecutor = taskLogicalPlan.initPlan(taskConnectContext, stmtExecutor); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 2a45a1b749f794..3d8851201ba421 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -48,6 +48,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergRuntimeContext; import org.apache.doris.datasource.iceberg.IcebergSnapshot; import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; @@ -196,6 +197,7 @@ public class IcebergScanNode extends FileQueryScanNode { private boolean isPartitionedTable; private int formatVersion; private ExecutionAuthenticator preExecutionAuthenticator; + private IcebergRuntimeContext runtimeContext; private TableScan icebergTableScan; private Schema querySchema; // Store PropertiesMap, including vended credentials or static credentials @@ -307,12 +309,16 @@ protected void doInitialize() throws UserException { // These tables are always readable regardless of format version formatVersion = MIN_DELETE_FILE_SUPPORT_VERSION; } - preExecutionAuthenticator = source.getCatalog().getExecutionAuthenticator(); - storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( - source.getCatalog().getCatalogProperty().getMetastoreProperties(), - source.getCatalog().getCatalogProperty().getStoragePropertiesMap(), - icebergTable - ); + if (runtimeContext == null) { + preExecutionAuthenticator = source.getCatalog().getExecutionAuthenticator(); + storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( + source.getCatalog().getCatalogProperty().getMetastoreProperties(), + source.getCatalog().getCatalogProperty().getStoragePropertiesMap(), icebergTable); + } else { + preExecutionAuthenticator = runtimeContext.getAuthenticator(); + storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( + runtimeContext.getMetastoreProperties(), runtimeContext.getStorageProperties(), icebergTable); + } storagePropertiesMap = IcebergUtils.selectEffectiveStorageProperties(storagePropertiesMap); backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); } finally { @@ -1754,7 +1760,7 @@ public TableScan createTableScan() throws UserException { this.pushdownIcebergPredicates.add(predicate.toString()); } - icebergTableScan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); + icebergTableScan = scan.planWith(getPlanningExecutor()); return icebergTableScan; } @@ -1766,18 +1772,7 @@ private Table useFrozenTableGeneration(Table currentTable) { ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); Optional
frozenTable = cacheValue.getIcebergTable(); if (frozenTable.isPresent()) { - // Planning the frozen generation (regular relations and snapshot-selectable - // system tables alike) uses the catalog's current authenticator, storage state - // and pre-authenticated executor. Those are only coherent with the retained - // frozen operations/FileIO while the catalog still serves the generation this - // statement pinned; after a credential/storage ALTER the statement must fail and - // be retried. Count-mode values retain no frozen handle and plan the live table, - // so they are not fenced; values without a captured context resolve nothing here. - if (cacheValue.getCapturedAuthenticator() != null) { - cacheValue.ensurePlannableUnder( - source.getCatalog().getExecutionAuthenticator(), - source.getTargetTable().getName()); - } + runtimeContext = cacheValue.getRuntimeContext(); Table frozenBaseTable = frozenTable.get(); if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); @@ -1799,6 +1794,12 @@ private Table useFrozenTableGeneration(Table currentTable) { return currentTable; } + private java.util.concurrent.ExecutorService getPlanningExecutor() { + return runtimeContext == null + ? source.getCatalog().getThreadPoolWithPreAuth() + : runtimeContext.getPlanningExecutor(); + } + @VisibleForTesting Schema getSystemTableProjectedSchema(List expressions, boolean caseSensitive) throws UserException { @@ -2701,7 +2702,7 @@ private List doGetPositionDeletesSystemTableSplits() throws UserException } long startTime = System.currentTimeMillis(); - scan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); + scan = scan.planWith(getPlanningExecutor()); BatchScan plannedScan = scan; try { positionDeleteTasks = getOrPlanPositionDeleteTasks(plannedScan, () -> { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalCatalogLifecycleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalCatalogLifecycleTest.java new file mode 100644 index 00000000000000..d987063980cb4c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalCatalogLifecycleTest.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.hudi.HudiExternalMetaCache; +import org.apache.doris.mysql.privilege.AccessControllerManager; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.function.Supplier; + +public class HMSExternalCatalogLifecycleTest { + + @Test + public void testScanRuntimeAndResetUseLifecycleFenceBeforeCatalogMonitor() { + TestCatalog catalog = new TestCatalog(); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getAccessManager()).thenReturn(Mockito.mock(AccessControllerManager.class)); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HudiExternalMetaCache hudiCache = Mockito.mock(HudiExternalMetaCache.class); + HudiExternalMetaCache.FsViewGeneration fsViewGeneration = + Mockito.mock(HudiExternalMetaCache.FsViewGeneration.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(cacheMgr.hudi(catalog.getId())).thenAnswer(invocation -> { + Assert.assertTrue("Hudi cache lookup must be inside the lifecycle fence", + catalog.lifecycleFenceHeld); + return hudiCache; + }); + Mockito.when(hudiCache.captureFsViewGeneration(catalog.getId())) + .thenReturn(fsViewGeneration); + Mockito.when(cacheMgr.withCatalogLifecycleLock( + Mockito.eq(catalog.getId()), Mockito.>any())) + .thenAnswer(invocation -> { + Assert.assertFalse("catalog monitor must not be held before lifecycle fence acquisition", + Thread.holdsLock(catalog)); + @SuppressWarnings("unchecked") + Supplier action = invocation.getArgument(1); + catalog.lifecycleFenceHeld = true; + try { + return action.get(); + } finally { + catalog.lifecycleFenceHeld = false; + } + }); + Mockito.doAnswer(invocation -> { + Assert.assertTrue("cache retirement must be inside the lifecycle fence", + catalog.lifecycleFenceHeld); + Assert.assertTrue("cache retirement must run while the catalog runtime is frozen", + Thread.holdsLock(catalog)); + return null; + }).when(cacheMgr).removeCatalogByEngine(Mockito.eq(catalog.getId()), Mockito.anyString()); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + HMSExternalCatalog.HudiScanRuntimeContext runtime = catalog.getHudiScanRuntimeContext(); + Assert.assertSame(catalog.authenticator, runtime.getAuthenticator()); + Assert.assertSame(fsViewGeneration, runtime.getFsViewGeneration()); + catalog.resetToUninitialized(false); + } + } + + private static final class TestCatalog extends HMSExternalCatalog { + private final ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { }; + private boolean lifecycleFenceHeld; + + private TestCatalog() { + super(7L, "hms-lifecycle", null, java.util.Collections.emptyMap(), ""); + initialized = true; + executionAuthenticator = authenticator; + } + + @Override + protected void initLocalObjectsImpl() { + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 66169e3a6ba38d..625e8b180fc802 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -632,7 +632,9 @@ protected CatalogIf getCatalog(long catalogId) { // Projections built from the published generation carry its execution context for // later planning-time validation. Assert.assertSame(authenticator, - cache.getSnapshotCache(dorisTable).getCapturedAuthenticator()); + cache.getSnapshotCacheWithRetainedTableForTest(dorisTable).getCapturedAuthenticator()); + Assert.assertFalse("background snapshots must not expose an unowned table", + cache.getSnapshotCache(dorisTable).getIcebergTable().isPresent()); initialized.set(false); Assert.assertSame(table, cache.getWritableIcebergTable(dorisTable)); @@ -965,7 +967,7 @@ MetaCacheSizeEstimate prepareTableForCachePublication( IcebergSchemaCacheKey.class, SchemaCacheValue.class); for (int i = 1; i <= 3; i++) { - IcebergSnapshotCacheValue projection = cache.getSnapshotCache(dorisTable); + IcebergSnapshotCacheValue projection = cache.getSnapshotCacheWithRetainedTableForTest(dorisTable); Assert.assertNotNull(projection); Assert.assertNull("rejected table handle must not be published", tables.peekIfPresent(mapping)); Assert.assertEquals("projections of an unpublished generation must be retired", @@ -1122,15 +1124,16 @@ MetaCacheSizeEstimate prepareTableForCachePublication( IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); - IcebergSnapshotCacheValue first = cache.getSnapshotCache(dorisTable); + IcebergSnapshotCacheValue first = cache.getSnapshotCacheWithRetainedTableForTest(dorisTable); Assert.assertNull(tables.peekIfPresent(mapping)); Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); // A new handle owns a distinct FileIO instance. Even with equal properties, retiring // that handle closes its exact IO, so the projection is rebound before retirement. - IcebergSnapshotCacheValue sameCredentialsProjection = cache.getSnapshotCache(dorisTable); + IcebergSnapshotCacheValue sameCredentialsProjection = + cache.getSnapshotCacheWithRetainedTableForTest(dorisTable); Assert.assertNotSame(first, sameCredentialsProjection); // Rotated credentials: the projection frozen on the first handle is rebuilt. - IcebergSnapshotCacheValue rebound = cache.getSnapshotCache(dorisTable); + IcebergSnapshotCacheValue rebound = cache.getSnapshotCacheWithRetainedTableForTest(dorisTable); Assert.assertNotSame(sameCredentialsProjection, rebound); Assert.assertSame(rotatedHandle.io(), rebound.getIcebergTable().get().io()); Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); @@ -1189,7 +1192,7 @@ MetaCacheSizeEstimate prepareTableForCachePublication( IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); - IcebergSnapshotCacheValue projection = cache.getSnapshotCache(dorisTable); + IcebergSnapshotCacheValue projection = cache.getSnapshotCacheWithRetainedTableForTest(dorisTable); IcebergTableCacheValue tableValue = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class).get(mapping); Assert.assertEquals(0, preparations.get()); @@ -2182,10 +2185,11 @@ public void testCountModeTimeTravelDoesNotEnableQueryIsolation() { ExternalTable table = Mockito.mock(IcebergExternalTable.class); Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); - Table queryTable = cache.getQueryScopedIcebergTable(table); - - Assert.assertSame(value.getRetainedIcebergTable(), queryTable); - Assert.assertFalse(value.isQueryIsolationPrepared()); + try (IcebergTableCacheValue.Lease lease = cache.borrowForTest(mapping)) { + Table queryTable = cache.createQueryScopedTable(table, lease.getValue()); + Assert.assertSame(value.getRetainedIcebergTable(), queryTable); + Assert.assertFalse(value.isQueryIsolationPrepared()); + } } finally { executor.shutdownNow(); } @@ -2221,13 +2225,15 @@ public void testTimeTravelGenerationBundleDoesNotMixReplacedTableValue() throws ExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); - Table queryTable = cache.getQueryScopedIcebergTable(dorisTable); - entry.put(mapping, new IcebergTableCacheValue(secondTable)); + try (IcebergTableCacheValue.Lease lease = cache.borrowForTest(mapping)) { + Table queryTable = cache.createQueryScopedTable(dorisTable, lease.getValue()); + entry.put(mapping, new IcebergTableCacheValue(secondTable)); - Assert.assertEquals(firstSnapshotId, - queryTable.currentSnapshot().snapshotId()); - Assert.assertEquals(firstTable.location(), - queryTable.location()); + Assert.assertEquals(firstSnapshotId, + queryTable.currentSnapshot().snapshotId()); + Assert.assertEquals(firstTable.location(), + queryTable.location()); + } } finally { executor.shutdownNow(); } @@ -2270,10 +2276,11 @@ protected CatalogIf getCatalog(long catalogId) { ExternalTable table = Mockito.mock(IcebergExternalTable.class); Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); - Table queryTable = cache.getQueryScopedIcebergTable(table); - - Assert.assertEquals(staleTable.schema().asStruct(), queryTable.schema().asStruct()); - Mockito.verify(metadataOps, Mockito.never()).loadTable("db", "tbl"); + try (IcebergTableCacheValue.Lease lease = cache.borrowForTest(mapping)) { + Table queryTable = cache.createQueryScopedTable(table, lease.getValue()); + Assert.assertEquals(staleTable.schema().asStruct(), queryTable.schema().asStruct()); + Mockito.verify(metadataOps, Mockito.never()).loadTable("db", "tbl"); + } } finally { cache.close(); executor.shutdownNow(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index 06497a6c14c811..462bb6fc7b8ff1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -78,6 +78,7 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; public class IcebergTransactionTest { @@ -210,8 +211,7 @@ public void testPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); // Allow parsePartitionValueFromString to call the real implementation mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -326,8 +326,7 @@ public void testUnPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(ctdList); @@ -343,6 +342,18 @@ private IcebergTransaction getTxn() { return new IcebergTransaction(ops); } + private void mockWritableTable(MockedStatic mockedUtils, Table table) { + mockedUtils.when(() -> IcebergUtils.acquireWritableIcebergTable( + ArgumentMatchers.any(ExternalTable.class), ArgumentMatchers.eq(ops))) + .thenAnswer(invocation -> new IcebergExternalMetaCache.WritableTableLease( + table, ops, testRuntimeContext(), false, false, () -> { })); + } + + private IcebergRuntimeContext testRuntimeContext() { + return new IcebergRuntimeContext(ops.getExecutionAuthenticator(), null, null, + Collections.emptyMap()); + } + @Test public void testSchemaSkewFailsBeforeOpeningInsertOrMergeTransaction() { Schema pinnedSchema = new Schema(90, @@ -363,9 +374,7 @@ public void testSchemaSkewFailsBeforeOpeningInsertOrMergeTransaction() { IcebergWriteSchemaContext.forSchema(pinnedSchema, 3, true, true))); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); mockedStatic.when(() -> IcebergUtils.getFormatVersion(table)).thenReturn(3); UserException insertException = Assert.assertThrows(UserException.class, @@ -406,9 +415,7 @@ public void testRecreatedTableFailsInsertOverwriteAndUpdateMergePreflight() { Mockito.when(dorisTable.getName()).thenReturn("recreated_table"); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(replacementTable); + mockWritableTable(mockedStatic, replacementTable); mockedStatic.when(() -> IcebergUtils.getFormatVersion(replacementTable)) .thenReturn(3); @@ -477,9 +484,7 @@ public void testStaticOverwriteRejectsConcurrentCurrentSpecReplacement() { insertContext.setWriteSchemaContext(Optional.of(context)); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); mockedStatic.when(() -> IcebergUtils.getFormatVersion(table)).thenReturn(3); mockedStatic.when(() -> IcebergUtils.dataLocation(table)).thenReturn(dataLocation); @@ -534,9 +539,7 @@ private void verifyDynamicOverwriteRejectsPartitionedToUnpartitionedSpecDrift( try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedUtils, table); txn.beginInsert(dorisTable, Optional.of(insertContext)); if (hasOutputFile) { txn.finishInsert(NameMapping.createForTest(dbName, tableName)); @@ -584,9 +587,7 @@ public void testCommitReplayRejectsRequiredSchemaChangeAfterStaging() throws Use txn.updateIcebergCommitData(Collections.singletonList(commitData)); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedUtils, table); txn.beginInsert(dorisTable, Optional.of(insertContext)); txn.finishInsert(NameMapping.createForTest(dbName, tableName)); @@ -629,9 +630,7 @@ public void testMergeCommitReplayRejectsRequiredSchemaChangeAfterStaging() throw IcebergTransaction txn = getTxn(); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedUtils, table); txn.beginMerge(dorisTable, Optional.of(insertContext)); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.finishMerge(NameMapping.createForTest(dbName, tableName)); @@ -677,9 +676,7 @@ public void testCommitRejectsTableReplacementAfterStaging() throws UserException txn.updateIcebergCommitData(Collections.singletonList(commitData)); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedUtils, table); txn.beginInsert(dorisTable, Optional.of(insertContext)); txn.finishInsert(NameMapping.createForTest(dbName, tableName)); @@ -738,9 +735,7 @@ public void testInsertCommitUsesStatementPinnedWriterMetadata() throws UserExcep try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedUtils, table); mockedUtils.when(() -> IcebergUtils.getFormatVersion(table)).thenReturn(3); mockedUtils.when(() -> IcebergUtils.dataLocation(table)).thenReturn(dataLocation); mockedWriterHelper.when(() -> IcebergWriterHelper.convertToWriterResult( @@ -856,8 +851,7 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(ctdList); @@ -882,8 +876,7 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); IcebergTransaction txn = getTxn(); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); @@ -932,8 +925,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) .thenCallRealMethod(); @@ -948,8 +940,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th checkPushDownByPartition(table, Expressions.equal("str1", "partition-b"), 1); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(table); + mockWritableTable(mockedStatic, table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) .thenCallRealMethod(); @@ -1028,8 +1019,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(icebergTable); + mockWritableTable(mockedUtils, icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(3); mockedWriterHelper.when(() -> IcebergWriterHelper.convertToDeleteFiles( @@ -1089,8 +1079,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) - .thenReturn(icebergTable); + mockWritableTable(mockedUtils, icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(formatVersion); mockedWriterHelper.when(() -> IcebergWriterHelper.convertToDeleteFiles( @@ -1123,10 +1112,13 @@ public void testBeginInsertUsesRetainedTargetTable() throws UserException { Mockito.mock(org.apache.iceberg.Transaction.class); Mockito.when(retainedTable.newTransaction()).thenReturn(retainedTransaction); - IcebergTransaction txn = getTxn(); - txn.beginInsert(dorisTable, retainedTable, Optional.empty()); + try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class)) { + mockWritableTable(mockedUtils, retainedTable); + IcebergTransaction txn = getTxn(); + txn.beginInsert(dorisTable, retainedTable, Optional.empty()); - Mockito.verify(retainedTable).newTransaction(); + Mockito.verify(retainedTable).newTransaction(); + } } @Test @@ -1138,10 +1130,76 @@ public void testBeginDeleteUsesRetainedTargetTable() throws UserException { Mockito.mock(org.apache.iceberg.Transaction.class); Mockito.when(retainedTable.newTransaction()).thenReturn(retainedTransaction); - IcebergTransaction txn = getTxn(); - txn.beginDelete(dorisTable, retainedTable); + try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class)) { + mockWritableTable(mockedUtils, retainedTable); + IcebergTransaction txn = getTxn(); + txn.beginDelete(dorisTable, retainedTable); + + Mockito.verify(retainedTable).newTransaction(); + } + } + + @Test + public void testWritableTableLeaseLivesUntilTransactionRollback() throws UserException { + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn("leased_target"); + Table table = Mockito.mock(Table.class); + Mockito.when(table.newTransaction()).thenReturn( + Mockito.mock(org.apache.iceberg.Transaction.class)); + AtomicBoolean released = new AtomicBoolean(); + + try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class)) { + mockedUtils.when(() -> IcebergUtils.acquireWritableIcebergTable( + ArgumentMatchers.any(ExternalTable.class), ArgumentMatchers.eq(ops))) + .thenReturn(new IcebergExternalMetaCache.WritableTableLease( + table, ops, testRuntimeContext(), false, false, + () -> released.set(true))); + IcebergTransaction txn = getTxn(); + txn.beginInsert(dorisTable, table, Optional.empty()); + + Assert.assertFalse("begin must retain the catalog generation", released.get()); + txn.rollback(); + Assert.assertTrue("rollback must release the catalog generation", released.get()); + } + } - Mockito.verify(retainedTable).newTransaction(); + @Test + public void testRewriteAdoptsEnclosingWritableTableLease() throws UserException { + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn("rewrite_leased_target"); + Table table = Mockito.mock(Table.class); + Mockito.when(table.newTransaction()).thenReturn( + Mockito.mock(org.apache.iceberg.Transaction.class)); + AtomicBoolean released = new AtomicBoolean(); + IcebergExternalMetaCache.WritableTableLease lease = + new IcebergExternalMetaCache.WritableTableLease( + table, ops, testRuntimeContext(), false, false, + () -> released.set(true)); + + try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class)) { + IcebergTransaction txn = getTxn(); + txn.beginRewrite(dorisTable, table, lease); + + Assert.assertFalse("rewrite must retain the enclosing generation", released.get()); + mockedUtils.verifyNoInteractions(); + txn.rollback(); + Assert.assertTrue("rewrite rollback must release the enclosing generation", released.get()); + } + } + + @Test + public void testWritableTableLeaseRetainsGenerationForAsyncBorrower() { + AtomicBoolean released = new AtomicBoolean(); + IcebergExternalMetaCache.WritableTableLease owner = + new IcebergExternalMetaCache.WritableTableLease( + Mockito.mock(Table.class), ops, testRuntimeContext(), false, false, + () -> released.set(true)); + IcebergExternalMetaCache.WritableTableLease borrower = owner.retain(); + + owner.close(); + Assert.assertFalse("owner close must not release an active async borrower", released.get()); + borrower.close(); + Assert.assertTrue("last borrower close must release the generation", released.get()); } @Test @@ -1169,8 +1227,7 @@ public void testQueryScopedGenerationCommitsThroughWritableOperations() throws U try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); + mockWritableTable(mockedUtils, liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, queryScopedTable, Optional.empty()); @@ -1200,8 +1257,7 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); + mockWritableTable(mockedUtils, liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -1251,8 +1307,7 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); + mockWritableTable(mockedUtils, liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -1281,8 +1336,7 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( - Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); + mockWritableTable(mockedUtils, liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/BaseIcebergActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/BaseIcebergActionTest.java new file mode 100644 index 00000000000000..b2d01d6b7e14ac --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/BaseIcebergActionTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg.action; + +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache.WritableTableLease; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergUtils; + +import org.apache.iceberg.Table; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; + +public class BaseIcebergActionTest { + + @Test + public void testActionRetainsWritableGenerationUntilExecutionFinishes() throws Exception { + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + AtomicBoolean released = new AtomicBoolean(); + WritableTableLease lease = Mockito.mock(WritableTableLease.class); + Mockito.when(lease.getTable()).thenReturn(icebergTable); + Mockito.when(lease.getAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + Mockito.doAnswer(invocation -> { + released.set(true); + return null; + }).when(lease).close(); + BaseIcebergAction action = new BaseIcebergAction( + "test", Collections.emptyMap(), Optional.empty(), Optional.empty()) { + @Override + protected void registerIcebergArguments() { + } + + @Override + protected List executeIcebergAction(TableIf table, Table retainedTable) { + Assert.assertSame(dorisTable, table); + Assert.assertSame(icebergTable, retainedTable); + Assert.assertFalse(released.get()); + return Collections.singletonList("ok"); + } + + @Override + public String getDescription() { + return "test action"; + } + }; + + try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class)) { + mockedUtils.when(() -> IcebergUtils.acquireWritableIcebergTable(dorisTable)) + .thenReturn(lease); + Assert.assertEquals(Collections.singletonList("ok"), action.executeAction(dorisTable)); + Assert.assertTrue(released.get()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java index e297ba478e46bf..d08a97e385c602 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java @@ -21,12 +21,15 @@ import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.scheduler.manager.TransientTaskManager; import org.junit.jupiter.api.Test; import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.util.Collections; + class RewriteDataFileExecutorTest { @Test @@ -47,4 +50,26 @@ void testInvalidateTableCacheAfterCommit() throws Exception { inOrder.verify(cacheMgr).invalidateTableCache(table); } } + + @Test + void testCancelRemovesTableBearingTaskFromManager() throws Exception { + Env env = Mockito.mock(Env.class); + TransientTaskManager taskManager = Mockito.mock(TransientTaskManager.class); + RewriteGroupTask task = Mockito.mock(RewriteGroupTask.class); + Mockito.when(task.getId()).thenReturn(7L); + RewriteDataFileExecutor.RewriteResultCollector collector = + new RewriteDataFileExecutor.RewriteResultCollector( + 1, Collections.singletonList(task)); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getTransientTaskManager()).thenReturn(taskManager); + + collector.cancelAllTasks(); + + InOrder inOrder = Mockito.inOrder(taskManager, task); + inOrder.verify(taskManager).removeMemoryTask(7L); + inOrder.verify(task).cancel(); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTaskTest.java index 93e92655da6881..421c3ae2a0d48e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTaskTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.iceberg.rewrite; import org.apache.doris.catalog.Env; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache.WritableTableLease; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -501,6 +502,22 @@ public void testCalculateRewriteStrategy_MultiplePartitions() throws Exception { Assertions.assertTrue(useGather); } + @Test + public void testCancelBeforeExecutionReleasesGenerationBorrower() throws Exception { + WritableTableLease lease = Mockito.mock(WritableTableLease.class); + RewriteGroupTask.RewriteResultCallback callback = + Mockito.mock(RewriteGroupTask.RewriteResultCallback.class); + RewriteGroupTask task = new RewriteGroupTask( + mockGroup, 1L, mockTable, null, lease, mockConnectContext, + 512 * MB, 1, callback); + + task.cancel(); + task.execute(); + + Mockito.verify(lease).close(); + Mockito.verify(callback).onTaskFailed(Mockito.eq(task.getId()), Mockito.any(Exception.class)); + } + // ========== Helper Methods ========== /** From 59f2a4133591cc3003a96b738685dfcaa05c3881 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 1 Sep 2026 12:41:29 +0800 Subject: [PATCH 33/38] [fix](fe) Stop cancelled Hudi listing submissions ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: Closing a statement could cancel Hudi file-system-view listing tasks while the partition submission loop kept submitting the remaining tasks. With the shared bounded file-listing executor at capacity, cancelled tasks could occupy the queue and leave the planner blocked in the rejection policy for up to ten seconds. Move task registration and submission behind ListingFsViewOwner, stop the loop after cancellation, remove cancelled queued tasks, and make the bounded rejection policy observe Future cancellation without interrupting or losing the caller interrupt state. ### Release note Fix Hudi planning cancellation so cancelled file-listing submissions stop promptly while active tasks retain their file-system-view lease until termination. ### Check List (For Author) - Test: Unit Test - HudiBatchFsViewOwnerTest, HudiScanNodeTest, ThreadPoolManagerTest - FE build and Checkstyle - Behavior changed: Yes. Hudi listing cancellation no longer submits remaining partitions or waits for cancelled tasks to enter a full queue. - Does this need documentation: No --- .../doris/common/ThreadPoolManager.java | 26 +++- .../datasource/hudi/source/HudiScanNode.java | 40 ++++-- .../doris/common/ThreadPoolManagerTest.java | 39 ++++++ .../hudi/source/HudiBatchFsViewOwnerTest.java | 130 ++++++++++++++++-- 4 files changed, 214 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java index 411ba6605aa666..69e35520280d1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java @@ -36,6 +36,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.PriorityBlockingQueue; @@ -390,6 +391,7 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { public static class BlockedPolicy implements RejectedExecutionHandler { private static final Logger LOG = LogManager.getLogger(BlockedPolicy.class); + private static final long CANCEL_CHECK_INTERVAL_NANOS = TimeUnit.MILLISECONDS.toNanos(100); private String threadPoolName; @@ -403,18 +405,34 @@ public BlockedPolicy(String threadPoolName, int timeoutSeconds) { @Override public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { try { - boolean ret = executor.getQueue().offer(r, timeoutSeconds, TimeUnit.SECONDS); - if (!ret) { - throw new RejectedExecutionException("submit task failed, queue size is full: " - + this.threadPoolName); + if (isCancelled(r) || executor.getQueue().offer(r)) { + return; + } + long remainingNanos = TimeUnit.SECONDS.toNanos(timeoutSeconds); + while (!isCancelled(r)) { + if (remainingNanos <= 0) { + throw new RejectedExecutionException("submit task failed, queue size is full: " + + this.threadPoolName); + } + long waitNanos = Math.min(remainingNanos, CANCEL_CHECK_INTERVAL_NANOS); + long startNanos = System.nanoTime(); + if (executor.getQueue().offer(r, waitNanos, TimeUnit.NANOSECONDS)) { + return; + } + remainingNanos -= System.nanoTime() - startNanos; } } catch (InterruptedException e) { + Thread.currentThread().interrupt(); String errMsg = String.format("Task %s wait to enqueue in %s %s failed", r.toString(), threadPoolName, executor.toString()); LOG.warn(errMsg); throw new RejectedExecutionException(errMsg); } } + + private boolean isCancelled(Runnable task) { + return task instanceof Future && ((Future) task).isCancelled(); + } } static class LogDiscardOldestPolicy implements RejectedExecutionHandler { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index d75086059f1b60..51cd3e50cf7872 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -101,6 +101,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -543,7 +544,7 @@ private List planPartitionSplits(HivePartition partition) throws IOEx private void getPartitionsSplits(List partitions, List splits) { Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); - ListingFsViewOwner createdOwner = new ListingFsViewOwner(fsViewLease); + ListingFsViewOwner createdOwner = new ListingFsViewOwner(fsViewLease, executor); ListingFsViewOwner owner = createdOwner; ConnectContext connectContext = ConnectContext.get(); StatementContext statementContext = connectContext == null ? null : connectContext.getStatementContext(); @@ -576,12 +577,12 @@ private void getPartitionsSplits(List partitions, List spl throwable.compareAndSet(null, t); } }, () -> { }); - owner.track(task); try { - executor.execute(task); + if (!owner.submit(task)) { + break; + } } catch (RuntimeException e) { submissionFailure = e; - task.cancelBeforeStart(); break; } } @@ -878,6 +879,7 @@ public void close() { @VisibleForTesting static class ListingFsViewOwner implements Closeable { private final HudiFsViewCacheValue.Lease lease; + private final Executor executor; private final AtomicInteger pendingTasks = new AtomicInteger(1); private final AtomicBoolean submissionFinished = new AtomicBoolean(); private final AtomicBoolean stopping = new AtomicBoolean(); @@ -885,19 +887,39 @@ static class ListingFsViewOwner implements Closeable { private final CompletableFuture tasksFinished = new CompletableFuture<>(); private final CompletableFuture cancelled = new CompletableFuture<>(); - ListingFsViewOwner(HudiFsViewCacheValue.Lease lease) { + ListingFsViewOwner(HudiFsViewCacheValue.Lease lease, Executor executor) { this.lease = lease; + this.executor = executor; } - void track(TerminalTask task) { + boolean submit(TerminalTask task) { pendingTasks.incrementAndGet(); task.setOwnerDone(() -> { tasks.remove(task); taskDone(); }); tasks.add(task); - if (stopping.get()) { - task.requestStop(); + try { + if (stopping.get()) { + stopTask(task); + return false; + } + executor.execute(task); + if (stopping.get()) { + stopTask(task); + return false; + } + return true; + } catch (RuntimeException e) { + stopTask(task); + throw e; + } + } + + private void stopTask(TerminalTask task) { + task.requestStop(); + if (executor instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) executor).remove(task); } } @@ -945,7 +967,7 @@ public void close() { // done() synchronously and may complete terminal accounting on this thread. // awaitCompletion must still observe cancellation rather than return partial splits. cancelled.complete(null); - tasks.forEach(TerminalTask::requestStop); + tasks.forEach(this::stopTask); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/ThreadPoolManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/ThreadPoolManagerTest.java index 2ed1ddd67e8922..bbabe512bd066c 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/common/ThreadPoolManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/ThreadPoolManagerTest.java @@ -20,10 +20,49 @@ import org.junit.Assert; import org.junit.Test; +import java.util.concurrent.FutureTask; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public class ThreadPoolManagerTest { + @Test + public void testBlockedPolicySkipsCancelledTask() { + LinkedBlockingQueue queue = new LinkedBlockingQueue<>(1); + Runnable queued = () -> { }; + queue.add(queued); + ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, queue); + FutureTask cancelled = new FutureTask<>(() -> null); + cancelled.cancel(false); + + new ThreadPoolManager.BlockedPolicy("test", 10).rejectedExecution(cancelled, executor); + + Assert.assertEquals(1, queue.size()); + Assert.assertSame(queued, queue.peek()); + executor.shutdownNow(); + } + + @Test + public void testBlockedPolicyRestoresInterrupt() { + LinkedBlockingQueue queue = new LinkedBlockingQueue<>(1); + queue.add(() -> { }); + ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, queue); + FutureTask task = new FutureTask<>(() -> null); + ThreadPoolManager.BlockedPolicy blockedPolicy = new ThreadPoolManager.BlockedPolicy("test", 10); + + try { + Thread.currentThread().interrupt(); + Assert.assertThrows(RejectedExecutionException.class, + () -> blockedPolicy.rejectedExecution(task, executor)); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + executor.shutdownNow(); + } + } + @Test public void testNormal() throws InterruptedException { ThreadPoolExecutor testCachedPool = ThreadPoolManager.newDaemonCacheThreadPool(2, "test_cache_pool", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index 77e2a861cff5dc..bc2d5f04dae115 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.hudi.source; +import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.datasource.SplitAssignment; import org.apache.doris.datasource.hudi.HudiFsViewCacheValue; @@ -24,13 +25,20 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; class HudiBatchFsViewOwnerTest { @@ -171,7 +179,8 @@ void assignmentStopInterruptsStartedTaskAndRetainsLeaseUntilTerminal() throws Ex @Test void synchronousListingCancellationReturnsBeforeBlockedTaskAndRetainsLease() throws Exception { HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease); + ExecutorService executor = Executors.newFixedThreadPool(2); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease, executor); CountDownLatch started = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); CountDownLatch terminated = new CountDownLatch(1); @@ -189,11 +198,9 @@ void synchronousListingCancellationReturnsBeforeBlockedTaskAndRetainsLease() thr terminated.countDown(); } }, () -> { }); - owner.track(task); + Assertions.assertTrue(owner.submit(task)); owner.submissionDone(); - ExecutorService executor = Executors.newFixedThreadPool(2); try { - executor.execute(task); Assertions.assertTrue(started.await(3, TimeUnit.SECONDS)); Future waiter = executor.submit(() -> Assertions.assertThrows(CancellationException.class, owner::awaitCompletion)); @@ -214,7 +221,7 @@ void synchronousListingCancellationReturnsBeforeBlockedTaskAndRetainsLease() thr @Test void synchronousListingDiscardBeforeSubmissionReleasesLease() { HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease, Runnable::run); owner.discardBeforeSubmission(); @@ -225,19 +232,126 @@ void synchronousListingDiscardBeforeSubmissionReleasesLease() { @Test void synchronousListingCancellationWinsAfterTerminalTasksCompleteInline() { HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease); + List submitted = new ArrayList<>(); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease, submitted::add); HudiScanNode.TerminalTask completed = new HudiScanNode.TerminalTask(() -> { }, () -> { }); HudiScanNode.TerminalTask queued = new HudiScanNode.TerminalTask( () -> Assertions.fail("cancelled task must not run"), () -> { }); - owner.track(completed); - owner.track(queued); + Assertions.assertTrue(owner.submit(completed)); + Assertions.assertTrue(owner.submit(queued)); completed.run(); owner.submissionDone(); owner.close(); + Assertions.assertEquals(2, submitted.size()); Assertions.assertTrue(queued.isCancelled()); Mockito.verify(lease).close(); Assertions.assertThrows(CancellationException.class, owner::awaitCompletion); } + + @Test + void synchronousListingCancellationRemovesQueuedTask() throws Exception { + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor executor = new ThreadPoolExecutor( + 1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + executor.execute(() -> { + workerStarted.countDown(); + try { + releaseWorker.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + try { + Assertions.assertTrue(workerStarted.await(3, TimeUnit.SECONDS)); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease, executor); + HudiScanNode.TerminalTask queued = new HudiScanNode.TerminalTask( + () -> Assertions.fail("cancelled task must not run"), () -> { }); + Assertions.assertTrue(owner.submit(queued)); + Assertions.assertEquals(1, executor.getQueue().size()); + + owner.close(); + + Assertions.assertTrue(queued.isCancelled()); + Assertions.assertTrue(executor.getQueue().isEmpty()); + owner.submissionDone(); + Mockito.verify(lease).close(); + Assertions.assertThrows(CancellationException.class, owner::awaitCompletion); + } finally { + releaseWorker.countDown(); + executor.shutdownNow(); + } + } + + @Test + void synchronousListingCloseStopsBlockedAndLaterSubmissions() throws Exception { + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch rejected = new CountDownLatch(1); + ThreadPoolManager.BlockedPolicy blockedPolicy = new ThreadPoolManager.BlockedPolicy("test", 10); + RejectedExecutionHandler handler = (task, executor) -> { + rejected.countDown(); + blockedPolicy.rejectedExecution(task, executor); + }; + ThreadPoolExecutor listingExecutor = new ThreadPoolExecutor( + 1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1), handler); + Runnable queued = () -> { }; + listingExecutor.execute(() -> { + workerStarted.countDown(); + try { + releaseWorker.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + Assertions.assertTrue(workerStarted.await(3, TimeUnit.SECONDS)); + listingExecutor.execute(queued); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease, listingExecutor); + HudiScanNode.TerminalTask blocked = new HudiScanNode.TerminalTask(() -> { }, () -> { }); + HudiScanNode.TerminalTask later = new HudiScanNode.TerminalTask( + () -> Assertions.fail("cancelled task must not run"), () -> { }); + ExecutorService submitter = Executors.newSingleThreadExecutor(); + try { + Future firstSubmission = submitter.submit(() -> owner.submit(blocked)); + Assertions.assertTrue(rejected.await(3, TimeUnit.SECONDS)); + + owner.close(); + + Assertions.assertFalse(firstSubmission.get(3, TimeUnit.SECONDS)); + Assertions.assertFalse(owner.submit(later)); + Assertions.assertTrue(blocked.isCancelled()); + Assertions.assertTrue(later.isCancelled()); + Assertions.assertEquals(1, listingExecutor.getQueue().size()); + Assertions.assertSame(queued, listingExecutor.getQueue().peek()); + owner.submissionDone(); + Mockito.verify(lease).close(); + Assertions.assertThrows(CancellationException.class, owner::awaitCompletion); + } finally { + submitter.shutdownNow(); + releaseWorker.countDown(); + listingExecutor.shutdownNow(); + } + } + + @Test + void synchronousListingCloseDuringImmediateSubmissionStopsTask() { + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + AtomicReference ownerRef = new AtomicReference<>(); + Executor immediateExecutor = task -> ownerRef.get().close(); + HudiScanNode.ListingFsViewOwner owner = new HudiScanNode.ListingFsViewOwner(lease, immediateExecutor); + ownerRef.set(owner); + HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask( + () -> Assertions.fail("cancelled task must not run"), () -> { }); + + Assertions.assertFalse(owner.submit(task)); + + Assertions.assertTrue(task.isCancelled()); + owner.submissionDone(); + Mockito.verify(lease).close(); + Assertions.assertThrows(CancellationException.class, owner::awaitCompletion); + } } From 695ebf145078171e280f6efe76c1b3d95c36fa9b Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 1 Sep 2026 13:38:28 +0800 Subject: [PATCH 34/38] [fix](fe) Complete Iceberg rewrites through transaction manager ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: Iceberg data-file rewrites registered transactions through the catalog transaction manager but committed and rolled back the underlying IcebergTransaction directly. This bypassed removal from the manager-local and global transaction registries, retaining the transaction, table, writable state, and resource generation after completion. It also left post-begin setup failures outside the rollback scope. Capture the exact transaction manager once, put all post-begin setup under its rollback scope, and complete both success and failure paths through the manager APIs. ### Release note Fix Iceberg rewrite transaction cleanup so completed and failed rewrites release their registered transaction state. ### Check List (For Author) - Test: Unit Test - RewriteDataFileExecutorTest, RewriteGroupTaskTest, IcebergTransactionTest - FE build and Checkstyle - Behavior changed: Yes. Iceberg rewrite transactions are removed from both transaction registries after commit or rollback. - Does this need documentation: No --- .../rewrite/RewriteDataFileExecutor.java | 22 +-- .../rewrite/RewriteDataFileExecutorTest.java | 150 ++++++++++++++++-- 2 files changed, 146 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java index c99b44b9b3dabd..fa95bf7f48ae00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java @@ -30,6 +30,7 @@ import org.apache.doris.scheduler.exception.JobException; import org.apache.doris.scheduler.executor.TransientTaskExecutor; import org.apache.doris.system.Backend; +import org.apache.doris.transaction.TransactionManager; import com.google.common.collect.Lists; // Keep third-party imports lexical to preserve the repository's CustomImportOrder invariant. @@ -62,16 +63,15 @@ public RewriteDataFileExecutor(IcebergExternalTable dorisTable, public RewriteResult executeGroupsConcurrently(List groups, long targetFileSizeBytes, WritableTableLease writableTableLease) throws UserException { - // Begin transaction - long transactionId = dorisTable.getCatalog().getTransactionManager().begin(); - IcebergTransaction transaction = (IcebergTransaction) dorisTable.getCatalog().getTransactionManager() - .getTransaction(transactionId); - MvccSnapshot targetSnapshot = new IcebergMvccSnapshot( - IcebergUtils.getSnapshotForWritableLease(dorisTable, writableTableLease)); List tasks = Lists.newArrayList(); RewriteResultCollector resultCollector = new RewriteResultCollector(groups.size(), tasks); + TransactionManager transactionManager = dorisTable.getCatalog().getTransactionManager(); + long transactionId = transactionManager.begin(); boolean committed = false; try { + IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(transactionId); + MvccSnapshot targetSnapshot = new IcebergMvccSnapshot( + IcebergUtils.getSnapshotForWritableLease(dorisTable, writableTableLease)); transaction.beginRewrite(dorisTable, writableTableLease.getTable(), writableTableLease); // Register files to delete @@ -132,7 +132,7 @@ public void onTaskFailed(Long taskId, Exception error) { long rewrittenBytesCount = groups.stream().mapToLong(group -> group.getTotalSize()).sum(); int removedDeleteFilesCount = groups.stream().mapToInt(group -> group.getDeleteFileCount()).sum(); - transaction.commit(); + transactionManager.commit(transactionId); committed = true; Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable); @@ -141,17 +141,11 @@ public void onTaskFailed(Long taskId, Exception error) { } finally { if (!committed) { resultCollector.cancelAllTasks(); - transaction.rollback(); + transactionManager.rollback(transactionId); } } } - void commitAndInvalidate(IcebergTransaction transaction) throws UserException { - transaction.commit(); - // Rewrite commits bypass the external-table DDL path, so evict the pre-rewrite snapshot before reuse. - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable); - } - /** * Wait for all tasks to complete using notification mechanism */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java index d08a97e385c602..dd6d363defee72 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java @@ -18,36 +18,70 @@ package org.apache.doris.datasource.iceberg.rewrite; import org.apache.doris.catalog.Env; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache.WritableTableLease; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.resource.computegroup.ComputeGroup; import org.apache.doris.scheduler.manager.TransientTaskManager; +import org.apache.doris.transaction.GlobalExternalTransactionInfoMgr; +import org.apache.doris.transaction.Transaction; +import org.apache.doris.transaction.TransactionManager; +import org.apache.iceberg.Table; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; class RewriteDataFileExecutorTest { @Test - void testInvalidateTableCacheAfterCommit() throws Exception { - IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); - IcebergTransaction transaction = Mockito.mock(IcebergTransaction.class); - Env env = Mockito.mock(Env.class); - ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + void testCompletedRewriteClearsTransactionRegistries() throws Exception { + RewriteFixture fixture = new RewriteFixture(); - try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { - mockedEnv.when(Env::getCurrentEnv).thenReturn(env); - Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class)) { + fixture.prepareSuccessfulExecution(mockedEnv, mockedIcebergUtils); - new RewriteDataFileExecutor(table, null).commitAndInvalidate(transaction); + fixture.executor.executeGroupsConcurrently(Collections.emptyList(), 1L, fixture.lease); - InOrder inOrder = Mockito.inOrder(transaction, cacheMgr); - inOrder.verify(transaction).commit(); - inOrder.verify(cacheMgr).invalidateTableCache(table); + fixture.assertTransactionRegistriesEmpty(); + InOrder inOrder = Mockito.inOrder(fixture.transaction, fixture.cacheManager); + inOrder.verify(fixture.transaction).commit(); + inOrder.verify(fixture.cacheManager).invalidateTableCache(fixture.table); + Mockito.verify(fixture.transaction, Mockito.never()).rollback(); + } + } + + @Test + void testPostBeginSetupFailureClearsTransactionRegistries() throws Exception { + RewriteFixture fixture = new RewriteFixture(); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class)) { + fixture.prepareEnvironment(mockedEnv); + mockedIcebergUtils.when(() -> IcebergUtils.getSnapshotForWritableLease(fixture.table, fixture.lease)) + .thenThrow(new IllegalStateException("snapshot setup failed")); + + Assertions.assertThrows(IllegalStateException.class, + () -> fixture.executor.executeGroupsConcurrently( + Collections.emptyList(), 1L, fixture.lease)); + + fixture.assertTransactionRegistriesEmpty(); + Mockito.verify(fixture.transaction).rollback(); + Mockito.verify(fixture.transaction, Mockito.never()).commit(); } } @@ -72,4 +106,96 @@ void testCancelRemovesTableBearingTaskFromManager() throws Exception { inOrder.verify(task).cancel(); } } + + private static class RewriteFixture { + private static final long TRANSACTION_ID = 7L; + + private final IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + private final ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + private final IcebergTransaction transaction = Mockito.mock(IcebergTransaction.class); + private final GlobalExternalTransactionInfoMgr globalTransactionManager = + new GlobalExternalTransactionInfoMgr(); + private final TrackingTransactionManager transactionManager = + new TrackingTransactionManager(transaction, globalTransactionManager); + private final WritableTableLease lease = Mockito.mock(WritableTableLease.class); + private final ConnectContext connectContext = Mockito.mock(ConnectContext.class); + private final RewriteDataFileExecutor executor = new RewriteDataFileExecutor(table, connectContext); + private final Env env = Mockito.mock(Env.class); + private final ExternalMetaCacheMgr cacheManager = Mockito.mock(ExternalMetaCacheMgr.class); + + private RewriteFixture() { + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getTransactionManager()).thenReturn(transactionManager); + } + + private void prepareEnvironment(MockedStatic mockedEnv) throws UserException { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheManager); + } + + private void prepareSuccessfulExecution(MockedStatic mockedEnv, + MockedStatic mockedIcebergUtils) throws UserException { + prepareEnvironment(mockedEnv); + IcebergSnapshotCacheValue snapshot = Mockito.mock(IcebergSnapshotCacheValue.class); + mockedIcebergUtils.when(() -> IcebergUtils.getSnapshotForWritableLease(table, lease)) + .thenReturn(snapshot); + Mockito.when(lease.getTable()).thenReturn(Mockito.mock(Table.class)); + ComputeGroup computeGroup = Mockito.mock(ComputeGroup.class); + Mockito.when(connectContext.getComputeGroup()).thenReturn(computeGroup); + Mockito.when(computeGroup.getBackendList()).thenReturn(Collections.emptyList()); + SessionVariable sessionVariable = Mockito.mock(SessionVariable.class); + Mockito.when(connectContext.getSessionVariable()).thenReturn(sessionVariable); + Mockito.when(sessionVariable.getInsertTimeoutS()).thenReturn(1); + } + + private void assertTransactionRegistriesEmpty() { + Assertions.assertTrue(transactionManager.transactions.isEmpty()); + Assertions.assertTrue(globalTransactionManager.idToTxn.isEmpty()); + } + + private static class TrackingTransactionManager implements TransactionManager { + private final Map transactions = new HashMap<>(); + private final Transaction transaction; + private final GlobalExternalTransactionInfoMgr globalTransactionManager; + + private TrackingTransactionManager(Transaction transaction, + GlobalExternalTransactionInfoMgr globalTransactionManager) { + this.transaction = transaction; + this.globalTransactionManager = globalTransactionManager; + } + + @Override + public long begin() { + transactions.put(TRANSACTION_ID, transaction); + globalTransactionManager.putTxnById(TRANSACTION_ID, transaction); + return TRANSACTION_ID; + } + + @Override + public void commit(long id) throws UserException { + getTransaction(id).commit(); + transactions.remove(id); + globalTransactionManager.removeTxnById(id); + } + + @Override + public void rollback(long id) { + Transaction registered = transactions.get(id); + if (registered != null) { + registered.rollback(); + } + transactions.remove(id); + globalTransactionManager.removeTxnById(id); + } + + @Override + public Transaction getTransaction(long id) throws UserException { + Transaction registered = transactions.get(id); + if (registered == null) { + throw new UserException("Missing transaction " + id); + } + return registered; + } + } + } } From e378b44d8d4a34e2f7027fc4a6100a424fade902 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 1 Sep 2026 16:46:46 +0800 Subject: [PATCH 35/38] [refactor](fe) Centralize Iceberg writable mutation commits ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: The branch-4.1 lifecycle repair acquired the exact writable table generation correctly, but repeated lease acquisition, authenticated commit, and exception conversion across every Iceberg schema, property, and partition mutation. Centralize those branch-native operations while preserving the same lease lifetime, authenticator, error text, refresh ordering, and root-cause reporting. ### Release note None ### Check List (For Author) - Test: Unit Test - 74 focused Iceberg metadata operation tests - JDK 17 ./build.sh --fe -j4 - FE Checkstyle - Behavior changed: No - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 206 +++++++----------- 1 file changed, 78 insertions(+), 128 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index e5995bc176c00e..be26873b251748 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -495,8 +495,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) @Override public void createOrReplaceBranchImpl(ExternalTable dorisTable, CreateOrReplaceBranchInfo branchInfo) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); ExecutionAuthenticator authenticator = lease.getAuthenticator(); BranchOptions branchOptions = branchInfo.getBranchOptions(); @@ -587,8 +586,7 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { @Override public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagInfo tagInfo) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); ExecutionAuthenticator authenticator = lease.getAuthenticator(); TagOptions tagOptions = tagInfo.getTagOptions(); @@ -644,8 +642,7 @@ public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagI public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws UserException { String tagName = tagInfo.getTagName(); boolean ifExists = tagInfo.getIfExists(); - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); SnapshotRef snapshotRef = icebergTable.refs().get(tagName); @@ -668,8 +665,7 @@ public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws Us public void dropBranchImpl(ExternalTable dorisTable, DropBranchInfo branchInfo) throws UserException { String branchName = branchInfo.getBranchName(); boolean ifExists = branchInfo.getIfExists(); - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); SnapshotRef snapshotRef = icebergTable.refs().get(branchName); @@ -770,12 +766,34 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { } } + private IcebergExternalMetaCache.WritableTableLease acquireWritableTable(ExternalTable dorisTable) { + return IcebergUtils.acquireWritableIcebergTable(dorisTable, this); + } + + private void commitTableUpdate(IcebergExternalMetaCache.WritableTableLease lease, + Runnable commit, String failureMessage) throws UserException { + try { + lease.getAuthenticator().execute(commit); + } catch (Exception e) { + throw new UserException(failureMessage + ", error message is: " + e.getMessage(), e); + } + } + + private void commitPartitionSpecUpdate(IcebergExternalMetaCache.WritableTableLease lease, + Runnable commit, String failureMessage) throws UserException { + try { + lease.getAuthenticator().execute(commit); + } catch (Exception e) { + throw new UserException(failureMessage + ", error message is: " + + ExceptionUtils.getRootCauseMessage(e), e); + } + } + @Override public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateAddColumnMetadata(column, true); - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); @@ -787,12 +805,8 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po if (position != null) { applyPosition(updateSchema, position, ColumnPath.of(column.getName()), schema, "add"); } - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to add column: " + column.getName() + " to table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to add column: " + column.getName() + " to table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -808,8 +822,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); @@ -829,20 +842,16 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co applyPosition(updateSchema, position, childPath(parentPath.getColumnPath(), columnPath.getLeafName()), icebergTable.schema(), "add"); } - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to add nested column: " + columnPath.getFullPath() + " to table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to add nested column: " + columnPath.getFullPath() + " to table: " + + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); for (Column column : columns) { validateAddColumnMetadata(column, true); @@ -855,31 +864,22 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda for (Column column : columns) { addOneColumn(updateSchema, column); } - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to add columns to table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to add columns to table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); validateRowLineageColumnMutation(icebergTable, columnName, "drop"); ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.deleteColumn(columnPath.getFullPath()); - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to drop column: " + columnName + " from table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to drop column: " + columnName + " from table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -890,19 +890,15 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); return; } - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.deleteColumn(resolvedPath.getFullPath()); - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to drop nested column: " + columnPath.getFullPath() + " from table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to drop nested column: " + columnPath.getFullPath() + " from table: " + + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -910,17 +906,12 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd @Override public void updateTableProperties(ExternalTable dorisTable, Map properties, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); UpdateProperties updateProperties = icebergTable.updateProperties(); properties.forEach(updateProperties::set); - try { - lease.getAuthenticator().execute(updateProperties::commit); - } catch (Exception e) { - throw new UserException("Failed to update properties for table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateProperties::commit, + "Failed to update properties for table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -928,8 +919,7 @@ public void updateTableProperties(ExternalTable dorisTable, Map @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); validateRowLineageColumnMutation(icebergTable, oldName, "rename"); validateRowLineageColumnMutation(icebergTable, newName, "rename to"); @@ -939,12 +929,9 @@ public void renameColumn(ExternalTable dorisTable, String oldName, String newNam schema.asStruct(), "", newName, oldPath.getField(), "rename"); UpdateSchema updateSchema = icebergTable.updateSchema(); applyRenameColumn(schema, updateSchema, oldPath, newName); - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to rename column: " + oldName + " to " + newName - + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to rename column: " + oldName + " to " + newName + " in table: " + + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -956,8 +943,7 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); return; } - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath( icebergTable.schema(), columnPath, "rename"); @@ -968,12 +954,9 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String UpdateSchema updateSchema = icebergTable.updateSchema(); applyRenameColumn(icebergTable.schema(), updateSchema, resolvedPath, newName); - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to rename nested column: " + columnPath.getFullPath() + " to " + newName - + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to rename nested column: " + columnPath.getFullPath() + " to " + newName + + " in table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -1023,8 +1006,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); NestedField currentCol = icebergTable.schema().asStruct() @@ -1077,12 +1059,8 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPat if (position != null) { applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); } - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to modify column: " + column.getName() + " in table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to modify column: " + column.getName() + " in table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -1095,8 +1073,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column return; } - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); @@ -1136,12 +1113,9 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); } - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to modify nested column: " + columnPath.getFullPath() + " in table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to modify nested column: " + columnPath.getFullPath() + " in table: " + + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -1149,8 +1123,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column @Override public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); if (!columnPath.isNested()) { validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); @@ -1161,12 +1134,9 @@ public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.updateColumnDoc(resolvedPath.getFullPath(), StringUtils.defaultString(comment)); - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to modify column comment: " + columnPath.getFullPath() + " in table: " - + icebergTable.name() + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to modify column comment: " + columnPath.getFullPath() + " in table: " + + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -1719,8 +1689,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); List canonicalOrder = new ArrayList<>(newOrder.size()); Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); @@ -1738,12 +1707,8 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long for (int i = 1; i < canonicalOrder.size(); i++) { updateSchema.moveAfter(canonicalOrder.get(i), canonicalOrder.get(i - 1)); } - try { - lease.getAuthenticator().execute(() -> updateSchema.commit()); - } catch (Exception e) { - throw new UserException("Failed to reorder columns in table: " + icebergTable.name() - + ", error message is: " + e.getMessage(), e); - } + commitTableUpdate(lease, updateSchema::commit, + "Failed to reorder columns in table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -1789,8 +1754,7 @@ private Term getTransform(String transformName, String columnName, Integer trans */ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -1806,12 +1770,8 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause updateSpec.addField(transform); } - try { - lease.getAuthenticator().execute(() -> updateSpec.commit()); - } catch (Exception e) { - throw new UserException("Failed to add partition field to table: " + icebergTable.name() - + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); - } + commitPartitionSpecUpdate(lease, updateSpec::commit, + "Failed to add partition field to table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -1821,8 +1781,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause */ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -1836,12 +1795,8 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus updateSpec.removeField(transform); } - try { - lease.getAuthenticator().execute(() -> updateSpec.commit()); - } catch (Exception e) { - throw new UserException("Failed to drop partition field from table: " + icebergTable.name() - + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); - } + commitPartitionSpecUpdate(lease, updateSpec::commit, + "Failed to drop partition field from table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } @@ -1851,8 +1806,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus */ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { - try (IcebergExternalMetaCache.WritableTableLease lease = - IcebergUtils.acquireWritableIcebergTable(dorisTable, this)) { + try (IcebergExternalMetaCache.WritableTableLease lease = acquireWritableTable(dorisTable)) { Table icebergTable = lease.getTable(); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -1880,12 +1834,8 @@ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFiel updateSpec.addField(newTransform); } - try { - lease.getAuthenticator().execute(() -> updateSpec.commit()); - } catch (Exception e) { - throw new UserException("Failed to replace partition field in table: " + icebergTable.name() - + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); - } + commitPartitionSpecUpdate(lease, updateSpec::commit, + "Failed to replace partition field in table: " + icebergTable.name()); } refreshTable(dorisTable, updateTime); } From 7963f7ce0a332060eb5f486385de8841835c482c Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 1 Sep 2026 18:43:46 +0800 Subject: [PATCH 36/38] [fix](fe) Close Iceberg reset-time lifecycle gaps ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: Iceberg transactions are registered before insert setup begins, but setup failures were outside the executor failure scope. Catalog reset could also clear the live authenticator before rollback, leaving transaction registries and the retained table generation pinned. Snapshot projection independently classified a live table generation instead of the retained projection generation, so refresh could mix partition eligibility from one generation with snapshot metadata from another. Move setup into the existing cleanup envelope, let Iceberg commit and rollback use the transaction's retained generation, and classify every native and HMS projection from its concrete Iceberg Table without a cross-generation boolean cache. ### Release note Fix Iceberg transaction and snapshot resource lifecycle across concurrent catalog refresh/reset. ### Check List (For Author) - Test: Unit Test - 101 focused Iceberg metadata and insert lifecycle tests - JDK 17 ./build.sh --fe -j4 - FE Checkstyle - Behavior changed: Yes, failed Iceberg setup and rollback now release the registered transaction and retained runtime, and snapshot partition classification remains generation-consistent. - Does this need documentation: No --- .../datasource/hive/IcebergDlaTable.java | 29 +--- .../iceberg/IcebergExternalMetaCache.java | 22 +-- .../iceberg/IcebergExternalTable.java | 26 +-- .../datasource/iceberg/IcebergUtils.java | 30 ++++ .../insert/AbstractInsertExecutor.java | 4 +- .../BaseExternalTableInsertExecutor.java | 18 ++- .../iceberg/IcebergExternalMetaCacheTest.java | 54 ++++++- .../IcebergInsertFailureLifecycleTest.java | 149 ++++++++++++++++++ 8 files changed, 256 insertions(+), 76 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java index 2d57c1e7683886..260fa7e55f486e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java @@ -29,10 +29,6 @@ import org.apache.doris.mtmv.MTMVSnapshotIf; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.iceberg.PartitionField; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Table; import java.util.List; import java.util.Map; @@ -114,31 +110,8 @@ protected boolean isValidRelatedTable() { if (isValidRelatedTableCached) { return isValidRelatedTable; } - isValidRelatedTable = IcebergUtils.withIcebergTable(hmsTable, this::isValidRelatedTable); + isValidRelatedTable = IcebergUtils.withIcebergTable(hmsTable, IcebergUtils::isValidRelatedTable); isValidRelatedTableCached = true; return isValidRelatedTable; } - - private boolean isValidRelatedTable(Table table) { - Set allFields = Sets.newHashSet(); - for (PartitionSpec spec : table.specs().values()) { - if (spec == null) { - return false; - } - List fields = spec.fields(); - if (fields.size() != 1) { - return false; - } - PartitionField partitionField = spec.fields().get(0); - String transformName = partitionField.transform().toString(); - if (!IcebergUtils.YEAR.equals(transformName) - && !IcebergUtils.MONTH.equals(transformName) - && !IcebergUtils.DAY.equals(transformName) - && !IcebergUtils.HOUR.equals(transformName)) { - return false; - } - allFields.add(table.schema().findColumnName(partitionField.sourceId())); - } - return allFields.size() == 1; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 023e6841ff1359..1cce05db8f7bdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -334,9 +334,7 @@ public IcebergSnapshotCacheValue getSnapshotForWritableLease( return execute(lease.getAuthenticator(), () -> loadSnapshotProjection( dorisTable, table, table, IcebergSnapshotCacheValue.retainCurrentSnapshotJson(table), true, lease.getAuthenticator(), lease.isEnableMappingVarbinary(), - lease.isEnableMappingTimestampTz(), - dorisTable instanceof IcebergExternalTable - ? ((IcebergExternalTable) dorisTable).isValidRelatedTable(table) : null) + lease.isEnableMappingTimestampTz()) .bindCapturedAuthenticator(lease.getAuthenticator()) .bindRuntimeContext(lease.getRuntimeContext()) .bindSchemaMappingOptions(lease.isEnableMappingVarbinary(), @@ -1028,27 +1026,17 @@ private IcebergSnapshotCacheValue loadSnapshotProjection( String retainedCurrentSnapshotJson, boolean isolateForQueries, ExecutionAuthenticator authenticator, boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { - return loadSnapshotProjection(dorisTable, projectionTable, retainedTable, - retainedCurrentSnapshotJson, isolateForQueries, authenticator, - enableMappingVarbinary, enableMappingTimestampTz, null); - } - - private IcebergSnapshotCacheValue loadSnapshotProjection( - ExternalTable dorisTable, Table projectionTable, Table retainedTable, - String retainedCurrentSnapshotJson, boolean isolateForQueries, - ExecutionAuthenticator authenticator, - boolean enableMappingVarbinary, boolean enableMappingTimestampTz, - @Nullable Boolean validRelatedTableOverride) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); } try { - MTMVRelatedTableIf table = (MTMVRelatedTableIf) dorisTable; IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(projectionTable); IcebergPartitionInfo icebergPartitionInfo; - boolean validRelatedTable = validRelatedTableOverride == null - ? table.isValidRelatedTable() : validRelatedTableOverride; + // The projection and its partition eligibility must be classified from the same + // retained table generation. The no-argument method performs another cache lookup, + // which can cross a concurrent refresh/reset boundary. + boolean validRelatedTable = IcebergUtils.isValidRelatedTable(projectionTable); if (!validRelatedTable) { icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index d3284942637ba9..e36d308f032b29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -53,7 +53,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -251,31 +250,8 @@ synchronized boolean isValidRelatedTable(Table table) { if (isValidRelatedTableCached) { return isValidRelatedTable; } - isValidRelatedTable = false; - Set allFields = Sets.newHashSet(); - for (PartitionSpec spec : table.specs().values()) { - if (spec == null) { - isValidRelatedTableCached = true; - return false; - } - List fields = spec.fields(); - if (fields.size() != 1) { - isValidRelatedTableCached = true; - return false; - } - PartitionField partitionField = spec.fields().get(0); - String transformName = partitionField.transform().toString(); - if (!IcebergUtils.YEAR.equals(transformName) - && !IcebergUtils.MONTH.equals(transformName) - && !IcebergUtils.DAY.equals(transformName) - && !IcebergUtils.HOUR.equals(transformName)) { - isValidRelatedTableCached = true; - return false; - } - allFields.add(table.schema().findColumnName(partitionField.sourceId())); - } + isValidRelatedTable = IcebergUtils.isValidRelatedTable(table); isValidRelatedTableCached = true; - isValidRelatedTable = allFields.size() == 1; return isValidRelatedTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index e7e3a5806d62b5..cc1117ebb0c529 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -261,6 +261,36 @@ private static StorageProperties chooseS3CompatibleStorage(ListThis method deliberately has no Doris-table-level cache. Callers that retain an Iceberg + * {@link Table} across a catalog refresh must classify that exact generation instead of + * reusing a result computed for another generation. + */ + public static boolean isValidRelatedTable(Table table) { + Set allFields = Sets.newHashSet(); + for (PartitionSpec spec : table.specs().values()) { + if (spec == null) { + return false; + } + List fields = spec.fields(); + if (fields.size() != 1) { + return false; + } + PartitionField partitionField = fields.get(0); + String transformName = partitionField.transform().toString(); + if (!YEAR.equals(transformName) + && !MONTH.equals(transformName) + && !DAY.equals(transformName) + && !HOUR.equals(transformName)) { + return false; + } + allFields.add(table.schema().findColumnName(partitionField.sourceId())); + } + return allFields.size() == 1; + } + public static boolean hasIcebergCatalogFormatVersion(Map catalogProperties) { return catalogProperties.containsKey(CatalogProperties.TABLE_OVERRIDE_PREFIX + TableProperties.FORMAT_VERSION) || catalogProperties.containsKey(CatalogProperties.TABLE_DEFAULT_PREFIX diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java index b8c34291c93d93..3710358e699436 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java @@ -257,8 +257,8 @@ private void checkStrictModeAndFilterRatio() throws Exception { * execute insert txn for insert into select command. */ public void executeSingleInsert(StmtExecutor executor) throws Exception { - beforeExec(); try { + beforeExec(); executor.updateProfile(false); execImpl(executor); checkStrictModeAndFilterRatio(); @@ -290,8 +290,8 @@ public void executeSingleInsert(StmtExecutor executor) throws Exception { * needs to commit a partition delete. */ public void executeEmptyInsert(StmtExecutor executor) throws Exception { - beforeExec(); try { + beforeExec(); for (InsertExecutorListener listener : listeners) { listener.beforeComplete(this, executor, jobId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java index 3d8f74d502a78b..e597732871b2e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -97,7 +97,12 @@ protected void onComplete() throws UserException { long t0 = System.currentTimeMillis(); doBeforeCommit(); long t1 = System.currentTimeMillis(); - if (table instanceof ExternalTable) { + if (transactionType() == TransactionType.ICEBERG) { + // IcebergTransaction already executes the remote commit through the authenticator + // captured by its writable generation. Looking up the live catalog authenticator + // here can fail after reset and strand the retained transaction generation. + transactionManager.commit(txnId); + } else if (table instanceof ExternalTable) { try { ExternalTable externalTable = (ExternalTable) table; externalTable.getCatalog().getExecutionAuthenticator().execute(() -> { @@ -169,7 +174,16 @@ protected void onFail(Throwable t) { String finalErrorMsg = InsertUtils.getFinalErrorMsg(t.getMessage(), firstErrorMsgPart, urlPart); ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, finalErrorMsg); - if (table instanceof ExternalTable) { + if (transactionType() == TransactionType.ICEBERG) { + // Iceberg rollback only clears transaction-local state and releases the generation + // lease retained by IcebergTransaction. It must remain possible after catalog reset, + // when the mutable catalog authenticator is intentionally unavailable. + try { + transactionManager.rollback(txnId); + } catch (Exception e) { + LOG.warn("errors when abort txn. {} for table: {}", txnId, table.getName(), e); + } + } else if (table instanceof ExternalTable) { try { ExternalTable externalTable = (ExternalTable) table; externalTable.getCatalog().getExecutionAuthenticator().execute(() -> { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 625e8b180fc802..be0dc7d9e98f1a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -23,6 +23,7 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; @@ -45,6 +46,7 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.Metrics; import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; @@ -77,6 +79,7 @@ import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -657,11 +660,10 @@ protected CatalogIf getCatalog(long catalogId) { } @Test - public void testSnapshotPartitionLoadUsesCapturedAuthenticator() throws Exception { + public void testSnapshotPartitionLoadUsesCapturedAuthenticatorAndProjectionGeneration() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); - Mockito.when(dorisTable.isValidRelatedTable()).thenReturn(true); Table projectionTable = Mockito.mock(Table.class); Snapshot snapshot = Mockito.mock(Snapshot.class); Schema schema = Mockito.mock(Schema.class); @@ -683,6 +685,8 @@ public void testSnapshotPartitionLoadUsesCapturedAuthenticator() throws Exceptio .thenReturn(IcebergPartitionInfo.empty()); icebergUtils.when(() -> IcebergUtils.getNameMapping(projectionTable)) .thenReturn(Optional.empty()); + icebergUtils.when(() -> IcebergUtils.isValidRelatedTable(projectionTable)) + .thenReturn(true); icebergUtils.clearInvocations(); loader.invoke(cache, dorisTable, projectionTable, projectionTable, @@ -690,6 +694,52 @@ public void testSnapshotPartitionLoadUsesCapturedAuthenticator() throws Exceptio icebergUtils.verify(() -> IcebergUtils.loadPartitionInfo( dorisTable, projectionTable, 11L, 3L, capturedAuthenticator, true, false)); + icebergUtils.verify(() -> IcebergUtils.isValidRelatedTable(projectionTable)); + Mockito.verify(dorisTable, Mockito.never()).isValidRelatedTable(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testHmsSnapshotProjectionIgnoresRelatedTableCacheFromAnotherGeneration() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + HMSExternalTable dorisTable = Mockito.mock(HMSExternalTable.class); + Mockito.when(dorisTable.isValidRelatedTable()).thenReturn(true); + Table projectionTable = Mockito.mock(Table.class); + PartitionSpec projectionSpec = Mockito.mock(PartitionSpec.class); + Mockito.when(projectionSpec.fields()).thenReturn(Arrays.asList( + Mockito.mock(PartitionField.class), Mockito.mock(PartitionField.class))); + Mockito.when(projectionTable.specs()).thenReturn(Collections.singletonMap(1, projectionSpec)); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Schema schema = Mockito.mock(Schema.class); + Mockito.when(projectionTable.currentSnapshot()).thenReturn(snapshot); + Mockito.when(snapshot.snapshotId()).thenReturn(11L); + Mockito.when(projectionTable.schema()).thenReturn(schema); + Mockito.when(schema.schemaId()).thenReturn(3); + ExecutionAuthenticator capturedAuthenticator = new ExecutionAuthenticator() { + }; + Method loader = IcebergExternalMetaCache.class.getDeclaredMethod( + "loadSnapshotProjection", ExternalTable.class, Table.class, Table.class, + String.class, boolean.class, ExecutionAuthenticator.class, + boolean.class, boolean.class); + loader.setAccessible(true); + try (MockedStatic icebergUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + icebergUtils.when(() -> IcebergUtils.getNameMapping(projectionTable)) + .thenReturn(Optional.empty()); + icebergUtils.clearInvocations(); + + loader.invoke(cache, dorisTable, projectionTable, projectionTable, + null, false, capturedAuthenticator, true, false); + + icebergUtils.verify(() -> IcebergUtils.isValidRelatedTable(projectionTable)); + icebergUtils.verify(() -> IcebergUtils.loadPartitionInfo( + dorisTable, projectionTable, 11L, 3L, capturedAuthenticator, true, false), + Mockito.never()); + Mockito.verify(dorisTable, Mockito.never()).isValidRelatedTable(); } finally { cache.close(); executor.shutdownNow(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java new file mode 100644 index 00000000000000..c33d3f8d69854c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.hive.HiveTransactionMgr; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMetadataOps; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.Coordinator; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TUniqueId; +import org.apache.doris.transaction.GlobalExternalTransactionInfoMgr; +import org.apache.doris.transaction.IcebergTransactionManager; +import org.apache.doris.transaction.TransactionType; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Optional; + +class IcebergInsertFailureLifecycleTest { + + @AfterEach + void tearDown() { + ConnectContext.remove(); + } + + @Test + void testNormalInsertBeginFailureClearsLocalAndGlobalRegistriesAfterReset() throws Exception { + assertBeginFailureClearsRegistries(false); + } + + @Test + void testEmptyInsertBeginFailureClearsLocalAndGlobalRegistriesAfterReset() throws Exception { + assertBeginFailureClearsRegistries(true); + } + + private void assertBeginFailureClearsRegistries(boolean emptyInsert) throws Exception { + Env env = Mockito.mock(Env.class); + GlobalExternalTransactionInfoMgr globalTransactions = new GlobalExternalTransactionInfoMgr(); + Mockito.when(env.getNextId()).thenReturn(101L); + Mockito.when(env.getGlobalExternalTransactionInfoMgr()).thenReturn(globalTransactions); + + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + HiveTransactionMgr hiveTransactionMgr = Mockito.mock(HiveTransactionMgr.class); + IcebergTransactionManager transactionManager = new IcebergTransactionManager(metadataOps); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("iceberg"); + Mockito.when(catalog.getTransactionManager()).thenReturn(transactionManager); + Mockito.when(catalog.getExecutionAuthenticator()).thenThrow( + new IllegalStateException("catalog reset cleared the live authenticator")); + + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getId()).thenReturn(1L); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getName()).thenReturn("tbl"); + + ConnectContext context = new ConnectContext(); + context.setThreadLocalInfo(); + context.setQueryId(new TUniqueId(1L, 2L)); + Coordinator coordinator = Mockito.mock(Coordinator.class); + EnvFactory factory = Mockito.mock(EnvFactory.class); + Mockito.when(factory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong())) + .thenReturn(coordinator); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class); + MockedStatic factoryMock = Mockito.mockStatic(EnvFactory.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(env); + envMock.when(Env::getCurrentHiveTransactionMgr).thenReturn(hiveTransactionMgr); + factoryMock.when(EnvFactory::getInstance).thenReturn(factory); + + FailingIcebergExecutor executor = new FailingIcebergExecutor( + context, table, emptyInsert); + executor.beginTransaction(); + long transactionId = executor.getTxnId(); + Assertions.assertNotNull(transactionManager.getTransaction(transactionId)); + Assertions.assertNotNull(globalTransactions.getTxnById(transactionId)); + + StmtExecutor stmtExecutor = Mockito.mock(StmtExecutor.class); + if (emptyInsert) { + executor.executeEmptyInsert(stmtExecutor); + } else { + executor.executeSingleInsert(stmtExecutor); + } + + Assertions.assertThrows(UserException.class, + () -> transactionManager.getTransaction(transactionId)); + Assertions.assertThrows(RuntimeException.class, + () -> globalTransactions.getTxnById(transactionId)); + Mockito.verify(catalog, Mockito.never()).getExecutionAuthenticator(); + Mockito.verify(coordinator).close(); + } + } + + private static class FailingIcebergExecutor extends BaseExternalTableInsertExecutor { + FailingIcebergExecutor(ConnectContext context, IcebergExternalTable table, + boolean emptyInsert) { + super(context, table, "label", Mockito.mock(NereidsPlanner.class), + Optional.empty(), emptyInsert, -1L); + } + + @Override + protected void beforeExec() throws UserException { + throw new UserException("catalog runtime changed before begin"); + } + + @Override + protected void doBeforeCommit() { + } + + @Override + protected TransactionType transactionType() { + return TransactionType.ICEBERG; + } + + @Override + protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink physicalSink) { + } + } +} From 7ccd2dd646ce4f13d8a18a016d5e640ff9651eaf Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 2 Sep 2026 09:16:09 +0800 Subject: [PATCH 37/38] [fix](fe) Keep Iceberg schema projection on one generation ### What problem does this PR solve? Issue Number: None Related PR: #66914 Problem Summary: Iceberg scans pinned a frozen table and runtime but still read schema mapping switches from the mutable live catalog. Background native and HMS Iceberg schema callers also released their snapshot generation before deriving schema and row-lineage metadata, so a concurrent catalog refresh could splice snapshot identifiers from one generation with a different table, authenticator, or mapping configuration. Capture scan mapping options from the pinned snapshot and perform background snapshot, schema, and row-lineage projection under one bounded generation lease. ### Release note Fix Iceberg schema projection consistency across concurrent catalog refresh. ### Check List (For Author) - Test: Unit Test - Four focused Iceberg native and HMS generation-lifecycle tests - FE Checkstyle - Behavior changed: Yes, Iceberg schema conversion and backend mapping flags stay on the pinned table generation. - Does this need documentation: No --- .../datasource/hive/HMSExternalTable.java | 14 ++++---- .../iceberg/IcebergExternalMetaCache.java | 13 +++++++ .../iceberg/IcebergExternalTable.java | 22 ++++++------ .../datasource/iceberg/IcebergUtils.java | 21 +++++++++++ .../iceberg/source/IcebergScanNode.java | 16 +++++++++ .../datasource/hive/HMSExternalTableTest.java | 36 +++++++++++++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 4 +++ .../iceberg/IcebergExternalTableTest.java | 34 ++++++++++++++++++ .../iceberg/source/IcebergScanNodeTest.java | 19 ++++++++++ 9 files changed, 162 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 5cc00497e2bb28..3573a5c824754b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -45,7 +45,6 @@ import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergSchemaCacheKey; -import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.mvcc.EmptyMvccSnapshot; import org.apache.doris.datasource.mvcc.MvccSnapshot; @@ -376,7 +375,7 @@ public List getFullSchema() { return ((HudiDlaTable) dlaTable).getHudiSchemaCacheValue(MvccUtil.getSnapshotFromContext(this)) .getSchema(); } else if (getDlaType() == DLAType.ICEBERG) { - return IcebergUtils.getIcebergSchema(this); + return getIcebergSchema(MvccUtil.getSnapshotFromContext(this)).getSchema(); } Optional schemaCacheValue = getSchemaCacheValue(); return schemaCacheValue.map(SchemaCacheValue::getSchema).orElse(null); @@ -390,7 +389,7 @@ public List getFullSchema(Optional snapshot) { if (getDlaType() == DLAType.HUDI) { return ((HudiDlaTable) dlaTable).getHudiSchemaCacheValue(snapshot).getSchema(); } else if (getDlaType() == DLAType.ICEBERG) { - return IcebergUtils.getIcebergSchema(this, snapshot); + return getIcebergSchema(snapshot).getSchema(); } return super.getFullSchema(snapshot); } @@ -402,13 +401,16 @@ public Optional getSchemaCacheValue() { return Optional.of( ((HudiDlaTable) dlaTable).getHudiSchemaCacheValue(MvccUtil.getSnapshotFromContext(this))); } else if (dlaType == DLAType.ICEBERG) { - IcebergSnapshotCacheValue snapshotValue = IcebergUtils.getSnapshotCacheValue( - MvccUtil.getSnapshotFromContext(this), this); - return Optional.of(IcebergUtils.getSchemaCacheValue(this, snapshotValue)); + return Optional.of(getIcebergSchema(MvccUtil.getSnapshotFromContext(this))); } return super.getSchemaCacheValue(); } + private SchemaCacheValue getIcebergSchema(Optional snapshot) { + return IcebergUtils.withSnapshotCacheValue(snapshot, this, + snapshotValue -> IcebergUtils.getSchemaCacheValue(this, snapshotValue)); + } + public List getPartitionColumnTypes(Optional snapshot) { makeSureInitialized(); if (getDlaType() == DLAType.HUDI) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 1cce05db8f7bdd..ff24164f8fa66d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -327,6 +327,19 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { } } + /** The action must return metadata derived from the snapshot rather than retain its table. */ + public T withSnapshotCacheValue( + ExternalTable dorisTable, Function action) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + IcebergTableCacheValue.Lease statementLease = statementLease(nameMapping); + if (statementLease != null) { + return action.apply(getSnapshotCache(dorisTable, nameMapping, statementLease.getValue())); + } + try (IcebergTableCacheValue.Lease operationLease = borrow(nameMapping)) { + return action.apply(getSnapshotCache(dorisTable, nameMapping, operationLease.getValue())); + } + } + /** Build a snapshot bound to an action's retained writable generation without another lookup. */ public IcebergSnapshotCacheValue getSnapshotForWritableLease( ExternalTable dorisTable, WritableTableLease lease) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index e36d308f032b29..29440441ef8dec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -107,9 +107,9 @@ public Optional initSchema(SchemaCacheKey key) { @Override public Optional getSchemaCacheValue() { - IcebergSnapshotCacheValue snapshotValue = IcebergUtils.getSnapshotCacheValue( - MvccUtil.getSnapshotFromContext(this), this); - return Optional.of(IcebergUtils.getSchemaCacheValue(this, snapshotValue)); + return Optional.of(IcebergUtils.withSnapshotCacheValue( + MvccUtil.getSnapshotFromContext(this), this, + snapshotValue -> IcebergUtils.getSchemaCacheValue(this, snapshotValue))); } @Override @@ -278,21 +278,21 @@ public List getFullSchema() { @Override public List getFullSchema(Optional snapshot) { - List schema = IcebergUtils.getIcebergSchema(this, snapshot); + return IcebergUtils.withSnapshotCacheValue(snapshot, this, this::projectFullSchema); + } + + private List projectFullSchema(IcebergSnapshotCacheValue snapshotValue) { + List schema = IcebergUtils.getSchemaCacheValue(this, snapshotValue).getSchema(); schema = new ArrayList<>(schema); if (Util.showHiddenColumns() || needInternalHiddenColumns()) { schema.add(createIcebergRowIdColumn()); } - Optional
snapshotTable = snapshot - .filter(IcebergMvccSnapshot.class::isInstance) - .map(IcebergMvccSnapshot.class::cast) - .flatMap(value -> value.getSnapshotCacheValue().getIcebergTable()); // Row-lineage fields are part of the pinned schema generation, not the refreshable table. - schema = IcebergUtils.appendRowLineageColumnsForV3( - schema, snapshotTable.orElseGet(this::getIcebergTable)); - return schema; + return IcebergUtils.appendRowLineageColumnsForV3(schema, + snapshotValue.getIcebergTable().orElseThrow( + () -> new IllegalStateException("Iceberg schema projection lost its table generation"))); } private Column createIcebergRowIdColumn() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index cc1117ebb0c529..3afcadd9646f09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -2287,6 +2287,27 @@ public static IcebergSnapshotCacheValue getLatestSnapshotCacheValue(ExternalTabl return icebergExternalMetaCache(dorisTable).getSnapshotCache(dorisTable); } + /** The action must return metadata derived from the snapshot rather than retain its table. */ + public static T withLatestSnapshotCacheValue( + ExternalTable dorisTable, Function action) { + return icebergExternalMetaCache(dorisTable).withSnapshotCacheValue(dorisTable, action); + } + + /** Project metadata from a pinned snapshot or one bounded latest-generation lease. */ + public static T withSnapshotCacheValue(Optional snapshot, + ExternalTable dorisTable, Function action) { + Optional pinnedSnapshot = snapshot + .filter(IcebergMvccSnapshot.class::isInstance) + .map(IcebergMvccSnapshot.class::cast) + .map(IcebergMvccSnapshot::getSnapshotCacheValue); + if (pinnedSnapshot.isPresent()) { + Preconditions.checkState(pinnedSnapshot.get().getIcebergTable().isPresent(), + "Pinned Iceberg snapshot does not retain its table generation"); + return action.apply(pinnedSnapshot.get()); + } + return withLatestSnapshotCacheValue(dorisTable, action); + } + public static IcebergSnapshotCacheValue getSnapshotCacheValue(Optional snapshot, ExternalTable dorisTable) { if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 3d8851201ba421..a1b783cfbfafc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -198,6 +198,8 @@ public class IcebergScanNode extends FileQueryScanNode { private int formatVersion; private ExecutionAuthenticator preExecutionAuthenticator; private IcebergRuntimeContext runtimeContext; + private Boolean frozenEnableMappingVarbinary; + private Boolean frozenEnableMappingTimestampTz; private TableScan icebergTableScan; private Schema querySchema; // Store PropertiesMap, including vended credentials or static credentials @@ -1770,6 +1772,8 @@ private Table useFrozenTableGeneration(Table currentTable) { if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { IcebergSnapshotCacheValue cacheValue = ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); + frozenEnableMappingVarbinary = cacheValue.isEnableMappingVarbinary(); + frozenEnableMappingTimestampTz = cacheValue.isEnableMappingTimestampTz(); Optional
frozenTable = cacheValue.getIcebergTable(); if (frozenTable.isPresent()) { runtimeContext = cacheValue.getRuntimeContext(); @@ -1794,6 +1798,18 @@ private Table useFrozenTableGeneration(Table currentTable) { return currentTable; } + @Override + protected boolean getEnableMappingVarbinary() { + return frozenEnableMappingVarbinary == null + ? super.getEnableMappingVarbinary() : frozenEnableMappingVarbinary; + } + + @Override + protected boolean getEnableMappingTimestampTz() { + return frozenEnableMappingTimestampTz == null + ? super.getEnableMappingTimestampTz() : frozenEnableMappingTimestampTz; + } + private java.util.concurrent.ExecutorService getPlanningExecutor() { return runtimeContext == null ? source.getCatalog().getThreadPoolWithPreAuth() diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java index e68d3704a0cdda..26e5ff2e2f6007 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java @@ -22,10 +22,15 @@ import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.iceberg.IcebergSchemaCacheValue; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.fs.FileSystemDirectoryLister; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableMap; @@ -43,6 +48,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.function.Function; /** @@ -189,6 +196,35 @@ public void testFetchRowCountFillsMetaCacheOnlyWhenRequested() throws Exception } } + @Test + public void testIcebergSchemaAndColumnWithoutConnectContextUseScopedSnapshotProjection() { + ConnectContext.remove(); + Deencapsulation.setField(table, "dlaType", HMSExternalTable.DLAType.ICEBERG); + Column column = new Column("id", PrimitiveType.INT); + IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( + Lists.newArrayList(column), Lists.newArrayList()); + IcebergSnapshotCacheValue snapshotValue = Mockito.mock(IcebergSnapshotCacheValue.class); + org.apache.iceberg.Table icebergTable = Mockito.mock(org.apache.iceberg.Table.class); + Mockito.when(snapshotValue.getIcebergTable()).thenReturn(Optional.of(icebergTable)); + + try (MockedStatic icebergUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + icebergUtils.when(() -> IcebergUtils.getSchemaCacheValue(table, snapshotValue)) + .thenReturn(schemaValue); + icebergUtils.when(() -> IcebergUtils.withSnapshotCacheValue( + Mockito.eq(Optional.empty()), Mockito.eq(table), + Mockito.>any())) + .thenAnswer(invocation -> { + Function action = + invocation.getArgument(2); + return action.apply(snapshotValue); + }); + + Assertions.assertEquals(Lists.newArrayList(column), table.getFullSchema()); + Assertions.assertSame(column, table.getColumn("ID")); + } + } + private static class TestHMSExternalTableWithRemote extends HMSExternalTable { private final Table remoteTable; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index be0dc7d9e98f1a..8bdd0efc9499a3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -638,6 +638,10 @@ protected CatalogIf getCatalog(long catalogId) { cache.getSnapshotCacheWithRetainedTableForTest(dorisTable).getCapturedAuthenticator()); Assert.assertFalse("background snapshots must not expose an unowned table", cache.getSnapshotCache(dorisTable).getIcebergTable().isPresent()); + Assert.assertTrue(cache.withSnapshotCacheValue(dorisTable, snapshotValue -> { + Assert.assertSame(authenticator, snapshotValue.getCapturedAuthenticator()); + return snapshotValue.getIcebergTable().isPresent(); + })); initialized.set(false); Assert.assertSame(table, cache.getWritableIcebergTable(dorisTable)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java index e5b339f11eb128..b2959868a292ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.common.AnalysisException; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -35,6 +36,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -42,6 +44,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; public class IcebergExternalTableTest { @@ -250,6 +253,37 @@ public void testSortRange() throws AnalysisException { // ── helpers ──────────────────────────────────────────────────────────── + @Test + public void testFullSchemaAndColumnWithoutConnectContextUseScopedSnapshotProjection() { + ConnectContext.remove(); + IcebergExternalTable table = createSpyTable(); + Column column = new Column("id", PrimitiveType.INT); + IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( + Lists.newArrayList(column), Lists.newArrayList()); + IcebergSnapshotCacheValue snapshotValue = Mockito.mock(IcebergSnapshotCacheValue.class); + Mockito.when(snapshotValue.getIcebergTable()).thenReturn(java.util.Optional.of(icebergTable)); + Mockito.when(icebergTable.properties()).thenReturn(java.util.Collections.emptyMap()); + Mockito.doThrow(new AssertionError("must not reacquire a statement-only table")) + .when(table).getIcebergTable(); + + try (MockedStatic icebergUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + icebergUtils.when(() -> IcebergUtils.getSchemaCacheValue(table, snapshotValue)) + .thenReturn(schemaValue); + icebergUtils.when(() -> IcebergUtils.withLatestSnapshotCacheValue( + Mockito.eq(table), Mockito.>any())) + .thenAnswer(invocation -> { + Function action = invocation.getArgument(1); + return action.apply(snapshotValue); + }); + + Assertions.assertEquals(Lists.newArrayList(column), table.getFullSchema()); + Assertions.assertSame(column, table.getColumn("ID")); + Assertions.assertSame(schemaValue, table.getSchemaCacheValue().orElseThrow(AssertionError::new)); + } + Mockito.verify(table, Mockito.never()).getIcebergTable(); + } + private IcebergExternalTable createSpyTable() { IcebergExternalDatabase db = new IcebergExternalDatabase(mockCatalog, 1L, "db", "db"); IcebergExternalTable t = new IcebergExternalTable(1, "tbl", "tbl", mockCatalog, db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 941f30c4dd7019..8fa716e131ac19 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -3127,6 +3127,25 @@ public void testPinnedNonEmptyTableUsesFrozenGenerationAfterRefresh() throws Exc Mockito.verify(refreshedTable, Mockito.never()).newScan(); } + @Test + public void testPinnedGenerationUsesFrozenSchemaMappingOptions() throws Exception { + Table frozenTable = Mockito.mock(Table.class); + Table refreshedTable = Mockito.mock(Table.class); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(101L, 21L), + Optional.empty(), frozenTable).bindSchemaMappingOptions(true, false); + IcebergScanNode node = new IcebergScanNode( + new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), + new SessionVariable(), ScanContext.EMPTY); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot(snapshotValue))); + + Assert.assertSame(frozenTable, useFrozenTableGeneration(node, refreshedTable)); + Assert.assertTrue(node.getEnableMappingVarbinary()); + Assert.assertFalse(node.getEnableMappingTimestampTz()); + } + @Test public void testSnapshotSelectableMetadataTableUsesFrozenBaseGeneration() throws Exception { Schema schema = new Schema(21, ImmutableList.of( From b0611e134dd33b2ea9eaea48ded31a0ebe6f6e21 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 2 Sep 2026 10:13:13 +0800 Subject: [PATCH 38/38] [fix](fe) Close remaining Iceberg and Hudi lifecycle races ### What problem does this PR solve? Issue Number: close #66892 Related PR: #66913 Problem Summary: Iceberg schema and partition-column projection could release a no-context snapshot lease before resolving schema/spec metadata, allowing a concurrent catalog reset to splice two generations. HMS Iceberg cache loads also promoted a table without rechecking the captured generation after the remote load. Hudi batch cancellation marked queued tasks done but left their FutureTask objects in shared executor queues. Keep Iceberg projection inside one retained generation, fence HMS loads before promotion, and remove cancelled Hudi tasks from their exact executor queues. ### Release note Fix Iceberg and Hudi metadata handle lifecycle races during catalog reset and query cancellation. ### Check List (For Author) - Test: Unit Test - HudiBatchFsViewOwnerTest - IcebergUtilsTest - IcebergExternalMetaCacheTest - ./build.sh --fe -j4 - Behavior changed: No - Does this need documentation: No --- .../datasource/hudi/source/HudiScanNode.java | 68 ++++++++++++---- .../iceberg/IcebergExternalMetaCache.java | 3 + .../datasource/iceberg/IcebergUtils.java | 11 ++- .../hudi/source/HudiBatchFsViewOwnerTest.java | 80 ++++++++++++++++--- .../iceberg/IcebergExternalMetaCacheTest.java | 47 +++++++++++ .../datasource/iceberg/IcebergUtilsTest.java | 53 ++++++++++++ 6 files changed, 233 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 51cd3e50cf7872..89383786c151a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -659,7 +659,8 @@ public void startSplit(int numBackends) { ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor(); Executor producerExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); long startTime = System.currentTimeMillis(); - BatchFsViewOwner createdOwner = new BatchFsViewOwner(splitAssignment, fsViewLease); + BatchFsViewOwner createdOwner = new BatchFsViewOwner( + splitAssignment, fsViewLease, scheduleExecutor, producerExecutor); BatchFsViewOwner batchOwner = createdOwner; ConnectContext connectContext = ConnectContext.get(); StatementContext statementContext = connectContext == null ? null : connectContext.getStatementContext(); @@ -723,12 +724,13 @@ public void startSplit(int numBackends) { splittersOnFlight.release(); taskFinished.run(); }); - finalBatchOwner.track(partitionTask); try { - scheduleExecutor.execute(partitionTask); + if (!finalBatchOwner.submitPartition(partitionTask)) { + break; + } } catch (RuntimeException e) { recordBatchException(e); - partitionTask.cancelBeforeStart(); + finalBatchOwner.cancelPartition(partitionTask); break; } } @@ -736,12 +738,11 @@ public void startSplit(int numBackends) { recordBatchException(t); } }, taskFinished); - finalBatchOwner.track(producerTask); try { - producerExecutor.execute(producerTask); + finalBatchOwner.submitProducer(producerTask); } catch (RuntimeException e) { recordBatchException(e); - producerTask.cancelBeforeStart(); + finalBatchOwner.cancelProducer(producerTask); } } @@ -830,13 +831,18 @@ private void finishBatchSplit(BatchFsViewOwner batchOwner, long startTime) { static class BatchFsViewOwner implements Closeable { private final SplitAssignment splitAssignment; private final HudiFsViewCacheValue.Lease lease; + private final Executor scheduleExecutor; + private final Executor producerExecutor; private final AtomicBoolean finished = new AtomicBoolean(); - private final ConcurrentLinkedQueue tasks = new ConcurrentLinkedQueue<>(); + private final ConcurrentHashMap tasks = new ConcurrentHashMap<>(); private final AtomicBoolean stopping = new AtomicBoolean(); - BatchFsViewOwner(SplitAssignment splitAssignment, HudiFsViewCacheValue.Lease lease) { + BatchFsViewOwner(SplitAssignment splitAssignment, HudiFsViewCacheValue.Lease lease, + Executor scheduleExecutor, Executor producerExecutor) { this.splitAssignment = splitAssignment; this.lease = lease; + this.scheduleExecutor = scheduleExecutor; + this.producerExecutor = producerExecutor; } void finish() { @@ -849,12 +855,46 @@ void finish() { } } - void track(TerminalTask task) { + boolean submitPartition(TerminalTask task) { + return submit(task, scheduleExecutor); + } + + boolean submitProducer(TerminalTask task) { + return submit(task, producerExecutor); + } + + private boolean submit(TerminalTask task, Executor executor) { task.setOwnerDone(() -> tasks.remove(task)); - tasks.add(task); + tasks.put(task, executor); + if (stopping.get()) { + stopTask(task, executor); + return false; + } + executor.execute(task); if (stopping.get()) { - task.requestStop(); + stopTask(task, executor); + return false; } + return true; + } + + void cancelPartition(TerminalTask task) { + stopTask(task, scheduleExecutor); + } + + void cancelProducer(TerminalTask task) { + stopTask(task, producerExecutor); + } + + private void stopTask(TerminalTask task, Executor executor) { + task.requestStop(); + if (executor instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) executor).remove(task); + } + } + + private void stopTasks() { + tasks.forEach(this::stopTask); } @Override @@ -865,10 +905,10 @@ public void close() { try { splitAssignment.stop(); } catch (RuntimeException e) { - tasks.forEach(TerminalTask::requestStop); + stopTasks(); throw e; } - tasks.forEach(TerminalTask::requestStop); + stopTasks(); // Already-started filesystem calls may be blocked in storage code that does not respond to // interruption. Their TerminalTask.done callbacks retain exact task accounting and eventually call // finish(), which releases the fs-view lease only after the last task exits. Cancellation must return diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index ff24164f8fa66d..8126f628a6b14f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -588,6 +588,7 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { if (catalog instanceof HMSExternalCatalog) { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; try (HMSExternalCatalog.IcebergTableLoadContext context = hmsCatalog.beginIcebergTableLoad()) { + IcebergMetadataOps ops = context.getOps(); boolean enableMappingVarbinary = context.isEnableMappingVarbinary(); boolean enableMappingTimestampTz = context.isEnableMappingTimestampTz(); Table table; @@ -597,6 +598,8 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } try (TableResourceOwner owner = new TableResourceOwner(() -> { })) { + ensureCatalogGenerationStable(catalog, ops, context.getAuthenticator(), nameMapping, + enableMappingVarbinary, enableMappingTimestampTz); try (TableResourceOwner catalogOwner = new TableResourceOwner(context.promote()::close)) { IcebergTableCacheValue value = execute(context.getAuthenticator(), () -> createLoadedTableValue( nameMapping, table, context.getExecutor(), context.getAuthenticator(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 3afcadd9646f09..67a993e2b5bbeb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -2360,12 +2360,17 @@ public static List getIcebergSchema(ExternalTable dorisTable) { } public static List getIcebergSchema(ExternalTable dorisTable, Optional snapshot) { - IcebergSnapshotCacheValue cacheValue = IcebergUtils.getSnapshotCacheValue(snapshot, dorisTable); - return IcebergUtils.getSchemaCacheValue(dorisTable, cacheValue).getSchema(); + return withSnapshotCacheValue(snapshot, dorisTable, + cacheValue -> getSchemaCacheValue(dorisTable, cacheValue).getSchema()); } public static List getIcebergPartitionColumns(Optional snapshot, ExternalTable dorisTable) { - IcebergSnapshotCacheValue snapshotValue = getSnapshotCacheValue(snapshot, dorisTable); + return withSnapshotCacheValue(snapshot, dorisTable, + snapshotValue -> getIcebergPartitionColumns(dorisTable, snapshotValue)); + } + + private static List getIcebergPartitionColumns( + ExternalTable dorisTable, IcebergSnapshotCacheValue snapshotValue) { Optional
snapshotTable = snapshotValue.getIcebergTable(); if (snapshotTable.isPresent()) { // Schema ID alone cannot identify the partition spec; metadata-only evolution may keep diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java index bc2d5f04dae115..166de92685b6da 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java @@ -46,7 +46,8 @@ class HudiBatchFsViewOwnerTest { void statementCloseReturnsWhileRunningTaskKeepsLeasePinned() throws Exception { SplitAssignment assignment = Mockito.mock(SplitAssignment.class); HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner( + assignment, lease, Runnable::run, Runnable::run); ExecutorService executor = Executors.newSingleThreadExecutor(); try { Future close = executor.submit(owner::close); @@ -66,7 +67,8 @@ void statementCloseReturnsWhileRunningTaskKeepsLeasePinned() throws Exception { void normalCompletionDoesNotStopFinishedAssignment() { SplitAssignment assignment = Mockito.mock(SplitAssignment.class); HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner( + assignment, lease, Runnable::run, Runnable::run); owner.finish(); owner.close(); @@ -79,10 +81,12 @@ void normalCompletionDoesNotStopFinishedAssignment() { void statementCloseCancelsAcceptedTaskBeforeItStarts() { SplitAssignment assignment = Mockito.mock(SplitAssignment.class); HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + List submitted = new ArrayList<>(); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner( + assignment, lease, submitted::add, submitted::add); HudiScanNode.TerminalTask task = new HudiScanNode.TerminalTask( () -> Assertions.fail("cancelled task must not run"), owner::finish); - owner.track(task); + Assertions.assertTrue(owner.submitPartition(task)); owner.close(); @@ -91,11 +95,50 @@ void statementCloseCancelsAcceptedTaskBeforeItStarts() { Mockito.verify(lease).close(); } + @Test + void statementCloseRemovesQueuedBatchTasksFromBothExecutors() throws Exception { + SplitAssignment assignment = Mockito.mock(SplitAssignment.class); + HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); + CountDownLatch workersStarted = new CountDownLatch(2); + CountDownLatch releaseWorkers = new CountDownLatch(1); + ThreadPoolExecutor scheduleExecutor = blockedExecutor(workersStarted, releaseWorkers); + ThreadPoolExecutor producerExecutor = blockedExecutor(workersStarted, releaseWorkers); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner( + assignment, lease, scheduleExecutor, producerExecutor); + HudiScanNode.TerminalTask partitionTask = new HudiScanNode.TerminalTask( + () -> Assertions.fail("cancelled partition task must not run"), () -> { }); + HudiScanNode.TerminalTask producerTask = new HudiScanNode.TerminalTask( + () -> Assertions.fail("cancelled producer task must not run"), () -> { }); + try { + Assertions.assertTrue(workersStarted.await(3, TimeUnit.SECONDS)); + Assertions.assertTrue(owner.submitPartition(partitionTask)); + Assertions.assertTrue(owner.submitProducer(producerTask)); + Assertions.assertEquals(1, scheduleExecutor.getQueue().size()); + Assertions.assertEquals(1, producerExecutor.getQueue().size()); + + owner.close(); + + Assertions.assertTrue(partitionTask.isCancelled()); + Assertions.assertTrue(producerTask.isCancelled()); + Assertions.assertTrue(scheduleExecutor.getQueue().isEmpty()); + Assertions.assertTrue(producerExecutor.getQueue().isEmpty()); + Mockito.verify(assignment).stop(); + } finally { + owner.finish(); + releaseWorkers.countDown(); + scheduleExecutor.shutdownNow(); + producerExecutor.shutdownNow(); + } + Mockito.verify(lease).close(); + } + @Test void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { SplitAssignment assignment = Mockito.mock(SplitAssignment.class); HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + ExecutorService executor = Executors.newSingleThreadExecutor(); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner( + assignment, lease, executor, executor); CountDownLatch started = new CountDownLatch(1); CountDownLatch interrupted = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); @@ -113,10 +156,8 @@ void statementCloseDoesNotWaitForAlreadyStartedBlockedTask() throws Exception { owner.finish(); terminal.countDown(); }); - owner.track(task); - ExecutorService executor = Executors.newSingleThreadExecutor(); + Assertions.assertTrue(owner.submitPartition(task)); try { - executor.execute(task); Assertions.assertTrue(started.await(3, TimeUnit.SECONDS)); owner.close(); @@ -138,7 +179,9 @@ void assignmentStopInterruptsStartedTaskAndRetainsLeaseUntilTerminal() throws Ex SplitAssignment assignment = new SplitAssignment( null, null, null, Collections.emptyMap(), Collections.emptyList(), false); HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); - HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner(assignment, lease); + ExecutorService executor = Executors.newSingleThreadExecutor(); + HudiScanNode.BatchFsViewOwner owner = new HudiScanNode.BatchFsViewOwner( + assignment, lease, executor, executor); assignment.addCloseable(owner); CountDownLatch started = new CountDownLatch(1); CountDownLatch interrupted = new CountDownLatch(1); @@ -157,10 +200,8 @@ void assignmentStopInterruptsStartedTaskAndRetainsLeaseUntilTerminal() throws Ex owner.finish(); terminal.countDown(); }); - owner.track(task); - ExecutorService executor = Executors.newSingleThreadExecutor(); + Assertions.assertTrue(owner.submitPartition(task)); try { - executor.execute(task); Assertions.assertTrue(started.await(3, TimeUnit.SECONDS)); assignment.stop(); @@ -354,4 +395,19 @@ void synchronousListingCloseDuringImmediateSubmissionStopsTask() { Mockito.verify(lease).close(); Assertions.assertThrows(CancellationException.class, owner::awaitCompletion); } + + private static ThreadPoolExecutor blockedExecutor( + CountDownLatch workersStarted, CountDownLatch releaseWorkers) { + ThreadPoolExecutor executor = new ThreadPoolExecutor( + 1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + executor.execute(() -> { + workersStarted.countDown(); + try { + releaseWorkers.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + return executor; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 8bdd0efc9499a3..533a00bc2ca25d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -23,6 +23,7 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; @@ -93,6 +94,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -584,6 +586,51 @@ protected CatalogIf getCatalog(long catalogId) { } } + @Test + public void testHmsTableLoadRejectsGenerationResetBeforePromotion() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + IcebergMetadataOps loadingOps = Mockito.mock(IcebergMetadataOps.class); + IcebergMetadataOps replacementOps = Mockito.mock(IcebergMetadataOps.class); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { }; + AtomicReference currentOps = new AtomicReference<>(loadingOps); + Mockito.when(catalog.getIcebergMetadataOps()).thenAnswer(invocation -> currentOps.get()); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + HMSExternalCatalog.IcebergTableLoadContext context = + Mockito.mock(HMSExternalCatalog.IcebergTableLoadContext.class); + Mockito.when(catalog.beginIcebergTableLoad()).thenReturn(context); + Mockito.when(context.getOps()).thenReturn(loadingOps); + Mockito.when(context.getAuthenticator()).thenReturn(authenticator); + Mockito.when(context.loadTable("remote_db", "remote_tbl")).thenAnswer(invocation -> { + currentOps.set(replacementOps); + return Mockito.mock(Table.class); + }); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + + try { + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + Assert.fail("an HMS load crossing a catalog reset must not be published"); + } catch (RuntimeException expected) { + Assert.assertTrue(exceptionChainContains(expected, + "was reset while acquiring iceberg table")); + } + Mockito.verify(context, Mockito.never()).promote(); + Mockito.verify(context).close(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testResetCatalogReinitializesBeforeCaptureAndWritableStaysOnDispatchGeneration() { IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index f495c97d25a1ca..7ad74fc9e9c92a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -20,11 +20,15 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.property.storage.OSSProperties; import org.apache.doris.datasource.property.storage.S3Properties; @@ -65,6 +69,7 @@ import org.apache.iceberg.types.Types.StructType; import org.junit.Assert; import org.junit.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.lang.reflect.Field; @@ -83,7 +88,9 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; public class IcebergUtilsTest { @Test @@ -224,6 +231,52 @@ public void testPartitionColumnsUseFrozenTableSpec() { .map(Column::getName).collect(java.util.stream.Collectors.toList())); } + @Test + public void testPartitionColumnsProjectInsideSnapshotLease() { + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheManager = Mockito.mock(ExternalMetaCacheMgr.class); + IcebergExternalMetaCache cache = Mockito.mock(IcebergExternalMetaCache.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table frozenTable = Mockito.mock(Table.class); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(11L, 17L), + Optional.empty(), frozenTable); + List partitionColumns = Collections.singletonList(new Column("p", Type.INT)); + IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( + partitionColumns, partitionColumns); + AtomicBoolean leaseActive = new AtomicBoolean(); + Mockito.when(dorisTable.getCatalog()).thenReturn(catalog); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + Mockito.when(catalog.getId()).thenReturn(1L); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheManager); + Mockito.when(cacheManager.iceberg(1L)).thenReturn(cache); + Mockito.when(cache.withSnapshotCacheValue(Mockito.eq(dorisTable), Mockito.any())) + .thenAnswer(invocation -> { + leaseActive.set(true); + try { + Function> projection = invocation.getArgument(1); + return projection.apply(snapshotValue); + } finally { + leaseActive.set(false); + } + }); + Mockito.when(cache.getIcebergSchemaCacheValue(mapping, 17L, frozenTable)) + .thenAnswer(invocation -> { + Assert.assertTrue("schema/spec projection must remain inside the snapshot lease", leaseActive.get()); + return schemaValue; + }); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + Assert.assertSame(partitionColumns, + IcebergUtils.getIcebergPartitionColumns(Optional.empty(), dorisTable)); + } + Mockito.verify(cache, Mockito.never()).getSnapshotCache(dorisTable); + } + @Test public void testParseTableName() { try {