From 0bce30e2a0b99ba079793a1fffd40ab5693476a0 Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 18 Jul 2025 17:06:10 +0530 Subject: [PATCH 1/2] fix: Fixed ExpiredMetadataHandler s3 path cleanup issue on retry --- CHANGELOG.md | 4 ++++ .../metadata/cleanup/handler/ExpiredMetadataHandler.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42821dbb..b7b909f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.6.4] - 2025-08-04 +## Fixed +- Fixed ExpiredMetadataHandler s3 path cleanup issue on retry. + ## [3.6.3] - 2025-07-28 ## Added - Added multiplatform support for beekeeper images. Added amd64 and arm64. diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java index 1db7c5cf..373228b8 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java @@ -165,7 +165,7 @@ private boolean cleanupPartition( log.info("Cleaning up metadata for \"{}.{}\"", databaseName, tableName); if (metadataCleaner.tableExists(client, databaseName, tableName)) { boolean partitionDeleted = metadataCleaner.dropPartition(housekeepingMetadata, client); - if (partitionDeleted) { + if (partitionDeleted || housekeepingMetadata.getCleanupAttempts() > 0) { pathCleaner.cleanupPath(housekeepingMetadata); } } else { From b31934bb69c0e8d8db6b434845f80787b1655475 Mon Sep 17 00:00:00 2001 From: rahul139vermaa Date: Tue, 5 Aug 2025 18:25:35 +0530 Subject: [PATCH 2/2] fix: Added UT for s3 path cleanup issue on retry --- .../handler/ExpiredMetadataHandlerTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java index 43f90d76..0a99227f 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java @@ -190,6 +190,27 @@ public void typicalRunDroppingPartition() { verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } + @Test + public void runDroppingPartition_retry() { + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); + when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); + when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); + when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); + when(housekeepingMetadata.getCleanupAttempts()).thenReturn(1); + + when(hiveMetadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(false); + when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); + + expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, false); + verify(s3PathCleaner).cleanupPath(housekeepingMetadata); + verify(hiveMetadataCleaner, never()).dropTable(housekeepingMetadata, hiveClient); + verify(housekeepingMetadata).setCleanupAttempts(2); + verify(housekeepingMetadata).setHousekeepingStatus(DELETED); + verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); + } + @Test public void dontDropTableWithInvalidPath() { when(hiveClientFactory.newInstance()).thenReturn(hiveClient);