From 655ac31eeecc0f66271f77d211d85285b328c430 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Mon, 29 Jul 2024 14:33:34 -0700 Subject: [PATCH 01/12] Updated index file recovery method with changes from fixed data recovery --- src/embedDB/embedDB.c | 71 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index 025adacc..6cb73982 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -657,35 +657,84 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { id_t logicalIndexPageId = 0; id_t maxLogicaIndexPageId = 0; id_t physicalIndexPageId = 0; + uint32_t count = 0; + count_t blockSize = state->eraseSizeInPages; + bool validData = false; + bool hasData = false; + void *buffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_READ_BUFFER; /* This will become zero if there is no more to read */ int8_t moreToRead = !(readIndexPage(state, physicalIndexPageId)); - bool haveWrappedInMemory = false; - int count = 0; - void *buffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_READ_BUFFER; + /* this handles the case where the first page may have been erased, so has junk data and we actually need to start from the second page */ + uint32_t i = 0; + int8_t indexCount = 0; + while (moreToRead && i < 2) { + memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); + validData = logicalIndexPageId % state->numIndexPages == count; + indexCount = EMBEDDB_GET_COUNT(buffer); + if (validData && indexCount > 0 && indexCount < state->maxIdxRecordsPerPage + 1) { + hasData = true; + maxLogicaIndexPageId = logicalIndexPageId; + physicalIndexPageId++; + count++; + i = 2; + } else { + physicalIndexPageId += blockSize; + count += blockSize; + } + moreToRead = !(readIndexPage(state, physicalIndexPageId)); + i++; + } + + /* if we have no valid data, we just have an empty file can can start from the scratch */ + if (!hasData) + return 0; while (moreToRead && count < state->numIndexPages) { memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); - if (count == 0 || logicalIndexPageId == maxLogicaIndexPageId + 1) { + validData = logicalIndexPageId % state->numIndexPages == count; + if (validData && logicalIndexPageId == maxLogicaIndexPageId + 1) { maxLogicaIndexPageId = logicalIndexPageId; physicalIndexPageId++; moreToRead = !(readIndexPage(state, physicalIndexPageId)); count++; } else { - haveWrappedInMemory = logicalIndexPageId == maxLogicaIndexPageId - state->numIndexPages + 1; break; } } - if (count == 0) - return 0; - - state->nextIdxPageId = maxLogicaIndexPageId + 1; + /* + * Now we need to find where the page with the smallest key that is still valid. + * The default case is we have not wrapped and the page number for the physical page with the smallest key is 0. + */ id_t physicalPageIDOfSmallestData = 0; - if (haveWrappedInMemory) { - physicalPageIDOfSmallestData = logicalIndexPageId % state->numIndexPages; + + /* check if data exists at this location */ + if (moreToRead && count < state->numIndexPages) { + /* find where the next block boundary is */ + id_t pagesToBlockBoundary = blockSize - (count % blockSize); + + /* go to the next block boundary */ + physicalIndexPageId = (physicalIndexPageId + pagesToBlockBoundary) % state->numIndexPages; + moreToRead = !(readIndexPage(state, physicalIndexPageId)); + + /* there should have been more to read becuase the file should not be empty at this point if it was not empty at the previous block */ + if (!moreToRead) { + return -1; + } + + /* check if data is valid or if it is junk */ + memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); + validData = logicalIndexPageId % state->numIndexPages == physicalIndexPageId; + + /* this means we have wrapped and our start is actually here */ + if (validData) { + physicalPageIDOfSmallestData = physicalIndexPageId; + } } + + state->nextIdxPageId = maxLogicaIndexPageId + 1; readIndexPage(state, physicalPageIDOfSmallestData); memcpy(&(state->minIndexPageId), buffer, sizeof(id_t)); state->numAvailIndexPages = state->numIndexPages + state->minIndexPageId - maxLogicaIndexPageId - 1; From c22c197cae3779dcf74130d37f20dba821051a0e Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Mon, 29 Jul 2024 15:34:21 -0700 Subject: [PATCH 02/12] Finished updating tests and index file recovery code --- src/embedDB/embedDB.c | 22 ++++- .../test_buffered_read/test_buffered_read.cpp | 2 +- .../test_embedDB_index_recovery.cpp | 83 +++++++++---------- .../test_record_level_consistency.cpp | 30 +++---- test/test_spline/test_spline.cpp | 4 +- .../test_var_data_buffered_read.cpp | 2 +- ...test_var_data_record_level_consistency.cpp | 16 ++-- 7 files changed, 84 insertions(+), 75 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index 6cb73982..44bed293 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -152,9 +152,9 @@ int8_t embedDBInit(embedDBState *state, size_t indexMaxError) { return -1; } - if (state->numDataPages < (EMBEDDB_USING_RECORD_LEVEL_CONSISTENCY(state->parameters) ? 4 : 2) * state->eraseSizeInPages) { + if (state->numDataPages < 4 * state->eraseSizeInPages) { #ifdef PRINT_ERRORS - printf("ERROR: The minimum number of data pages is twice the eraseSizeInPages or 4 times the eraseSizeInPages if using record-level consistency.\n"); + printf("ERROR: The minimum number of data pages is 4 times the eraseSizeInPages.\n"); #endif return -1; } @@ -167,6 +167,14 @@ int8_t embedDBInit(embedDBState *state, size_t indexMaxError) { #endif return -1; } + + if (state->numVarPages < 4 * state->eraseSizeInPages) { +#ifdef PRINT_ERRORS + printf("ERROR: The minimum number of variable data pages is 4 times the eraseSizeInPages.\n"); +#endif + return -1; + } + state->recordSize += 4; } @@ -183,6 +191,14 @@ int8_t embedDBInit(embedDBState *state, size_t indexMaxError) { #endif return -1; } + + if (state->numIndexPages < 2 * state->eraseSizeInPages) { +#ifdef PRINT_ERRORS + printf("ERROR: The minimum number of index pages is 4 times the eraseSizeInPages.\n"); +#endif + return -1; + } + state->headerSize += state->bitmapSize; } @@ -668,7 +684,7 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { /* this handles the case where the first page may have been erased, so has junk data and we actually need to start from the second page */ uint32_t i = 0; - int8_t indexCount = 0; + count_t indexCount = 0; while (moreToRead && i < 2) { memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); validData = logicalIndexPageId % state->numIndexPages == count; diff --git a/test/test_buffered_read/test_buffered_read.cpp b/test/test_buffered_read/test_buffered_read.cpp index 83d9d690..c918f22a 100644 --- a/test/test_buffered_read/test_buffered_read.cpp +++ b/test/test_buffered_read/test_buffered_read.cpp @@ -316,7 +316,7 @@ embedDBState* init_state() { // address level parameters state->numDataPages = 256; - state->numIndexPages = 8; + state->numIndexPages = 16; state->eraseSizeInPages = 4; // configure file interface diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index 9bd581bf..74ba439a 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -68,8 +68,9 @@ #include "unity.h" embedDBState *state; +const int16_t RECOVERY_PARAMETERS = EMBEDDB_USE_INDEX; -void setupEmbedDB() { +void setupEmbedDB(int16_t parameters) { state = (embedDBState *)malloc(sizeof(embedDBState)); state->keySize = 4; state->dataSize = 4; @@ -86,38 +87,9 @@ void setupEmbedDB() { state->numDataPages = 10000; state->eraseSizeInPages = 2; - state->numIndexPages = 4; + state->numIndexPages = 16; state->bitmapSize = 1; - state->parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA; - state->inBitmap = inBitmapInt8; - state->updateBitmap = updateBitmapInt8; - state->buildBitmapFromRange = buildBitmapInt8FromRange; - state->compareKey = int32Comparator; - state->compareData = int32Comparator; - int8_t result = embedDBInit(state, 1); - TEST_ASSERT_EQUAL_INT8_MESSAGE(0, result, "EmbedDB did not initialize correctly."); -} - -void initalizeEmbedDBFromFile() { - state = (embedDBState *)malloc(sizeof(embedDBState)); - state->keySize = 4; - state->dataSize = 4; - state->pageSize = 512; - state->bufferSizeInBlocks = 4; - state->numSplinePoints = 8; - state->buffer = calloc(1, state->pageSize * state->bufferSizeInBlocks); - TEST_ASSERT_NOT_NULL_MESSAGE(state->buffer, "Failed to allocate EmbedDB buffer."); - - /* initialize EmbedDB storage */ - state->fileInterface = getFileInterface(); - state->dataFile = setupFile(DATA_FILE_PATH); - state->indexFile = setupFile(INDEX_FILE_PATH); - - state->numDataPages = 10000; - state->numIndexPages = 4; - state->eraseSizeInPages = 2; - state->bitmapSize = 1; - state->parameters = EMBEDDB_USE_INDEX; + state->parameters = parameters; state->inBitmap = inBitmapInt8; state->updateBitmap = updateBitmapInt8; state->buildBitmapFromRange = buildBitmapInt8FromRange; @@ -128,7 +100,8 @@ void initalizeEmbedDBFromFile() { } void setUp() { - setupEmbedDB(); + int16_t parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA; + setupEmbedDB(parameters); } void tearDown() { @@ -155,38 +128,56 @@ void insertRecordsLinearly(int32_t startingKey, int32_t startingData, int32_t nu void embedDB_index_file_correctly_reloads_with_no_data() { tearDown(); - initalizeEmbedDBFromFile(); + setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(496, state->maxIdxRecordsPerPage, "EmbedDB maxIdxRecordsPerPage was initialized incorrectly when no data was present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when no data was present in the index file."); - TEST_ASSERT_EQUAL_UINT32_MESSAGE(4, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when no data was present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when no data was present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when no data was present in the index file."); } void embedDB_index_file_correctly_reloads_with_one_page_of_data() { insertRecordsLinearly(100, 100, 31312); tearDown(); - initalizeEmbedDBFromFile(); + setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when one index page was present in the index file."); - TEST_ASSERT_EQUAL_UINT32_MESSAGE(3, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when one index page was present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(15, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when one index page was present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when one index page was present in the index file."); } void embedDB_index_file_correctly_reloads_with_four_pages_of_data() { insertRecordsLinearly(100, 100, 125056); tearDown(); - initalizeEmbedDBFromFile(); + setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(4, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); - TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(12, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); } -void embedDB_index_file_correctly_reloads_with_eleven_pages_of_data() { - insertRecordsLinearly(100, 100, 343792); +void embedDB_index_file_correctly_reloads_with_eight_pages_of_data() { + insertRecordsLinearly(100, 100, 250111); tearDown(); - initalizeEmbedDBFromFile(); - TEST_ASSERT_EQUAL_UINT32_MESSAGE(11, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + setupEmbedDB(RECOVERY_PARAMETERS); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(8, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(8, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); +} + +void embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data() { + insertRecordsLinearly(100, 100, 500222); + tearDown(); + setupEmbedDB(RECOVERY_PARAMETERS); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); - TEST_ASSERT_EQUAL_UINT32_MESSAGE(7, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); +} + +void embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data() { + insertRecordsLinearly(100, 100, 532288); + tearDown(); + setupEmbedDB(RECOVERY_PARAMETERS); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(17, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(2, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); } int runUnityTests() { @@ -194,7 +185,9 @@ int runUnityTests() { RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); RUN_TEST(embedDB_index_file_correctly_reloads_with_one_page_of_data); RUN_TEST(embedDB_index_file_correctly_reloads_with_four_pages_of_data); - RUN_TEST(embedDB_index_file_correctly_reloads_with_eleven_pages_of_data); + RUN_TEST(embedDB_index_file_correctly_reloads_with_eight_pages_of_data); + RUN_TEST(embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data); + RUN_TEST(embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data); return UNITY_END(); } diff --git a/test/test_record_level_consistency/test_record_level_consistency.cpp b/test/test_record_level_consistency/test_record_level_consistency.cpp index e42bcdf6..2e4fab56 100644 --- a/test/test_record_level_consistency/test_record_level_consistency.cpp +++ b/test/test_record_level_consistency/test_record_level_consistency.cpp @@ -34,7 +34,7 @@ embedDBState *state; -void setupEmbedDB(int8_t parameters) { +void setupEmbedDB(int16_t parameters) { /* The setup below will result in having 42 records per page */ state = (embedDBState *)malloc(sizeof(embedDBState)); TEST_ASSERT_NOT_NULL_MESSAGE(state, "Unable to allocate embedDBState."); @@ -65,7 +65,7 @@ void setupEmbedDB(int8_t parameters) { } void setUp() { - int8_t setupParamaters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_RESET_DATA; + int16_t setupParamaters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_RESET_DATA; setupEmbedDB(setupParamaters); } @@ -180,7 +180,7 @@ void record_level_consistency_blocks_should_wrap_when_storage_is_full() { void embedDBInit_should_detect_when_no_records_written_with_record_level_consistency() { /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* test that we recovered correctly to the default state */ @@ -197,7 +197,7 @@ void embedDBInit_should_recover_record_level_consistency_records_when_no_permane /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* test that we recovered correctly */ @@ -235,7 +235,7 @@ void embedDBInit_should_recover_record_level_consistency_records_when_one_perman /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -258,7 +258,7 @@ void embedDBInit_should_recover_record_level_consistency_records_when_four_perma /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -303,7 +303,7 @@ void embedDBInit_should_recover_record_level_consistency_records_when_eight_perm /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -333,7 +333,7 @@ void embedDBInit_should_recover_record_level_consistency_records_when_twenty_one /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -355,7 +355,7 @@ void embedDBInit_should_recover_record_level_consistency_records_when_twenty_thr /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -387,7 +387,7 @@ void embedDBInit_should_recover_correctly_with_one_wraped_record_level_consisten /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -419,7 +419,7 @@ void embedDBInit_should_recover_correctly_with_both_record_level_consistency_blo /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -451,7 +451,7 @@ void embedDBInit_should_recover_correctly_with_junk_data_at_start_of_data_file() /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that 32 pages of records and 33 RLC recordsy */ @@ -479,7 +479,7 @@ void embedDBInit_should_recover_correctly_after_wrapping_with_one_page_of_data_a /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -507,7 +507,7 @@ void embedDBInit_should_recover_correctly_when_old_permanent_records_in_record_l /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ @@ -524,7 +524,7 @@ void embedDBInit_should_recover_correctly_after_wrapping_several_times() { /* close embedDB and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY; setupEmbedDB(setupParameters); /* Check that data was initialised correctly */ diff --git a/test/test_spline/test_spline.cpp b/test/test_spline/test_spline.cpp index 741b5480..18ee469b 100644 --- a/test/test_spline/test_spline.cpp +++ b/test/test_spline/test_spline.cpp @@ -32,7 +32,7 @@ embedDBState *state; -void setupEmbedDB(int8_t parameters) { +void setupEmbedDB(int16_t parameters) { /* The setup below will result in having 42 records per page */ state = (embedDBState *)malloc(sizeof(embedDBState)); TEST_ASSERT_NOT_NULL_MESSAGE(state, "Unable to allocate embedDBState."); @@ -63,7 +63,7 @@ void setupEmbedDB(int8_t parameters) { } void setUp() { - int8_t setupParamaters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_RESET_DATA; + int16_t setupParamaters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_RESET_DATA; setupEmbedDB(setupParamaters); } diff --git a/test/test_var_data_buffered_read/test_var_data_buffered_read.cpp b/test/test_var_data_buffered_read/test_var_data_buffered_read.cpp index 46224133..eb0d97e6 100644 --- a/test/test_var_data_buffered_read/test_var_data_buffered_read.cpp +++ b/test/test_var_data_buffered_read/test_var_data_buffered_read.cpp @@ -462,7 +462,7 @@ embedDBState *init_state() { // address level parameters state->numDataPages = 32; state->numIndexPages = 8; - state->numVarPages = 12; + state->numVarPages = 16; state->eraseSizeInPages = 4; // configure file interface diff --git a/test/test_var_data_record_level_consistency/test_var_data_record_level_consistency.cpp b/test/test_var_data_record_level_consistency/test_var_data_record_level_consistency.cpp index 8e7c7e06..4eaae3f3 100644 --- a/test/test_var_data_record_level_consistency/test_var_data_record_level_consistency.cpp +++ b/test/test_var_data_record_level_consistency/test_var_data_record_level_consistency.cpp @@ -38,7 +38,7 @@ embedDBState *state; -void setupEmbedDB(int8_t parameters) { +void setupEmbedDB(int16_t parameters) { /* The setup below will result in having 42 records per page */ state = (embedDBState *)malloc(sizeof(embedDBState)); TEST_ASSERT_NOT_NULL_MESSAGE(state, "Unable to allocate embedDBState."); @@ -73,7 +73,7 @@ void setupEmbedDB(int8_t parameters) { } void setUp() { - int8_t setupParamaters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_RESET_DATA | EMBEDDB_USE_VDATA; + int16_t setupParamaters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_RESET_DATA | EMBEDDB_USE_VDATA; setupEmbedDB(setupParamaters); } @@ -119,7 +119,7 @@ void variable_data_record_level_consistency_records_should_be_readable() { /* tear down state and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; setupEmbedDB(setupParameters); /* Check that the state was correctly setup again */ @@ -166,7 +166,7 @@ void variable_data_record_level_consistency_should_recover_64_records_correctly( /* tear down state and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; setupEmbedDB(setupParameters); /* Check that state was initialised correctly */ @@ -219,7 +219,7 @@ void variable_data_record_level_consistency_should_recover_four_pages_data_recor /* tear down state and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; setupEmbedDB(setupParameters); /* Check that state was initialised correctly */ @@ -283,7 +283,7 @@ void variable_data_record_level_consistency_should_recover_71_pages_data_and_19_ /* tear down state and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; setupEmbedDB(setupParameters); /* Check that state was initialised correctly */ @@ -338,7 +338,7 @@ void variable_data_record_level_consistency_should_recover_variable_data_longer_ /* tear down state and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; setupEmbedDB(setupParameters); /* Check that state was initialised correctly */ @@ -404,7 +404,7 @@ void variable_data_record_level_consistency_should_recover_after_inserting_131_p /* tear down state and recover */ tearDown(); - int8_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; + int16_t setupParameters = EMBEDDB_RECORD_LEVEL_CONSISTENCY | EMBEDDB_USE_VDATA; setupEmbedDB(setupParameters); /* Check that state was initialised correctly */ From c7d663a9dacf6c987d298e37145401f73024a076 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Tue, 30 Jul 2024 10:55:21 -0700 Subject: [PATCH 03/12] Refactored data index into seperate method and removed index page write from embedDBFlush when not required --- src/embedDB/embedDB.c | 112 +++++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 44 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index 44bed293..cda3dd0a 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -673,7 +673,8 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { id_t logicalIndexPageId = 0; id_t maxLogicaIndexPageId = 0; id_t physicalIndexPageId = 0; - uint32_t count = 0; + id_t count = 0; + id_t numberOfBitmaps = 0; count_t blockSize = state->eraseSizeInPages; bool validData = false; bool hasData = false; @@ -695,6 +696,7 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { physicalIndexPageId++; count++; i = 2; + numberOfBitmaps += indexCount; } else { physicalIndexPageId += blockSize; count += blockSize; @@ -714,6 +716,8 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { maxLogicaIndexPageId = logicalIndexPageId; physicalIndexPageId++; moreToRead = !(readIndexPage(state, physicalIndexPageId)); + indexCount = EMBEDDB_GET_COUNT(buffer); + numberOfBitmaps += indexCount; count++; } else { break; @@ -755,6 +759,9 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { memcpy(&(state->minIndexPageId), buffer, sizeof(id_t)); state->numAvailIndexPages = state->numIndexPages + state->minIndexPageId - maxLogicaIndexPageId - 1; + /* Compute how many bitmaps we should have and how many we do have */ + id_t numberOfDataPages = state->nextDataPageId - state->minDataPageId; + return 0; } @@ -1059,6 +1066,40 @@ void indexPage(embedDBState *state, uint32_t pageNumber) { } } +/** + * @brief Adds an entry into the data index for a given page. + * @param state embedDB algorithm state structure + * @param pageNumber Page number to index + * @return Return 0 if success. Non-zero value if error. + */ +int8_t indexData(embedDBState *state, id_t pageNumber) { + void *buf = (int8_t *)state->buffer + state->pageSize * (EMBEDDB_INDEX_WRITE_BUFFER); + count_t idxcount = EMBEDDB_GET_COUNT(buf); + if (idxcount >= state->maxIdxRecordsPerPage) { + /* Save index page */ + id_t indexPageNumber = writeIndexPage(state, buf); + if (indexPageNumber == -1) { +#ifdef PRINT_ERRORS + printf("ERROR: failed to write index page to storage"); +#endif + return -1; + } + + idxcount = 0; + initBufferPage(state, EMBEDDB_INDEX_WRITE_BUFFER); + + /* Add page id to minimum value spot in page */ + memcpy((int8_t *)buf + 8, &pageNumber, sizeof(id_t)); + } + + EMBEDDB_INC_COUNT(buf); + + /* Copy record onto index page */ + void *bm = EMBEDDB_GET_BITMAP(state->buffer); + memcpy((void *)((int8_t *)buf + EMBEDDB_IDX_HEADER_SIZE + state->bitmapSize * idxcount), bm, state->bitmapSize); + return 0; +} + /** * @brief Puts a given key, data pair into structure. * @param state embedDB algorithm state structure @@ -1090,35 +1131,29 @@ int8_t embedDBPut(embedDBState *state, void *key, void *data) { /* Write current page if full */ bool wrotePage = false; if (count >= state->maxRecordsPerPage) { - // As the first buffer is the data write buffer, no manipulation is required - id_t pageNum = writePage(state, state->buffer); + void *dataWriteBuffer = state->buffer + state->pageSize * EMBEDDB_DATA_WRITE_BUFFER; + id_t pageNum = writePage(state, dataWriteBuffer); + if (pageNum == -1) { +#ifdef PRINT_ERRORS + printf("ERROR: Unable to write data page when buffer is full. \n"); +#endif + return 2; + } indexPage(state, pageNum); /* Save record in index file */ - if (state->indexFile != NULL) { - void *buf = (int8_t *)state->buffer + state->pageSize * (EMBEDDB_INDEX_WRITE_BUFFER); - count_t idxcount = EMBEDDB_GET_COUNT(buf); - if (idxcount >= state->maxIdxRecordsPerPage) { - /* Save index page */ - writeIndexPage(state, buf); - - idxcount = 0; - initBufferPage(state, EMBEDDB_INDEX_WRITE_BUFFER); - - /* Add page id to minimum value spot in page */ - id_t *ptr = (id_t *)((int8_t *)buf + 8); - *ptr = pageNum; + if (EMBEDDB_USING_INDEX(state->parameters)) { + int8_t indexResult = indexData(state, pageNum); + if (pageNum == -1) { +#ifdef PRINT_ERRORS + printf("ERROR: Unable to index data. \n"); +#endif + return 3; } - - EMBEDDB_INC_COUNT(buf); - - /* Copy record onto index page */ - void *bm = EMBEDDB_GET_BITMAP(state->buffer); - memcpy((void *)((int8_t *)buf + EMBEDDB_IDX_HEADER_SIZE + state->bitmapSize * idxcount), bm, state->bitmapSize); } - updateMaxiumError(state, state->buffer); + updateMaxiumError(state, dataWriteBuffer); count = 0; initBufferPage(state, 0); @@ -1746,28 +1781,9 @@ int8_t embedDBFlush(embedDBState *state) { state->fileInterface->flush(state->dataFile); indexPage(state, pageNum); - + int8_t indexResult = 0; if (EMBEDDB_USING_INDEX(state->parameters)) { - void *buf = (int8_t *)state->buffer + state->pageSize * (EMBEDDB_INDEX_WRITE_BUFFER); - count_t idxcount = EMBEDDB_GET_COUNT(buf); - EMBEDDB_INC_COUNT(buf); - - /* Copy record onto index page */ - void *bm = EMBEDDB_GET_BITMAP(state->buffer); - memcpy((void *)((int8_t *)buf + EMBEDDB_IDX_HEADER_SIZE + state->bitmapSize * idxcount), bm, state->bitmapSize); - - id_t writeResult = writeIndexPage(state, buf); - if (writeResult == -1) { -#ifdef PRINT_ERRORS - printf("Failed to write index page during embedDBFlush."); -#endif - return -1; - } - - state->fileInterface->flush(state->indexFile); - - /* Reinitialize buffer */ - initBufferPage(state, EMBEDDB_INDEX_WRITE_BUFFER); + indexResult = indexData(state, pageNum); } /* Reinitialize buffer */ @@ -1783,6 +1799,14 @@ int8_t embedDBFlush(embedDBState *state) { return -1; } } + + if (indexResult == -1) { +#ifdef PRINT_ERRORS + printf("ERROR: Failed to index data when flushing data to storage."); +#endif + return -1; + } + return 0; } From 8768f79bf231bc6376d544659f71976aba6b7cd8 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Tue, 30 Jul 2024 12:14:38 -0700 Subject: [PATCH 04/12] Added flag to use bitmap to all tests which were using the index but did not have the bitmap flag --- .../test_embedDB_index_recovery.cpp | 20 +++++++++---------- .../test_embedDB_multiple_instances.cpp | 4 ++-- .../test_embedDB_storage.cpp | 2 +- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index 74ba439a..1dce0684 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -68,7 +68,7 @@ #include "unity.h" embedDBState *state; -const int16_t RECOVERY_PARAMETERS = EMBEDDB_USE_INDEX; +const int16_t RECOVERY_PARAMETERS = EMBEDDB_USE_INDEX | EMBEDDB_USE_BMAP; void setupEmbedDB(int16_t parameters) { state = (embedDBState *)malloc(sizeof(embedDBState)); @@ -100,7 +100,7 @@ void setupEmbedDB(int16_t parameters) { } void setUp() { - int16_t parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA; + int16_t parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA | EMBEDDB_USE_BMAP; setupEmbedDB(parameters); } @@ -113,17 +113,15 @@ void tearDown() { free(state); } -void insertRecordsLinearly(int32_t startingKey, int32_t startingData, int32_t numRecords) { - int8_t *data = (int8_t *)malloc(state->recordSize); - *((int32_t *)data) = startingKey; - *((int32_t *)(data + 4)) = startingData; - for (int i = 0; i < numRecords; i++) { - *((int32_t *)data) += 1; - *((int64_t *)(data + 4)) += 1; - int8_t result = embedDBPut(state, data, (void *)(data + 4)); +void insertRecordsLinearly(int32_t startingKey, uint32_t numRecords) { + /* TODO: Decide on how to vary the data to get better bitmap results*/ + int32_t key = startingKey; + int32_t data = 0; + for (uint32_t i = 0; i < numRecords; i++) { + int8_t result = embedDBPut(state, &startingKey, &data); TEST_ASSERT_EQUAL_INT8_MESSAGE(0, result, "EmbedDB Put did not correctly insert data (returned non-zero code)"); + key++; } - free(data); } void embedDB_index_file_correctly_reloads_with_no_data() { diff --git a/test/test_embedDB_multiple_instances/test_embedDB_multiple_instances.cpp b/test/test_embedDB_multiple_instances/test_embedDB_multiple_instances.cpp index 4396be10..99fc4e8c 100644 --- a/test/test_embedDB_multiple_instances/test_embedDB_multiple_instances.cpp +++ b/test/test_embedDB_multiple_instances/test_embedDB_multiple_instances.cpp @@ -278,7 +278,7 @@ void setupembedDBInstanceKeySize4DataSize12(embedDBState *state, uint32_t number TEST_ASSERT_NOT_NULL_MESSAGE(state->buffer, "Failed to allocate EmbedDB buffer."); state->numDataPages = 20000; state->numIndexPages = 1000; - state->parameters = EMBEDDB_RESET_DATA | EMBEDDB_USE_INDEX; + state->parameters = EMBEDDB_RESET_DATA | EMBEDDB_USE_INDEX | EMBEDDB_USE_BMAP; state->eraseSizeInPages = 4; state->fileInterface = getFileInterface(); char path[40]; @@ -307,7 +307,7 @@ void setupembedDBInstanceKeySize4DataSize12WithVarData(embedDBState *state, uint state->numDataPages = 22000; state->numIndexPages = 1000; state->numVarPages = 44000; - state->parameters = EMBEDDB_RESET_DATA | EMBEDDB_USE_INDEX | EMBEDDB_USE_VDATA; + state->parameters = EMBEDDB_RESET_DATA | EMBEDDB_USE_INDEX | EMBEDDB_USE_VDATA | EMBEDDB_USE_BMAP; state->eraseSizeInPages = 4; state->fileInterface = getFileInterface(); char path[40]; diff --git a/test/test_embedDB_storage/test_embedDB_storage.cpp b/test/test_embedDB_storage/test_embedDB_storage.cpp index aeab89ea..4f847090 100644 --- a/test/test_embedDB_storage/test_embedDB_storage.cpp +++ b/test/test_embedDB_storage/test_embedDB_storage.cpp @@ -103,7 +103,7 @@ void embedDBInit_should_return_erorr_if_numIndexPages_is_not_divisible_by_eraseS state->numDataPages = 300; state->eraseSizeInPages = 3; state->bitmapSize = 1; - state->parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA; + state->parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA | EMBEDDB_USE_BMAP; state->inBitmap = inBitmapInt8; state->updateBitmap = updateBitmapInt8; state->buildBitmapFromRange = buildBitmapInt8FromRange; From 29b12706e545e756394ee3dd3aeacec88f7178d9 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Tue, 30 Jul 2024 15:57:49 -0700 Subject: [PATCH 05/12] Started work on recovering bitmips which were written to data file but not to index file --- src/embedDB/embedDB.c | 15 ++++++++ .../test_embedDB_index_recovery.cpp | 36 +++++++++++++++---- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index cda3dd0a..ade4a56f 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -185,6 +185,7 @@ int8_t embedDBInit(embedDBState *state, size_t indexMaxError) { /* Header size depends on bitmap size: 6 + X bytes: 4 byte id, 2 for record count, X for bitmap. */ state->headerSize = 6; if (EMBEDDB_USING_INDEX(state->parameters)) { + /* TODO: Add check that bitmap is also enabled */ if (state->numIndexPages % state->eraseSizeInPages != 0) { #ifdef PRINT_ERRORS printf("ERROR: The number of allocated index pages must be divisible by the erase size in pages.\n"); @@ -199,6 +200,13 @@ int8_t embedDBInit(embedDBState *state, size_t indexMaxError) { return -1; } + if (!EMBEDDB_USING_BMAP(state->parameters)) { +#ifdef PRINT_ERRORS + printf("WARNING: Using the data index requires the bitmap to be enabled. This will automatically be enabled.\n"); +#endif + state->parameters |= EMBEDDB_USE_BMAP; + } + state->headerSize += state->bitmapSize; } @@ -760,8 +768,15 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { state->numAvailIndexPages = state->numIndexPages + state->minIndexPageId - maxLogicaIndexPageId - 1; /* Compute how many bitmaps we should have and how many we do have */ + id_t minDataPageWithIndex = 0; + memcpy(&minDataPageWithIndex, (int8_t *)buffer + 8, sizeof(id_t)); id_t numberOfDataPages = state->nextDataPageId - state->minDataPageId; + /* Check if the minDataPage on record is less than the minIndexPage on record */ + if (state->minDataPageId < minDataPageWithIndex) { + /* code */ + } + return 0; } diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index 1dce0684..7d8d711e 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -71,6 +71,7 @@ embedDBState *state; const int16_t RECOVERY_PARAMETERS = EMBEDDB_USE_INDEX | EMBEDDB_USE_BMAP; void setupEmbedDB(int16_t parameters) { + /* This setup results in having 63 records per page */ state = (embedDBState *)malloc(sizeof(embedDBState)); state->keySize = 4; state->dataSize = 4; @@ -114,13 +115,18 @@ void tearDown() { } void insertRecordsLinearly(int32_t startingKey, uint32_t numRecords) { - /* TODO: Decide on how to vary the data to get better bitmap results*/ int32_t key = startingKey; int32_t data = 0; for (uint32_t i = 0; i < numRecords; i++) { - int8_t result = embedDBPut(state, &startingKey, &data); + int8_t result = embedDBPut(state, &key, &data); TEST_ASSERT_EQUAL_INT8_MESSAGE(0, result, "EmbedDB Put did not correctly insert data (returned non-zero code)"); key++; + if (i % 2 == 0) { + data++; + } + if (data % 110 == 0) { + data = 0; + } } } @@ -133,8 +139,23 @@ void embedDB_index_file_correctly_reloads_with_no_data() { TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when no data was present in the index file."); } +void embedDBFlush_should_not_flush_index_pages() { + /* Check that there is the correct number of indicies in buffer before flushing */ + insertRecordsLinearly(100, 24948); + void *buffer = (int8_t *)state->buffer + (state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER); + count_t recordCount = EMBEDDB_GET_COUNT(buffer); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(395, recordCount, "Count of data indicies was incorrect before flushing to storage."); + + /* Flush to storage */ + embedDBFlush(state); + + /* Check that we only added one for the new page */ + recordCount = EMBEDDB_GET_COUNT(buffer); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(396, recordCount, "Count of data indicies was incorrect before flushing to storage."); +} + void embedDB_index_file_correctly_reloads_with_one_page_of_data() { - insertRecordsLinearly(100, 100, 31312); + insertRecordsLinearly(100, 31312); tearDown(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when one index page was present in the index file."); @@ -143,7 +164,7 @@ void embedDB_index_file_correctly_reloads_with_one_page_of_data() { } void embedDB_index_file_correctly_reloads_with_four_pages_of_data() { - insertRecordsLinearly(100, 100, 125056); + insertRecordsLinearly(100, 125056); tearDown(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(4, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -152,7 +173,7 @@ void embedDB_index_file_correctly_reloads_with_four_pages_of_data() { } void embedDB_index_file_correctly_reloads_with_eight_pages_of_data() { - insertRecordsLinearly(100, 100, 250111); + insertRecordsLinearly(100, 250111); tearDown(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(8, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -161,7 +182,7 @@ void embedDB_index_file_correctly_reloads_with_eight_pages_of_data() { } void embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data() { - insertRecordsLinearly(100, 100, 500222); + insertRecordsLinearly(100, 500222); tearDown(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -170,7 +191,7 @@ void embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data() { } void embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data() { - insertRecordsLinearly(100, 100, 532288); + insertRecordsLinearly(100, 532288); tearDown(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(17, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -181,6 +202,7 @@ void embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data() { int runUnityTests() { UNITY_BEGIN(); RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); + RUN_TEST(embedDBFlush_should_not_flush_index_pages); RUN_TEST(embedDB_index_file_correctly_reloads_with_one_page_of_data); RUN_TEST(embedDB_index_file_correctly_reloads_with_four_pages_of_data); RUN_TEST(embedDB_index_file_correctly_reloads_with_eight_pages_of_data); From 399c68a1fabbe0aaa8e2f462b1334942adadb774 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Wed, 31 Jul 2024 11:53:57 -0700 Subject: [PATCH 06/12] Implemented recovery for bitmaps written to datafile but not to index file --- src/embedDB/embedDB.c | 125 ++++++++++-------- .../test_embedDB_index_recovery.cpp | 50 +++++++ 2 files changed, 123 insertions(+), 52 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index ade4a56f..2fd19ca9 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -623,8 +623,8 @@ void embedDBInitSplineFromFile(embedDBState *state) { int8_t embedDBInitIndex(embedDBState *state) { /* Setup index file. */ - /* 4 for id, 2 for count, 2 unused, 4 for minKey (pageId), 4 for maxKey (pageId) */ - state->maxIdxRecordsPerPage = (state->pageSize - 16) / state->bitmapSize; + /* 4 for id, 2 for count, 2 unused, 4 for minKey (pageId), 4 that are currently empty */ + state->maxIdxRecordsPerPage = (state->pageSize - EMBEDDB_IDX_HEADER_SIZE) / state->bitmapSize; /* Allocate third page of buffer as index output page */ initBufferPage(state, EMBEDDB_INDEX_WRITE_BUFFER); @@ -713,68 +713,89 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { i++; } - /* if we have no valid data, we just have an empty file can can start from the scratch */ - if (!hasData) - return 0; - - while (moreToRead && count < state->numIndexPages) { - memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); - validData = logicalIndexPageId % state->numIndexPages == count; - if (validData && logicalIndexPageId == maxLogicaIndexPageId + 1) { - maxLogicaIndexPageId = logicalIndexPageId; - physicalIndexPageId++; - moreToRead = !(readIndexPage(state, physicalIndexPageId)); - indexCount = EMBEDDB_GET_COUNT(buffer); - numberOfBitmaps += indexCount; - count++; - } else { - break; + /** + * If we have index data, we need to scan through the file for recovery. + * If there is no data in the file, there still may be bitmaps written to storage on the datafile that we can recover. + **/ + if (hasData) { + while (moreToRead && count < state->numIndexPages) { + memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); + validData = logicalIndexPageId % state->numIndexPages == count; + if (validData && logicalIndexPageId == maxLogicaIndexPageId + 1) { + maxLogicaIndexPageId = logicalIndexPageId; + physicalIndexPageId++; + moreToRead = !(readIndexPage(state, physicalIndexPageId)); + indexCount = EMBEDDB_GET_COUNT(buffer); + numberOfBitmaps += indexCount; + count++; + } else { + break; + } } - } - /* - * Now we need to find where the page with the smallest key that is still valid. - * The default case is we have not wrapped and the page number for the physical page with the smallest key is 0. - */ - id_t physicalPageIDOfSmallestData = 0; + /* + * Now we need to find where the page with the smallest key that is still valid. + * The default case is we have not wrapped and the page number for the physical page with the smallest key is 0. + */ + id_t physicalPageIDOfSmallestData = 0; - /* check if data exists at this location */ - if (moreToRead && count < state->numIndexPages) { - /* find where the next block boundary is */ - id_t pagesToBlockBoundary = blockSize - (count % blockSize); + /* check if data exists at this location */ + if (moreToRead && count < state->numIndexPages) { + /* find where the next block boundary is */ + id_t pagesToBlockBoundary = blockSize - (count % blockSize); - /* go to the next block boundary */ - physicalIndexPageId = (physicalIndexPageId + pagesToBlockBoundary) % state->numIndexPages; - moreToRead = !(readIndexPage(state, physicalIndexPageId)); + /* go to the next block boundary */ + physicalIndexPageId = (physicalIndexPageId + pagesToBlockBoundary) % state->numIndexPages; + moreToRead = !(readIndexPage(state, physicalIndexPageId)); - /* there should have been more to read becuase the file should not be empty at this point if it was not empty at the previous block */ - if (!moreToRead) { - return -1; - } + /* there should have been more to read becuase the file should not be empty at this point if it was not empty at the previous block */ + if (!moreToRead) { + return -1; + } - /* check if data is valid or if it is junk */ - memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); - validData = logicalIndexPageId % state->numIndexPages == physicalIndexPageId; + /* check if data is valid or if it is junk */ + memcpy(&logicalIndexPageId, buffer, sizeof(id_t)); + validData = logicalIndexPageId % state->numIndexPages == physicalIndexPageId; - /* this means we have wrapped and our start is actually here */ - if (validData) { - physicalPageIDOfSmallestData = physicalIndexPageId; + /* this means we have wrapped and our start is actually here */ + if (validData) { + physicalPageIDOfSmallestData = physicalIndexPageId; + } } - } - state->nextIdxPageId = maxLogicaIndexPageId + 1; - readIndexPage(state, physicalPageIDOfSmallestData); - memcpy(&(state->minIndexPageId), buffer, sizeof(id_t)); - state->numAvailIndexPages = state->numIndexPages + state->minIndexPageId - maxLogicaIndexPageId - 1; + state->nextIdxPageId = maxLogicaIndexPageId + 1; + readIndexPage(state, physicalPageIDOfSmallestData); + memcpy(&(state->minIndexPageId), buffer, sizeof(id_t)); + state->numAvailIndexPages = state->numIndexPages + state->minIndexPageId - maxLogicaIndexPageId - 1; + } /* Compute how many bitmaps we should have and how many we do have */ - id_t minDataPageWithIndex = 0; - memcpy(&minDataPageWithIndex, (int8_t *)buffer + 8, sizeof(id_t)); - id_t numberOfDataPages = state->nextDataPageId - state->minDataPageId; + id_t expectedNumberOfIndices = state->nextDataPageId; + id_t numDataIndicies = state->nextIdxPageId * state->maxIdxRecordsPerPage; + id_t numPagesToRead = expectedNumberOfIndices - numDataIndicies; + + if (numPagesToRead > 0) { + id_t pageToRead = state->nextDataPageId - numPagesToRead; + int8_t readResult = 0; + void *dataPageBuffer = state->buffer + state->pageSize * EMBEDDB_DATA_READ_BUFFER; + void *indexWriteBuffer = state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; - /* Check if the minDataPage on record is less than the minIndexPage on record */ - if (state->minDataPageId < minDataPageWithIndex) { - /* code */ + /* Add page id to minimum value spot in page */ + memcpy((int8_t *)indexWriteBuffer + 8, &pageToRead, sizeof(id_t)); + + /* Put all bitmaps that were written to storgae on the data page but not yet on the index page into the buffer */ + for (id_t i = 0; i < numPagesToRead; i++) { + readResult = readPage(state, pageToRead % state->numDataPages); + if (readResult == -1) { +#ifdef PRINT_ERRORS + printf("Error: Unable to read page from data file!\n"); +#endif + return -1; + } + void *bitmap = EMBEDDB_GET_BITMAP(dataPageBuffer); + EMBEDDB_INC_COUNT(indexWriteBuffer); + memcpy((int8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE + state->bitmapSize * i, bitmap, state->bitmapSize); + } } return 0; diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index 7d8d711e..f0b7a887 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -137,6 +137,11 @@ void embedDB_index_file_correctly_reloads_with_no_data() { TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when no data was present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when no data was present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when no data was present in the index file."); + + /* Check that index buffer also has no records. */ + void *indexWriteBuffer = state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); + TEST_ASSERT_EQUAL_UINT16(0, numIndices); } void embedDBFlush_should_not_flush_index_pages() { @@ -199,6 +204,49 @@ void embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data() { TEST_ASSERT_EQUAL_UINT32_MESSAGE(2, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); } +void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written() { + insertRecordsLinearly(100, 11907); + embedDBFlush(state); + tearDown(); + setupEmbedDB(RECOVERY_PARAMETERS); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); + + /* Check that index buffer also . */ + void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); + TEST_ASSERT_EQUAL_UINT16(189, numIndices); + + /* Check that the bitmap is correct */ + uint8_t expectedBitmap = 128 | 64 | 32 | 16; + uint8_t actualBitmap = 0; + memcpy(&actualBitmap, (uint8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE, state->bitmapSize); + TEST_ASSERT_EQUAL_UINT8_MESSAGE(expectedBitmap, actualBitmap, "embedDBIndexRecovery did not correctly recover the bitmap for the first data index."); +} + +void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written() { + /* This number of inserts results in 7 full index pages being written and then three data pages whose indicies are only in the buffer before tearDown */ + insertRecordsLinearly(100, 218925); + embedDBFlush(state); + tearDown(); + setupEmbedDB(RECOVERY_PARAMETERS); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(7, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(9, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); + + /* Check that index buffer also . */ + void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); + TEST_ASSERT_EQUAL_UINT16(3, numIndices); + + /* Check that the bitmap is correct */ + uint8_t expectedBitmap = 128 | 64 | 32 | 16; + uint8_t actualBitmap = 0; + memcpy(&actualBitmap, (uint8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE, state->bitmapSize); + TEST_ASSERT_EQUAL_UINT8_MESSAGE(expectedBitmap, actualBitmap, "embedDBIndexRecovery did not correctly recover the bitmap for the first data index."); +} + int runUnityTests() { UNITY_BEGIN(); RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); @@ -208,6 +256,8 @@ int runUnityTests() { RUN_TEST(embedDB_index_file_correctly_reloads_with_eight_pages_of_data); RUN_TEST(embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data); RUN_TEST(embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data); + RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written); + RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written); return UNITY_END(); } From c03da8c7f0a83ef5a6cb4114dcb2a8485adc6830 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Wed, 31 Jul 2024 15:29:22 -0700 Subject: [PATCH 07/12] Added some test cases for index recovery --- .../test_embedDB_index_recovery.cpp | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index f0b7a887..e4d21df0 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -139,7 +139,7 @@ void embedDB_index_file_correctly_reloads_with_no_data() { TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when no data was present in the index file."); /* Check that index buffer also has no records. */ - void *indexWriteBuffer = state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); TEST_ASSERT_EQUAL_UINT16(0, numIndices); } @@ -235,18 +235,59 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_page TEST_ASSERT_EQUAL_UINT32_MESSAGE(9, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); - /* Check that index buffer also . */ + /* Check that index buffer also has correct data. */ void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); TEST_ASSERT_EQUAL_UINT16(3, numIndices); /* Check that the bitmap is correct */ - uint8_t expectedBitmap = 128 | 64 | 32 | 16; + uint8_t expectedBitmap = 32 | 16 | 8 | 4; uint8_t actualBitmap = 0; memcpy(&actualBitmap, (uint8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE, state->bitmapSize); TEST_ASSERT_EQUAL_UINT8_MESSAGE(expectedBitmap, actualBitmap, "embedDBIndexRecovery did not correctly recover the bitmap for the first data index."); } +void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written() { + /* Write out 16 index pages and have 289 on the datafile but not buffered */ + insertRecordsLinearly(4000, 518175); + embedDBFlush(state); + tearDown(); + setupEmbedDB(RECOVERY_PARAMETERS); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); + + void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); + TEST_ASSERT_EQUAL_UINT16(289, numIndices); + + /* Check that the bitmap is correct */ + uint8_t expectedBitmap = 2; + uint8_t actualBitmap = 0; + memcpy(&actualBitmap, (uint8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE, state->bitmapSize); + TEST_ASSERT_EQUAL_UINT8_MESSAGE(expectedBitmap, actualBitmap, "embedDBIndexRecovery did not correctly recover the bitmap for the first data index."); +} + +void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_21_pages_of_data_written() { + /* 21 pages of index written and then five extra indicies */ + insertRecordsLinearly(4000, 656523); + embedDBFlush(state); + tearDown(); + setupEmbedDB(RECOVERY_PARAMETERS); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(21, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(6, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); + + void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); + TEST_ASSERT_EQUAL_UINT16(5, numIndices); + + uint8_t expectedBitmap = 1 | 128 | 2; + uint8_t actualBitmap = 0; + memcpy(&actualBitmap, (uint8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE + state->bitmapSize * (numIndices - 1), state->bitmapSize); + TEST_ASSERT_EQUAL_UINT8_MESSAGE(expectedBitmap, actualBitmap, "embedDBIndexRecovery did not correctly recover the bitmap for the first data index."); +} + int runUnityTests() { UNITY_BEGIN(); RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); @@ -258,6 +299,8 @@ int runUnityTests() { RUN_TEST(embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data); RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written); RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written); + RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written); + RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_21_pages_of_data_written); return UNITY_END(); } From b9ac199eab4643cb8f2e6416888ac0583c5e5f8a Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Tue, 6 Aug 2024 11:05:51 -0700 Subject: [PATCH 08/12] Removed old todos --- src/embedDB/embedDB.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index 2fd19ca9..8bc8e051 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -185,7 +185,6 @@ int8_t embedDBInit(embedDBState *state, size_t indexMaxError) { /* Header size depends on bitmap size: 6 + X bytes: 4 byte id, 2 for record count, X for bitmap. */ state->headerSize = 6; if (EMBEDDB_USING_INDEX(state->parameters)) { - /* TODO: Add check that bitmap is also enabled */ if (state->numIndexPages % state->eraseSizeInPages != 0) { #ifdef PRINT_ERRORS printf("ERROR: The number of allocated index pages must be divisible by the erase size in pages.\n"); @@ -2172,7 +2171,6 @@ int8_t writeTemporaryPage(embedDBState *state, void *buffer) { } /* Setup page number in header */ - /* TODO: Maybe talk to Ramon about optimizing this */ memcpy(buffer, &(state->nextDataPageId), sizeof(id_t)); /* Wrap if needed */ From 3247d8185cf66c75875816cbc8ac77b353cbf977 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Tue, 6 Aug 2024 11:34:48 -0700 Subject: [PATCH 09/12] Removed unneeded variable --- src/embedDB/embedDB.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index 8bc8e051..6069d62f 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -681,7 +681,6 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { id_t maxLogicaIndexPageId = 0; id_t physicalIndexPageId = 0; id_t count = 0; - id_t numberOfBitmaps = 0; count_t blockSize = state->eraseSizeInPages; bool validData = false; bool hasData = false; @@ -703,7 +702,6 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { physicalIndexPageId++; count++; i = 2; - numberOfBitmaps += indexCount; } else { physicalIndexPageId += blockSize; count += blockSize; @@ -724,8 +722,6 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { maxLogicaIndexPageId = logicalIndexPageId; physicalIndexPageId++; moreToRead = !(readIndexPage(state, physicalIndexPageId)); - indexCount = EMBEDDB_GET_COUNT(buffer); - numberOfBitmaps += indexCount; count++; } else { break; From 69da4ab05eed1f70bd85b2f391f443c02c48f459 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Tue, 6 Aug 2024 12:19:03 -0700 Subject: [PATCH 10/12] Added additional tests for buffer which discovered issuewith reading wrong pages but the tests are currently seg faulting --- src/embedDB/embedDB.c | 1 + .../test_embedDB_index_recovery.cpp | 57 ++++++++++++++----- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/embedDB/embedDB.c b/src/embedDB/embedDB.c index 6069d62f..85864bdc 100644 --- a/src/embedDB/embedDB.c +++ b/src/embedDB/embedDB.c @@ -790,6 +790,7 @@ int8_t embedDBInitIndexFromFile(embedDBState *state) { void *bitmap = EMBEDDB_GET_BITMAP(dataPageBuffer); EMBEDDB_INC_COUNT(indexWriteBuffer); memcpy((int8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE + state->bitmapSize * i, bitmap, state->bitmapSize); + pageToRead++; } } diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index e4d21df0..1223334f 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -68,6 +68,7 @@ #include "unity.h" embedDBState *state; +void *indexWriteBufferBeforeTearDown = NULL; const int16_t RECOVERY_PARAMETERS = EMBEDDB_USE_INDEX | EMBEDDB_USE_BMAP; void setupEmbedDB(int16_t parameters) { @@ -101,11 +102,14 @@ void setupEmbedDB(int16_t parameters) { } void setUp() { + /* Setup extra buffer for testing */ + indexWriteBufferBeforeTearDown = malloc(state->pageSize); + int16_t parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA | EMBEDDB_USE_BMAP; setupEmbedDB(parameters); } -void tearDown() { +void tearDownEmbedDB() { free(state->buffer); embedDBClose(state); tearDownFile(state->dataFile); @@ -114,6 +118,11 @@ void tearDown() { free(state); } +void tearDown() { + tearDownEmbedDB(); + free(indexWriteBufferBeforeTearDown); +} + void insertRecordsLinearly(int32_t startingKey, uint32_t numRecords) { int32_t key = startingKey; int32_t data = 0; @@ -131,7 +140,7 @@ void insertRecordsLinearly(int32_t startingKey, uint32_t numRecords) { } void embedDB_index_file_correctly_reloads_with_no_data() { - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(496, state->maxIdxRecordsPerPage, "EmbedDB maxIdxRecordsPerPage was initialized incorrectly when no data was present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when no data was present in the index file."); @@ -161,7 +170,7 @@ void embedDBFlush_should_not_flush_index_pages() { void embedDB_index_file_correctly_reloads_with_one_page_of_data() { insertRecordsLinearly(100, 31312); - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when one index page was present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(15, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when one index page was present in the index file."); @@ -170,7 +179,7 @@ void embedDB_index_file_correctly_reloads_with_one_page_of_data() { void embedDB_index_file_correctly_reloads_with_four_pages_of_data() { insertRecordsLinearly(100, 125056); - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(4, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(12, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -179,7 +188,7 @@ void embedDB_index_file_correctly_reloads_with_four_pages_of_data() { void embedDB_index_file_correctly_reloads_with_eight_pages_of_data() { insertRecordsLinearly(100, 250111); - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(8, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(8, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -188,7 +197,7 @@ void embedDB_index_file_correctly_reloads_with_eight_pages_of_data() { void embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data() { insertRecordsLinearly(100, 500222); - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -197,7 +206,7 @@ void embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data() { void embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data() { insertRecordsLinearly(100, 532288); - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(17, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -205,16 +214,25 @@ void embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data() { } void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written() { + /* Inerst records into embedDB */ insertRecordsLinearly(100, 11907); embedDBFlush(state); - tearDown(); + + /* Copy current buffer into another place to compare with after teardown */ + void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + memcpy(indexWriteBufferBeforeTearDown, indexWriteBuffer, state->pageSize); + + /* Tear down and recover embedDB */ + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); + + /* Check that the index parameters are what is expected when no pages have been written to storage yet */ TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); - /* Check that index buffer also . */ - void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + /* Check that index buffer is recovered correctly */ + indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); TEST_ASSERT_EQUAL_UINT16(189, numIndices); @@ -223,20 +241,28 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_ uint8_t actualBitmap = 0; memcpy(&actualBitmap, (uint8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE, state->bitmapSize); TEST_ASSERT_EQUAL_UINT8_MESSAGE(expectedBitmap, actualBitmap, "embedDBIndexRecovery did not correctly recover the bitmap for the first data index."); + + /* Compare with buffer before tearDown */ + TEST_ASSERT_EQUAL_MEMORY(indexWriteBufferBeforeTearDown, indexWriteBuffer, state->pageSize); } void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written() { /* This number of inserts results in 7 full index pages being written and then three data pages whose indicies are only in the buffer before tearDown */ insertRecordsLinearly(100, 218925); embedDBFlush(state); - tearDown(); + + /* Copy current buffer into another place to compare with after teardown */ + void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + memcpy(indexWriteBufferBeforeTearDown, indexWriteBuffer, state->pageSize); + + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(7, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(9, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->minIndexPageId, "EmbedDB minIndexPageId was initialized incorrectly when four index pages were present in the index file."); /* Check that index buffer also has correct data. */ - void *indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; + indexWriteBuffer = (int8_t *)state->buffer + state->pageSize * EMBEDDB_INDEX_WRITE_BUFFER; count_t numIndices = EMBEDDB_GET_COUNT(indexWriteBuffer); TEST_ASSERT_EQUAL_UINT16(3, numIndices); @@ -245,13 +271,16 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_page uint8_t actualBitmap = 0; memcpy(&actualBitmap, (uint8_t *)indexWriteBuffer + EMBEDDB_IDX_HEADER_SIZE, state->bitmapSize); TEST_ASSERT_EQUAL_UINT8_MESSAGE(expectedBitmap, actualBitmap, "embedDBIndexRecovery did not correctly recover the bitmap for the first data index."); + + /* Compare with buffer from before recovery */ + TEST_ASSERT_EQUAL_MEMORY(indexWriteBufferBeforeTearDown, indexWriteBuffer, state->pageSize); } void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written() { /* Write out 16 index pages and have 289 on the datafile but not buffered */ insertRecordsLinearly(4000, 518175); embedDBFlush(state); - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(16, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); @@ -272,7 +301,7 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_21_pages_of_dat /* 21 pages of index written and then five extra indicies */ insertRecordsLinearly(4000, 656523); embedDBFlush(state); - tearDown(); + tearDownEmbedDB(); setupEmbedDB(RECOVERY_PARAMETERS); TEST_ASSERT_EQUAL_UINT32_MESSAGE(21, state->nextIdxPageId, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, state->numAvailIndexPages, "EmbedDB nextIdxPageId was initialized incorrectly when four index pages were present in the index file."); From 7535534e17793daf0e93c4c4edf86be6e34dd477 Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Wed, 8 Jan 2025 20:29:12 -0700 Subject: [PATCH 11/12] Fixed issue with segmentation fault --- .../test_embedDB_index_recovery.cpp | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index 1223334f..c5b32ed5 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -68,7 +68,7 @@ #include "unity.h" embedDBState *state; -void *indexWriteBufferBeforeTearDown = NULL; +// void *indexWriteBufferBeforeTearDown = NULL; const int16_t RECOVERY_PARAMETERS = EMBEDDB_USE_INDEX | EMBEDDB_USE_BMAP; void setupEmbedDB(int16_t parameters) { @@ -103,7 +103,7 @@ void setupEmbedDB(int16_t parameters) { void setUp() { /* Setup extra buffer for testing */ - indexWriteBufferBeforeTearDown = malloc(state->pageSize); + // indexWriteBufferBeforeTearDown = malloc(state->pageSize); int16_t parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA | EMBEDDB_USE_BMAP; setupEmbedDB(parameters); @@ -120,7 +120,7 @@ void tearDownEmbedDB() { void tearDown() { tearDownEmbedDB(); - free(indexWriteBufferBeforeTearDown); + // free(indexWriteBufferBeforeTearDown); } void insertRecordsLinearly(int32_t startingKey, uint32_t numRecords) { @@ -214,6 +214,9 @@ void embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data() { } void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written() { + /* Setup Buffer */ + void *indexWriteBufferBeforeTearDown = malloc(state->pageSize); + /* Inerst records into embedDB */ insertRecordsLinearly(100, 11907); embedDBFlush(state); @@ -244,9 +247,14 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_ /* Compare with buffer before tearDown */ TEST_ASSERT_EQUAL_MEMORY(indexWriteBufferBeforeTearDown, indexWriteBuffer, state->pageSize); + + free(indexWriteBufferBeforeTearDown); } void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written() { + /* Setup Buffer */ + void *indexWriteBufferBeforeTearDown = malloc(state->pageSize); + /* This number of inserts results in 7 full index pages being written and then three data pages whose indicies are only in the buffer before tearDown */ insertRecordsLinearly(100, 218925); embedDBFlush(state); @@ -274,6 +282,9 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_page /* Compare with buffer from before recovery */ TEST_ASSERT_EQUAL_MEMORY(indexWriteBufferBeforeTearDown, indexWriteBuffer, state->pageSize); + + /* Teardownn */ + free(indexWriteBufferBeforeTearDown); } void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written() { @@ -319,16 +330,16 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_21_pages_of_dat int runUnityTests() { UNITY_BEGIN(); - RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); - RUN_TEST(embedDBFlush_should_not_flush_index_pages); - RUN_TEST(embedDB_index_file_correctly_reloads_with_one_page_of_data); - RUN_TEST(embedDB_index_file_correctly_reloads_with_four_pages_of_data); - RUN_TEST(embedDB_index_file_correctly_reloads_with_eight_pages_of_data); - RUN_TEST(embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data); - RUN_TEST(embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data); - RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written); - RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written); - RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written); + // RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); + // RUN_TEST(embedDBFlush_should_not_flush_index_pages); + // RUN_TEST(embedDB_index_file_correctly_reloads_with_one_page_of_data); + // RUN_TEST(embedDB_index_file_correctly_reloads_with_four_pages_of_data); + // RUN_TEST(embedDB_index_file_correctly_reloads_with_eight_pages_of_data); + // RUN_TEST(embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data); + // RUN_TEST(embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data); + // RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written); + // RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written); + // RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written); RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_21_pages_of_data_written); return UNITY_END(); } From 5254127e8159b32af3d451237357faf25687fe2e Mon Sep 17 00:00:00 2001 From: Seth Akins Date: Wed, 8 Jan 2025 20:29:47 -0700 Subject: [PATCH 12/12] Added back all tests and removed commented code --- .../test_embedDB_index_recovery.cpp | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp index c5b32ed5..87e4b31d 100644 --- a/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp +++ b/test/test_embedDB_index_recovery/test_embedDB_index_recovery.cpp @@ -68,7 +68,6 @@ #include "unity.h" embedDBState *state; -// void *indexWriteBufferBeforeTearDown = NULL; const int16_t RECOVERY_PARAMETERS = EMBEDDB_USE_INDEX | EMBEDDB_USE_BMAP; void setupEmbedDB(int16_t parameters) { @@ -102,9 +101,6 @@ void setupEmbedDB(int16_t parameters) { } void setUp() { - /* Setup extra buffer for testing */ - // indexWriteBufferBeforeTearDown = malloc(state->pageSize); - int16_t parameters = EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA | EMBEDDB_USE_BMAP; setupEmbedDB(parameters); } @@ -120,7 +116,6 @@ void tearDownEmbedDB() { void tearDown() { tearDownEmbedDB(); - // free(indexWriteBufferBeforeTearDown); } void insertRecordsLinearly(int32_t startingKey, uint32_t numRecords) { @@ -330,16 +325,16 @@ void embedDBIndexRecovery_should_recover_indicies_in_buffer_with_21_pages_of_dat int runUnityTests() { UNITY_BEGIN(); - // RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); - // RUN_TEST(embedDBFlush_should_not_flush_index_pages); - // RUN_TEST(embedDB_index_file_correctly_reloads_with_one_page_of_data); - // RUN_TEST(embedDB_index_file_correctly_reloads_with_four_pages_of_data); - // RUN_TEST(embedDB_index_file_correctly_reloads_with_eight_pages_of_data); - // RUN_TEST(embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data); - // RUN_TEST(embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data); - // RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written); - // RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written); - // RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written); + RUN_TEST(embedDB_index_file_correctly_reloads_with_no_data); + RUN_TEST(embedDBFlush_should_not_flush_index_pages); + RUN_TEST(embedDB_index_file_correctly_reloads_with_one_page_of_data); + RUN_TEST(embedDB_index_file_correctly_reloads_with_four_pages_of_data); + RUN_TEST(embedDB_index_file_correctly_reloads_with_eight_pages_of_data); + RUN_TEST(embedDB_index_file_correctly_reloads_with_sixteen_pages_of_data); + RUN_TEST(embedDB_index_file_correctly_reloads_with_seventeen_pages_of_data); + RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_no_index_pages_written); + RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_with_seven_pages_written); + RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_sixteen_pages_of_data_written); RUN_TEST(embedDBIndexRecovery_should_recover_indicies_in_buffer_with_21_pages_of_data_written); return UNITY_END(); }