From 830284cc09ea79cd7a4544ef81f52cf98ed774fc Mon Sep 17 00:00:00 2001 From: RockyLim92 Date: Thu, 6 Sep 2018 02:04:25 +0900 Subject: [PATCH 1/6] for crc mismatch error --- Makefile | 2 +- nvm/env_nvm_file.cc | 2 +- nvm/run2.sh | 113 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 nvm/run2.sh diff --git a/Makefile b/Makefile index 02eb50571128..8660b95b6a44 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ SHELL := $(shell which bash) CLEAN_FILES = # deliberately empty, so we can append below. CFLAGS += ${EXTRA_CFLAGS} CXXFLAGS += ${EXTRA_CXXFLAGS} -LDFLAGS += $(EXTRA_LDFLAGS) +LDFLAGS += -fopenmp -llightnvm $(EXTRA_LDFLAGS) MACHINE ?= $(shell uname -m) ARFLAGS = rs diff --git a/nvm/env_nvm_file.cc b/nvm/env_nvm_file.cc index af76aa004e32..da7de0f76236 100644 --- a/nvm/env_nvm_file.cc +++ b/nvm/env_nvm_file.cc @@ -519,7 +519,7 @@ Status NvmFile::Read( // Now we know that: '0 < n <= nbytes_from_offset' uint64_t aligned_offset = offset - offset % align_nbytes_; - uint64_t aligned_n = (((n + align_nbytes_ - 1) / align_nbytes_)) * align_nbytes_; + uint64_t aligned_n = ((((n + align_nbytes_ - 1) / align_nbytes_)) + 1) * align_nbytes_; uint64_t skip_head_nbytes = offset - aligned_offset; uint64_t skip_tail_nbytes = aligned_n - n; diff --git a/nvm/run2.sh b/nvm/run2.sh new file mode 100644 index 000000000000..8dbc31519a89 --- /dev/null +++ b/nvm/run2.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +BENCH_DEV_NAME=nvme0n1 +RBENCH_BENCHMARKS=filluniquerandom +RBENCH_DEV_MODE=nvm + +if [ $UID != 0 ]; then + echo "You don't have sufficient privileges to run this script." + exit 1 +fi + +if [ -z "$RBENCH_DEV_NAME" ]; then + echo "Please set RBENCH_DEV_NAME e.g. RBENCH_DEV_NAME='nvme0n1'" + exit 1 +fi + +if [ -z "$RBENCH_BENCHMARKS" ]; then + RBENCH_BENCHMARKS="fillseq,readwhilewriting,readseq,readrandom" +fi + +if [ -z "$RBENCH_USE_EXISTING_DB" ]; then + RBENCH_USE_EXISTING_DB="0" +fi + +DB_ROOT="/opt/rocks" + +RBENCH_DB="$DB_ROOT/${RBENCH_DEV_NAME}_${RBENCH_DEV_MODE}" +mkdir -p $RBENCH_DB + +case "$RBENCH_DEV_MODE" in +legacy) + RBENCH_USE_DIRECT_READS=true + RBENCH_USE_DIRECT_IO_FOR_FLUSH_AND_COMPACTION=true + + if [ "$RBENCH_USE_EXISTING_DB" -eq "0" ]; then + umount $RBENCH_DB + mkfs.ext4 -F "/dev/$RBENCH_DEV_NAME" + mount "/dev/$RBENCH_DEV_NAME" $RBENCH_DB + fi +;; +pblk) + RBENCH_USE_DIRECT_READS=true + RBENCH_USE_DIRECT_IO_FOR_FLUSH_AND_COMPACTION=true + + if [ "$RBENCH_USE_EXISTING_DB" -eq "0" ]; then + PBLK_BGN=0 + PBLK_END=127 + PBLK_NAME="pblk_${PBLK_BGN}_${PBLK_END}" + umount $RBENCH_DB + nvme lnvm remove $PBLK_NAME + nvme lnvm create -d $RBENCH_DEV_NAME -n $PBLK_NAME -t pblk -b $PBLK_BGN -e $PBLK_END -f + mkfs.ext4 -F "/dev/$PBLK_NAME" + mount "/dev/$PBLK_NAME" $RBENCH_DB + fi +;; +nvm) + NVM_BGN=0 + NVM_END=127 + + RBENCH_ENV_URI="nvm://punits:${NVM_BGN}-${NVM_END}@${RBENCH_DEV_NAME}${DB_ROOT}/${RBENCH_DEV_NAME}_${NVM_BGN}_${NVM_END}.meta" + + if [ "$RBENCH_USE_EXISTING_DB" == "0" ]; then + rm $RBENCH_DB/* + fi +;; +*) + echo "Please set $RBENCH_DEV_NAME to one of: legacy|pblk|nvm" +esac + +KB=1024 +MB=$((1024 * KB)) +GB=$((1024 * MB)) + +#RBENCH_CMD_PREFIX="taskset -c 0-$(nproc)" +#RBENCH_CMD_PREFIX="valgrind" +RBENCH_BIN="../db_bench" +RBENCH_NUM=10000 +RBENCH_VALUE_SIZE=$((1 * MB)) +RBENCH_BLOCK_SIZE=$((64 * KB)) # default: 4k +RBENCH_BLOOM_BITS=10 +RBENCH_CACHE_SIZE=$((1 * MB)) +#RBENCH_RANDOM_ACCESS_MAX_BUFFER_SIZE=$((128 * MB)) # default: 1MB +#RBENCH_WRITABLE_FILE_MAX_BUFFER_SIZE=$((128 * MB)) # default: 1MB +#RBENCH_WRITE_BUFFER_SIZE=$((2000 * MB)) # default: 64MB +RBENCH_MAX_WRITE_BUFFER_NUMBER=2 +RBENCH_MIN_WRITE_BUFFER_NUMBER_TO_MERGE=1 +RBENCH_TARGET_FILE_SIZE_MULTIPLIER=1 +#RBENCH_TARGET_FILE_SIZE_BASE=$((2000 * MB)) +#RBENCH_MAX_BYTES_FOR_LEVEL_BASE=$((4000 * MB)) +RBENCH_CACHE_NUMSHARDBITS=4 +RBENCH_COMPRESSION_RATIO=1 # default: 0.5 +RBENCH_COMPRESSION_TYPE=none +RBENCH_DELETE_OBSOLETE_FILES_PERIOD_MICROS=300000000 +RBENCH_DISABLE_SEEK_COMPACTION=1 +RBENCH_DISABLE_WAL=1 +#RBENCH_LEVEL0_FILE_NUM_COMPACTION_TRIGGER=1 # default: 4 +#RBENCH_LEVEL0_SLOWDOWN_WRITES_TRIGGER=8 # default: 20 +#RBENCH_LEVEL0_STOP_WRITES_TRIGGER=12 # default: 36 +RBENCH_MAX_BACKGROUND_COMPACTIONS=20 +#RBENCH_MIN_LEVEL_TO_COMPRESS=2 # default: -1(compress all level) +RBENCH_MMAP_READ=1 # default: false +RBENCH_NUM_LEVELS=6 # default: 7 +RBENCH_OPEN_FILES=500000 +RBENCH_STATISTICS=1 +#RBENCH_STATS_INTERVAL=100000 # defalut: 0 +#RBENCH_STATS_PER_INTERVAL=1 # default: 0 +RBENCH_SYNC=0 +RBENCH_THREADS=1 +#RBENCH_THREADS=$(nproc) +RBENCH_VERIFY_CHECKSUM=1 + +source rbench.sh +rbench # Run them From f847e224c808ec2f6490c8aebb48c93bebbb1b9a Mon Sep 17 00:00:00 2001 From: RockyLim92 Date: Sun, 6 Jan 2019 07:25:30 -0500 Subject: [PATCH 2/6] sync --- nvm/run2.sh | 113 ---------------------------------------------------- 1 file changed, 113 deletions(-) delete mode 100644 nvm/run2.sh diff --git a/nvm/run2.sh b/nvm/run2.sh deleted file mode 100644 index 8dbc31519a89..000000000000 --- a/nvm/run2.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -BENCH_DEV_NAME=nvme0n1 -RBENCH_BENCHMARKS=filluniquerandom -RBENCH_DEV_MODE=nvm - -if [ $UID != 0 ]; then - echo "You don't have sufficient privileges to run this script." - exit 1 -fi - -if [ -z "$RBENCH_DEV_NAME" ]; then - echo "Please set RBENCH_DEV_NAME e.g. RBENCH_DEV_NAME='nvme0n1'" - exit 1 -fi - -if [ -z "$RBENCH_BENCHMARKS" ]; then - RBENCH_BENCHMARKS="fillseq,readwhilewriting,readseq,readrandom" -fi - -if [ -z "$RBENCH_USE_EXISTING_DB" ]; then - RBENCH_USE_EXISTING_DB="0" -fi - -DB_ROOT="/opt/rocks" - -RBENCH_DB="$DB_ROOT/${RBENCH_DEV_NAME}_${RBENCH_DEV_MODE}" -mkdir -p $RBENCH_DB - -case "$RBENCH_DEV_MODE" in -legacy) - RBENCH_USE_DIRECT_READS=true - RBENCH_USE_DIRECT_IO_FOR_FLUSH_AND_COMPACTION=true - - if [ "$RBENCH_USE_EXISTING_DB" -eq "0" ]; then - umount $RBENCH_DB - mkfs.ext4 -F "/dev/$RBENCH_DEV_NAME" - mount "/dev/$RBENCH_DEV_NAME" $RBENCH_DB - fi -;; -pblk) - RBENCH_USE_DIRECT_READS=true - RBENCH_USE_DIRECT_IO_FOR_FLUSH_AND_COMPACTION=true - - if [ "$RBENCH_USE_EXISTING_DB" -eq "0" ]; then - PBLK_BGN=0 - PBLK_END=127 - PBLK_NAME="pblk_${PBLK_BGN}_${PBLK_END}" - umount $RBENCH_DB - nvme lnvm remove $PBLK_NAME - nvme lnvm create -d $RBENCH_DEV_NAME -n $PBLK_NAME -t pblk -b $PBLK_BGN -e $PBLK_END -f - mkfs.ext4 -F "/dev/$PBLK_NAME" - mount "/dev/$PBLK_NAME" $RBENCH_DB - fi -;; -nvm) - NVM_BGN=0 - NVM_END=127 - - RBENCH_ENV_URI="nvm://punits:${NVM_BGN}-${NVM_END}@${RBENCH_DEV_NAME}${DB_ROOT}/${RBENCH_DEV_NAME}_${NVM_BGN}_${NVM_END}.meta" - - if [ "$RBENCH_USE_EXISTING_DB" == "0" ]; then - rm $RBENCH_DB/* - fi -;; -*) - echo "Please set $RBENCH_DEV_NAME to one of: legacy|pblk|nvm" -esac - -KB=1024 -MB=$((1024 * KB)) -GB=$((1024 * MB)) - -#RBENCH_CMD_PREFIX="taskset -c 0-$(nproc)" -#RBENCH_CMD_PREFIX="valgrind" -RBENCH_BIN="../db_bench" -RBENCH_NUM=10000 -RBENCH_VALUE_SIZE=$((1 * MB)) -RBENCH_BLOCK_SIZE=$((64 * KB)) # default: 4k -RBENCH_BLOOM_BITS=10 -RBENCH_CACHE_SIZE=$((1 * MB)) -#RBENCH_RANDOM_ACCESS_MAX_BUFFER_SIZE=$((128 * MB)) # default: 1MB -#RBENCH_WRITABLE_FILE_MAX_BUFFER_SIZE=$((128 * MB)) # default: 1MB -#RBENCH_WRITE_BUFFER_SIZE=$((2000 * MB)) # default: 64MB -RBENCH_MAX_WRITE_BUFFER_NUMBER=2 -RBENCH_MIN_WRITE_BUFFER_NUMBER_TO_MERGE=1 -RBENCH_TARGET_FILE_SIZE_MULTIPLIER=1 -#RBENCH_TARGET_FILE_SIZE_BASE=$((2000 * MB)) -#RBENCH_MAX_BYTES_FOR_LEVEL_BASE=$((4000 * MB)) -RBENCH_CACHE_NUMSHARDBITS=4 -RBENCH_COMPRESSION_RATIO=1 # default: 0.5 -RBENCH_COMPRESSION_TYPE=none -RBENCH_DELETE_OBSOLETE_FILES_PERIOD_MICROS=300000000 -RBENCH_DISABLE_SEEK_COMPACTION=1 -RBENCH_DISABLE_WAL=1 -#RBENCH_LEVEL0_FILE_NUM_COMPACTION_TRIGGER=1 # default: 4 -#RBENCH_LEVEL0_SLOWDOWN_WRITES_TRIGGER=8 # default: 20 -#RBENCH_LEVEL0_STOP_WRITES_TRIGGER=12 # default: 36 -RBENCH_MAX_BACKGROUND_COMPACTIONS=20 -#RBENCH_MIN_LEVEL_TO_COMPRESS=2 # default: -1(compress all level) -RBENCH_MMAP_READ=1 # default: false -RBENCH_NUM_LEVELS=6 # default: 7 -RBENCH_OPEN_FILES=500000 -RBENCH_STATISTICS=1 -#RBENCH_STATS_INTERVAL=100000 # defalut: 0 -#RBENCH_STATS_PER_INTERVAL=1 # default: 0 -RBENCH_SYNC=0 -RBENCH_THREADS=1 -#RBENCH_THREADS=$(nproc) -RBENCH_VERIFY_CHECKSUM=1 - -source rbench.sh -rbench # Run them From 9b672320c6251de4827909a99126778273d0873f Mon Sep 17 00:00:00 2001 From: RockyLim92 Date: Sun, 6 Jan 2019 07:35:30 -0500 Subject: [PATCH 3/6] sync --- nvm/Makefile | 4 ++-- nvm/env_nvm_file.cc | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nvm/Makefile b/nvm/Makefile index 7bb74ea0a4d6..70588c81a20d 100644 --- a/nvm/Makefile +++ b/nvm/Makefile @@ -36,8 +36,8 @@ tags: dev: clean configure make rocks: - #cd .. && make release -j$(expr `nproc` - 2) - cd .. && make release -j8 + cd .. && make release -j$(expr `nproc` - 2) + #cd .. && make release -j8 .PHONY: hash hash: diff --git a/nvm/env_nvm_file.cc b/nvm/env_nvm_file.cc index da7de0f76236..bd27892bb9da 100644 --- a/nvm/env_nvm_file.cc +++ b/nvm/env_nvm_file.cc @@ -38,6 +38,7 @@ NvmFile::NvmFile( stripe_nbytes_(), blk_nbytes_(), blks_(), lu_bound_(8) { NVM_DBG(this, "mpath_:" << mpath_); + struct nvm_dev *dev = env_->store_->GetDev(); const struct nvm_geo *geo = nvm_dev_get_geo(env_->store_->GetDev()); if (env_->posix_->FileExists(mpath_).ok()) { // Read meta from file @@ -76,7 +77,7 @@ NvmFile::NvmFile( buf_nbytes_ = 0; // Setup buffer buf_nbytes_max_ = lu_bound_ * stripe_nbytes_; - buf_ = (char*)nvm_buf_alloc(geo, buf_nbytes_max_); + buf_ = (char*)nvm_buf_alloc(dev, buf_nbytes_max_, NULL); if (!buf_) { NVM_DBG(this, "FAILED: allocating buffer"); throw std::runtime_error("FAILED: allocating buffer"); From 0bd29cc51bb1018ee5de12a8777a83499d4e9760 Mon Sep 17 00:00:00 2001 From: RockyLim92 Date: Tue, 8 Jan 2019 00:46:19 -0500 Subject: [PATCH 4/6] sync --- nvm/run.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nvm/run.sh b/nvm/run.sh index 659102d6dda9..a1b32a5a0aad 100755 --- a/nvm/run.sh +++ b/nvm/run.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash + +RBENCH_DEV_NAME="nvme0n1" +RBENCH_BENCHMARKS="fillseq, stats" +RBENCH_DEV_MODE="nvm" + if [ $UID != 0 ]; then echo "You don't have sufficient privileges to run this script." exit 1 @@ -84,10 +89,10 @@ RBENCH_TARGET_FILE_SIZE_BASE=$((2000 * MB)) RBENCH_MAX_BYTES_FOR_LEVEL_BASE=$((4000 * MB)) RBENCH_CACHE_NUMSHARDBITS=4 RBENCH_COMPRESSION_RATIO=1 -RBENCH_COMPRESSION_TYPE=snappy +RBENCH_COMPRESSION_TYPE=none RBENCH_DELETE_OBSOLETE_FILES_PERIOD_MICROS=300000000 RBENCH_DISABLE_SEEK_COMPACTION=1 -RBENCH_DISABLE_WAL=1 +RBENCH_DISABLE_WAL=0 RBENCH_LEVEL0_FILE_NUM_COMPACTION_TRIGGER=1 RBENCH_LEVEL0_SLOWDOWN_WRITES_TRIGGER=8 RBENCH_LEVEL0_STOP_WRITES_TRIGGER=12 @@ -100,8 +105,8 @@ RBENCH_STATISTICS=1 RBENCH_STATS_INTERVAL=100000 RBENCH_STATS_PER_INTERVAL=1 RBENCH_SYNC=0 -#RBENCH_THREADS=1 -RBENCH_THREADS=$(nproc) +RBENCH_THREADS=1 +#RBENCH_THREADS=$(nproc) RBENCH_VERIFY_CHECKSUM=1 source rbench.sh From a06511d99127f8821db786076b2359bbd1ffc48f Mon Sep 17 00:00:00 2001 From: RockyLim92 Date: Wed, 16 Jan 2019 06:18:35 -0500 Subject: [PATCH 5/6] sync --- nvm/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nvm/run.sh b/nvm/run.sh index a1b32a5a0aad..1440d9718664 100755 --- a/nvm/run.sh +++ b/nvm/run.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash RBENCH_DEV_NAME="nvme0n1" -RBENCH_BENCHMARKS="fillseq, stats" +RBENCH_BENCHMARKS="fillrandom, stats" RBENCH_DEV_MODE="nvm" if [ $UID != 0 ]; then @@ -97,6 +97,7 @@ RBENCH_LEVEL0_FILE_NUM_COMPACTION_TRIGGER=1 RBENCH_LEVEL0_SLOWDOWN_WRITES_TRIGGER=8 RBENCH_LEVEL0_STOP_WRITES_TRIGGER=12 RBENCH_MAX_BACKGROUND_COMPACTIONS=20 +RBENCH_MAX_BACKGROUND_FLUSHES=20 RBENCH_MIN_LEVEL_TO_COMPRESS=2 RBENCH_MMAP_READ=0 RBENCH_NUM_LEVELS=6 From 2c7374dab40f5d57d7d298fcf2bf1b79c9fceebe Mon Sep 17 00:00:00 2001 From: HeerakLim Date: Mon, 8 Jul 2019 21:12:10 +0900 Subject: [PATCH 6/6] delete duplicated file(stack_trace.cc) --- src.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/src.mk b/src.mk index 35a87de18951..016aee21547f 100644 --- a/src.mk +++ b/src.mk @@ -89,7 +89,6 @@ LIB_SOURCES = \ nvm/env_nvm_file.cc \ port/stack_trace.cc \ port/port_posix.cc \ - port/stack_trace.cc \ table/adaptive_table_factory.cc \ table/block.cc \ table/block_based_filter_block.cc \