Merge branch 'master' of github:google/leveldb

This commit is contained in:
Patrick Widener 2020-01-07 15:19:02 -07:00
commit 7ea78c8090
145 changed files with 6614 additions and 5075 deletions

36
.appveyor.yml Normal file
View File

@ -0,0 +1,36 @@
# Build matrix / environment variables are explained on:
# https://www.appveyor.com/docs/appveyor-yml/
# This file can be validated on: https://ci.appveyor.com/tools/validate-yaml
version: "{build}"
environment:
matrix:
# AppVeyor currently has no custom job name feature.
# http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs
- JOB: Visual Studio 2019
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
platform:
- x86
- x64
configuration:
- RelWithDebInfo
- Debug
build_script:
- git submodule update --init --recursive
- mkdir build
- cd build
- if "%platform%"=="x86" (set CMAKE_GENERATOR_PLATFORM="Win32")
else (set CMAKE_GENERATOR_PLATFORM="%platform%")
- cmake --version
- cmake .. -G "%CMAKE_GENERATOR%" -A "%CMAKE_GENERATOR_PLATFORM%"
-DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%"
- cmake --build . --config "%CONFIGURATION%"
- cd ..
test_script:
- cd build && ctest --verbose --build-config "%CONFIGURATION%" && cd ..

18
.clang-format Normal file
View File

@ -0,0 +1,18 @@
# Run manually to reformat a file:
# clang-format -i --style=file <file>
# find . -iname '*.cc' -o -iname '*.h' -o -iname '*.h.in' | xargs clang-format -i --style=file
BasedOnStyle: Google
DerivePointerAlignment: false
# Public headers are in a different location in the internal Google repository.
# Order them so that when imported to the authoritative repository they will be
# in correct alphabetical order.
IncludeCategories:
- Regex: '^(<|"(benchmarks|db|helpers)/)'
Priority: 1
- Regex: '^"(leveldb)/'
Priority: 2
- Regex: '^(<|"(issues|port|table|third_party|util)/)'
Priority: 3
- Regex: '.*'
Priority: 4

17
.gitignore vendored
View File

@ -1,9 +1,8 @@
build_config.mk # Editors.
*.a *.sw*
*.o .vscode
*.dylib* .DS_Store
*.so
*.so.* # Build directory.
*_test build/
db_bench out/
leveldbutil

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "third_party/googletest"]
path = third_party/googletest
url = https://github.com/google/googletest.git

View File

@ -1,10 +1,10 @@
# Build matrix / environment variable are explained on: # Build matrix / environment variables are explained on:
# http://about.travis-ci.org/docs/user/build-configuration/ # http://about.travis-ci.org/docs/user/build-configuration/
# This file can be validated on: http://lint.travis-ci.org/ # This file can be validated on: http://lint.travis-ci.org/
sudo: false
dist: trusty
language: cpp language: cpp
dist: bionic
osx_image: xcode10.3
compiler: compiler:
- gcc - gcc
@ -19,38 +19,49 @@ env:
addons: addons:
apt: apt:
# List of whitelisted in travis packages for ubuntu-trusty can be found here:
# https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-trusty
# List of whitelisted in travis apt-sources:
# https://github.com/travis-ci/apt-source-whitelist/blob/master/ubuntu.json
sources: sources:
- ubuntu-toolchain-r-test - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main'
- llvm-toolchain-trusty-5.0 key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
- sourceline: 'ppa:ubuntu-toolchain-r/test'
packages: packages:
- clang-9
- cmake - cmake
- gcc-7 - gcc-9
- g++-7 - g++-9
- clang-5.0
- libgoogle-perftools-dev - libgoogle-perftools-dev
- libkyotocabinet-dev - libkyotocabinet-dev
- libsnappy-dev - libsnappy-dev
- libsqlite3-dev - libsqlite3-dev
- ninja-build
homebrew:
packages:
- cmake
- crc32c
- gcc@9
- gperftools
- kyoto-cabinet
- llvm@9
- ninja
- snappy
- sqlite3
update: true
install: install:
# Travis doesn't have a DSL for installing homebrew packages yet. Status tracked # The following Homebrew packages aren't linked by default, and need to be
# in https://github.com/travis-ci/travis-ci/issues/5377 # prepended to the path explicitly.
# The Travis VM image for Mac already has a link at /usr/local/include/c++, - if [ "$TRAVIS_OS_NAME" = "osx" ]; then
# causing Homebrew's gcc@7 installation to error out. This was reported to export PATH="$(brew --prefix llvm)/bin:$PATH";
# Homebrew maintainers at https://github.com/Homebrew/brew/issues/1742 and fi
# removing the link emerged as a workaround. # /usr/bin/gcc points to an older compiler on both Linux and macOS.
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then - if [ "$CXX" = "g++" ]; then export CXX="g++-9" CC="gcc-9"; fi
brew update; # /usr/bin/clang points to an older compiler on both Linux and macOS.
if [ -L /usr/local/include/c++ ]; then rm /usr/local/include/c++; fi; #
brew install gcc@7; # Homebrew's llvm package doesn't ship a versioned clang++ binary, so the values
brew install crc32c gperftools kyoto-cabinet snappy sqlite3; # below don't work on macOS. Fortunately, the path change above makes the
# default values (clang and clang++) resolve to the correct compiler on macOS.
- if [ "$TRAVIS_OS_NAME" = "linux" ]; then
if [ "$CXX" = "clang++" ]; then export CXX="clang++-9" CC="clang-9"; fi;
fi fi
# /usr/bin/gcc is stuck to old versions on both Linux and OSX.
- if [ "$CXX" = "g++" ]; then export CXX="g++-7" CC="gcc-7"; fi
- echo ${CC} - echo ${CC}
- echo ${CXX} - echo ${CXX}
- ${CXX} --version - ${CXX} --version
@ -58,12 +69,14 @@ install:
before_script: before_script:
- mkdir -p build && cd build - mkdir -p build && cd build
- cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=$BUILD_TYPE
-DCMAKE_INSTALL_PREFIX=$HOME/.local
- cmake --build . - cmake --build .
- cd .. - cd ..
script: script:
- cd build ; ctest --verbose ; cd .. - cd build && ctest --verbose && cd ..
- "if [ -f build/db_bench ] ; then build/db_bench ; fi" - "if [ -f build/db_bench ] ; then build/db_bench ; fi"
- "if [ -f build/db_bench_sqlite3 ] ; then build/db_bench_sqlite3 ; fi" - "if [ -f build/db_bench_sqlite3 ] ; then build/db_bench_sqlite3 ; fi"
- "if [ -f build/db_bench_tree_db ] ; then build/db_bench_tree_db ; fi" - "if [ -f build/db_bench_tree_db ] ; then build/db_bench_tree_db ; fi"
- cd build && cmake --build . --target install

View File

@ -3,17 +3,32 @@
# found in the LICENSE file. See the AUTHORS file for names of contributors. # found in the LICENSE file. See the AUTHORS file for names of contributors.
cmake_minimum_required(VERSION 3.9) cmake_minimum_required(VERSION 3.9)
project(leveldb VERSION 1.21.0 LANGUAGES C CXX) # Keep the version below in sync with the one in db.h
project(leveldb VERSION 1.22.0 LANGUAGES C CXX)
# C standard can be overridden when this is used as a sub-project.
if(NOT CMAKE_C_STANDARD)
# This project can use C11, but will gracefully decay down to C89. # This project can use C11, but will gracefully decay down to C89.
set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED OFF) set(CMAKE_C_STANDARD_REQUIRED OFF)
set(CMAKE_C_EXTENSIONS OFF) set(CMAKE_C_EXTENSIONS OFF)
endif(NOT CMAKE_C_STANDARD)
# C++ standard can be overridden when this is used as a sub-project.
if(NOT CMAKE_CXX_STANDARD)
# This project requires C++11. # This project requires C++11.
set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_EXTENSIONS OFF)
endif(NOT CMAKE_CXX_STANDARD)
if (WIN32)
set(LEVELDB_PLATFORM_NAME LEVELDB_PLATFORM_WINDOWS)
# TODO(cmumford): Make UNICODE configurable for Windows.
add_definitions(-D_UNICODE -DUNICODE)
else (WIN32)
set(LEVELDB_PLATFORM_NAME LEVELDB_PLATFORM_POSIX)
endif (WIN32)
option(LEVELDB_BUILD_TESTS "Build LevelDB's unit tests" ON) option(LEVELDB_BUILD_TESTS "Build LevelDB's unit tests" ON)
option(LEVELDB_BUILD_BENCHMARKS "Build LevelDB's benchmarks" ON) option(LEVELDB_BUILD_BENCHMARKS "Build LevelDB's benchmarks" ON)
@ -30,28 +45,50 @@ check_library_exists(crc32c crc32c_value "" HAVE_CRC32C)
check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)
check_library_exists(tcmalloc malloc "" HAVE_TCMALLOC) check_library_exists(tcmalloc malloc "" HAVE_TCMALLOC)
include(CheckSymbolExists) include(CheckCXXSymbolExists)
check_symbol_exists(fdatasync "unistd.h" HAVE_FDATASYNC) # Using check_cxx_symbol_exists() instead of check_c_symbol_exists() because
# we're including the header from C++, and feature detection should use the same
# compiler language that the project will use later. Principles aside, some
# versions of do not expose fdatasync() in <unistd.h> in standard C mode
# (-std=c11), but do expose the function in standard C++ mode (-std=c++11).
check_cxx_symbol_exists(fdatasync "unistd.h" HAVE_FDATASYNC)
check_cxx_symbol_exists(F_FULLFSYNC "fcntl.h" HAVE_FULLFSYNC)
check_cxx_symbol_exists(O_CLOEXEC "fcntl.h" HAVE_O_CLOEXEC)
include(CheckCXXSourceCompiles) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Disable C++ exceptions.
string(REGEX REPLACE "/EH[a-z]+" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
add_definitions(-D_HAS_EXCEPTIONS=0)
# Disable RTTI.
string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Enable strict prototype warnings for C code in clang and gcc.
if(NOT CMAKE_C_FLAGS MATCHES "-Wstrict-prototypes")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes")
endif(NOT CMAKE_C_FLAGS MATCHES "-Wstrict-prototypes")
# Disable C++ exceptions.
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
# Disable RTTI.
string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Test whether -Wthread-safety is available. See # Test whether -Wthread-safety is available. See
# https://clang.llvm.org/docs/ThreadSafetyAnalysis.html # https://clang.llvm.org/docs/ThreadSafetyAnalysis.html
# -Werror is necessary because unknown attributes only generate warnings. include(CheckCXXCompilerFlag)
set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) check_cxx_compiler_flag(-Wthread-safety HAVE_CLANG_THREAD_SAFETY)
list(APPEND CMAKE_REQUIRED_FLAGS -Werror -Wthread-safety)
check_cxx_source_compiles(" # Used by googletest.
struct __attribute__((lockable)) Lock { check_cxx_compiler_flag(-Wno-missing-field-initializers
void Acquire() __attribute__((exclusive_lock_function())); LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS)
void Release() __attribute__((unlock_function()));
}; include(CheckCXXSourceCompiles)
struct ThreadSafeType {
Lock lock_;
int data_ __attribute__((guarded_by(lock_)));
};
int main() { return 0; }
" HAVE_CLANG_THREAD_SAFETY)
set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
# Test whether C++17 __has_include is available. # Test whether C++17 __has_include is available.
check_cxx_source_compiles(" check_cxx_source_compiles("
@ -65,13 +102,13 @@ set(LEVELDB_PUBLIC_INCLUDE_DIR "include/leveldb")
set(LEVELDB_PORT_CONFIG_DIR "include/port") set(LEVELDB_PORT_CONFIG_DIR "include/port")
configure_file( configure_file(
"${PROJECT_SOURCE_DIR}/port/port_config.h.in" "port/port_config.h.in"
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
) )
include_directories( include_directories(
"${PROJECT_BINARY_DIR}/include" "${PROJECT_BINARY_DIR}/include"
"${PROJECT_SOURCE_DIR}" "."
) )
if(BUILD_SHARED_LIBS) if(BUILD_SHARED_LIBS)
@ -79,79 +116,82 @@ if(BUILD_SHARED_LIBS)
add_compile_options(-fvisibility=hidden) add_compile_options(-fvisibility=hidden)
endif(BUILD_SHARED_LIBS) endif(BUILD_SHARED_LIBS)
# Must be included before CMAKE_INSTALL_INCLUDEDIR is used.
include(GNUInstallDirs)
add_library(leveldb "") add_library(leveldb "")
target_sources(leveldb target_sources(leveldb
PRIVATE PRIVATE
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
"${PROJECT_SOURCE_DIR}/db/builder.cc" "db/builder.cc"
"${PROJECT_SOURCE_DIR}/db/builder.h" "db/builder.h"
"${PROJECT_SOURCE_DIR}/db/c.cc" "db/c.cc"
"${PROJECT_SOURCE_DIR}/db/db_impl.cc" "db/db_impl.cc"
"${PROJECT_SOURCE_DIR}/db/db_impl.h" "db/db_impl.h"
"${PROJECT_SOURCE_DIR}/db/db_iter.cc" "db/db_iter.cc"
"${PROJECT_SOURCE_DIR}/db/db_iter.h" "db/db_iter.h"
"${PROJECT_SOURCE_DIR}/db/dbformat.cc" "db/dbformat.cc"
"${PROJECT_SOURCE_DIR}/db/dbformat.h" "db/dbformat.h"
"${PROJECT_SOURCE_DIR}/db/dumpfile.cc" "db/dumpfile.cc"
"${PROJECT_SOURCE_DIR}/db/filename.cc" "db/filename.cc"
"${PROJECT_SOURCE_DIR}/db/filename.h" "db/filename.h"
"${PROJECT_SOURCE_DIR}/db/log_format.h" "db/log_format.h"
"${PROJECT_SOURCE_DIR}/db/log_reader.cc" "db/log_reader.cc"
"${PROJECT_SOURCE_DIR}/db/log_reader.h" "db/log_reader.h"
"${PROJECT_SOURCE_DIR}/db/log_writer.cc" "db/log_writer.cc"
"${PROJECT_SOURCE_DIR}/db/log_writer.h" "db/log_writer.h"
"${PROJECT_SOURCE_DIR}/db/memtable.cc" "db/memtable.cc"
"${PROJECT_SOURCE_DIR}/db/memtable.h" "db/memtable.h"
"${PROJECT_SOURCE_DIR}/db/repair.cc" "db/repair.cc"
"${PROJECT_SOURCE_DIR}/db/skiplist.h" "db/skiplist.h"
"${PROJECT_SOURCE_DIR}/db/snapshot.h" "db/snapshot.h"
"${PROJECT_SOURCE_DIR}/db/table_cache.cc" "db/table_cache.cc"
"${PROJECT_SOURCE_DIR}/db/table_cache.h" "db/table_cache.h"
"${PROJECT_SOURCE_DIR}/db/version_edit.cc" "db/version_edit.cc"
"${PROJECT_SOURCE_DIR}/db/version_edit.h" "db/version_edit.h"
"${PROJECT_SOURCE_DIR}/db/version_set.cc" "db/version_set.cc"
"${PROJECT_SOURCE_DIR}/db/version_set.h" "db/version_set.h"
"${PROJECT_SOURCE_DIR}/db/write_batch_internal.h" "db/write_batch_internal.h"
"${PROJECT_SOURCE_DIR}/db/write_batch.cc" "db/write_batch.cc"
"${PROJECT_SOURCE_DIR}/port/atomic_pointer.h" "port/port_stdcxx.h"
"${PROJECT_SOURCE_DIR}/port/port_stdcxx.h" "port/port.h"
"${PROJECT_SOURCE_DIR}/port/port.h" "port/thread_annotations.h"
"${PROJECT_SOURCE_DIR}/port/thread_annotations.h" "table/block_builder.cc"
"${PROJECT_SOURCE_DIR}/table/block_builder.cc" "table/block_builder.h"
"${PROJECT_SOURCE_DIR}/table/block_builder.h" "table/block.cc"
"${PROJECT_SOURCE_DIR}/table/block.cc" "table/block.h"
"${PROJECT_SOURCE_DIR}/table/block.h" "table/filter_block.cc"
"${PROJECT_SOURCE_DIR}/table/filter_block.cc" "table/filter_block.h"
"${PROJECT_SOURCE_DIR}/table/filter_block.h" "table/format.cc"
"${PROJECT_SOURCE_DIR}/table/format.cc" "table/format.h"
"${PROJECT_SOURCE_DIR}/table/format.h" "table/iterator_wrapper.h"
"${PROJECT_SOURCE_DIR}/table/iterator_wrapper.h" "table/iterator.cc"
"${PROJECT_SOURCE_DIR}/table/iterator.cc" "table/merger.cc"
"${PROJECT_SOURCE_DIR}/table/merger.cc" "table/merger.h"
"${PROJECT_SOURCE_DIR}/table/merger.h" "table/table_builder.cc"
"${PROJECT_SOURCE_DIR}/table/table_builder.cc" "table/table.cc"
"${PROJECT_SOURCE_DIR}/table/table.cc" "table/two_level_iterator.cc"
"${PROJECT_SOURCE_DIR}/table/two_level_iterator.cc" "table/two_level_iterator.h"
"${PROJECT_SOURCE_DIR}/table/two_level_iterator.h" "util/arena.cc"
"${PROJECT_SOURCE_DIR}/util/arena.cc" "util/arena.h"
"${PROJECT_SOURCE_DIR}/util/arena.h" "util/bloom.cc"
"${PROJECT_SOURCE_DIR}/util/bloom.cc" "util/cache.cc"
"${PROJECT_SOURCE_DIR}/util/cache.cc" "util/coding.cc"
"${PROJECT_SOURCE_DIR}/util/coding.cc" "util/coding.h"
"${PROJECT_SOURCE_DIR}/util/coding.h" "util/comparator.cc"
"${PROJECT_SOURCE_DIR}/util/comparator.cc" "util/crc32c.cc"
"${PROJECT_SOURCE_DIR}/util/crc32c.cc" "util/crc32c.h"
"${PROJECT_SOURCE_DIR}/util/crc32c.h" "util/env.cc"
"${PROJECT_SOURCE_DIR}/util/env.cc" "util/filter_policy.cc"
"${PROJECT_SOURCE_DIR}/util/filter_policy.cc" "util/hash.cc"
"${PROJECT_SOURCE_DIR}/util/hash.cc" "util/hash.h"
"${PROJECT_SOURCE_DIR}/util/hash.h" "util/logging.cc"
"${PROJECT_SOURCE_DIR}/util/logging.cc" "util/logging.h"
"${PROJECT_SOURCE_DIR}/util/logging.h" "util/mutexlock.h"
"${PROJECT_SOURCE_DIR}/util/mutexlock.h" "util/no_destructor.h"
"${PROJECT_SOURCE_DIR}/util/options.cc" "util/options.cc"
"${PROJECT_SOURCE_DIR}/util/random.h" "util/random.h"
"${PROJECT_SOURCE_DIR}/util/status.cc" "util/status.cc"
# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
$<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC> $<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
@ -172,18 +212,25 @@ target_sources(leveldb
"${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h"
) )
# POSIX code is specified separately so we can leave it out in the future. if (WIN32)
target_sources(leveldb target_sources(leveldb
PRIVATE PRIVATE
"${PROJECT_SOURCE_DIR}/util/env_posix.cc" "util/env_windows.cc"
"${PROJECT_SOURCE_DIR}/util/posix_logger.h" "util/windows_logger.h"
) )
else (WIN32)
target_sources(leveldb
PRIVATE
"util/env_posix.cc"
"util/posix_logger.h"
)
endif (WIN32)
# MemEnv is not part of the interface and could be pulled to a separate library. # MemEnv is not part of the interface and could be pulled to a separate library.
target_sources(leveldb target_sources(leveldb
PRIVATE PRIVATE
"${PROJECT_SOURCE_DIR}/helpers/memenv/memenv.cc" "helpers/memenv/memenv.cc"
"${PROJECT_SOURCE_DIR}/helpers/memenv/memenv.h" "helpers/memenv/memenv.h"
) )
target_include_directories(leveldb target_include_directories(leveldb
@ -198,12 +245,15 @@ if(LEVELDB_INSTALL)
) )
endif(LEVELDB_INSTALL) endif(LEVELDB_INSTALL)
set_target_properties(leveldb
PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
target_compile_definitions(leveldb target_compile_definitions(leveldb
PRIVATE PRIVATE
# Used by include/export.h when building shared libraries. # Used by include/export.h when building shared libraries.
LEVELDB_COMPILE_LIBRARY LEVELDB_COMPILE_LIBRARY
# Used by port/port.h. # Used by port/port.h.
LEVELDB_PLATFORM_POSIX=1 ${LEVELDB_PLATFORM_NAME}=1
) )
if (NOT HAVE_CXX17_HAS_INCLUDE) if (NOT HAVE_CXX17_HAS_INCLUDE)
target_compile_definitions(leveldb target_compile_definitions(leveldb
@ -241,13 +291,30 @@ find_package(Threads REQUIRED)
target_link_libraries(leveldb Threads::Threads) target_link_libraries(leveldb Threads::Threads)
add_executable(leveldbutil add_executable(leveldbutil
"${PROJECT_SOURCE_DIR}/db/leveldbutil.cc" "db/leveldbutil.cc"
) )
target_link_libraries(leveldbutil leveldb) target_link_libraries(leveldbutil leveldb)
if(LEVELDB_BUILD_TESTS) if(LEVELDB_BUILD_TESTS)
enable_testing() enable_testing()
# Prevent overriding the parent project's compiler/linker settings on Windows.
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
set(install_gtest OFF)
set(install_gmock OFF)
set(build_gmock ON)
# This project is tested using GoogleTest.
add_subdirectory("third_party/googletest")
# GoogleTest triggers a missing field initializers warning.
if(LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS)
set_property(TARGET gtest
APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers)
set_property(TARGET gmock
APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers)
endif(LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS)
function(leveldb_test test_file) function(leveldb_test test_file)
get_filename_component(test_target_name "${test_file}" NAME_WE) get_filename_component(test_target_name "${test_file}" NAME_WE)
@ -255,17 +322,15 @@ if(LEVELDB_BUILD_TESTS)
target_sources("${test_target_name}" target_sources("${test_target_name}"
PRIVATE PRIVATE
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
"${PROJECT_SOURCE_DIR}/util/testharness.cc" "util/testutil.cc"
"${PROJECT_SOURCE_DIR}/util/testharness.h" "util/testutil.h"
"${PROJECT_SOURCE_DIR}/util/testutil.cc"
"${PROJECT_SOURCE_DIR}/util/testutil.h"
"${test_file}" "${test_file}"
) )
target_link_libraries("${test_target_name}" leveldb) target_link_libraries("${test_target_name}" leveldb gmock gtest)
target_compile_definitions("${test_target_name}" target_compile_definitions("${test_target_name}"
PRIVATE PRIVATE
LEVELDB_PLATFORM_POSIX=1 ${LEVELDB_PLATFORM_NAME}=1
) )
if (NOT HAVE_CXX17_HAS_INCLUDE) if (NOT HAVE_CXX17_HAS_INCLUDE)
target_compile_definitions("${test_target_name}" target_compile_definitions("${test_target_name}"
@ -277,44 +342,50 @@ if(LEVELDB_BUILD_TESTS)
add_test(NAME "${test_target_name}" COMMAND "${test_target_name}") add_test(NAME "${test_target_name}" COMMAND "${test_target_name}")
endfunction(leveldb_test) endfunction(leveldb_test)
leveldb_test("${PROJECT_SOURCE_DIR}/db/c_test.c") leveldb_test("db/c_test.c")
leveldb_test("${PROJECT_SOURCE_DIR}/db/fault_injection_test.cc") leveldb_test("db/fault_injection_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/issues/issue178_test.cc") leveldb_test("issues/issue178_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/issues/issue200_test.cc") leveldb_test("issues/issue200_test.cc")
leveldb_test("issues/issue320_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/env_test.cc") leveldb_test("util/env_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/status_test.cc") leveldb_test("util/status_test.cc")
leveldb_test("util/no_destructor_test.cc")
if(NOT BUILD_SHARED_LIBS) if(NOT BUILD_SHARED_LIBS)
leveldb_test("${PROJECT_SOURCE_DIR}/db/autocompact_test.cc") leveldb_test("db/autocompact_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/corruption_test.cc") leveldb_test("db/corruption_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/db_test.cc") leveldb_test("db/db_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/dbformat_test.cc") leveldb_test("db/dbformat_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/filename_test.cc") leveldb_test("db/filename_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/log_test.cc") leveldb_test("db/log_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/recovery_test.cc") leveldb_test("db/recovery_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/skiplist_test.cc") leveldb_test("db/skiplist_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/version_edit_test.cc") leveldb_test("db/version_edit_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/version_set_test.cc") leveldb_test("db/version_set_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/db/write_batch_test.cc") leveldb_test("db/write_batch_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/helpers/memenv/memenv_test.cc") leveldb_test("helpers/memenv/memenv_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/table/filter_block_test.cc") leveldb_test("table/filter_block_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/table/table_test.cc") leveldb_test("table/table_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/arena_test.cc") leveldb_test("util/arena_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/bloom_test.cc") leveldb_test("util/bloom_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/cache_test.cc") leveldb_test("util/cache_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/coding_test.cc") leveldb_test("util/coding_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/crc32c_test.cc") leveldb_test("util/crc32c_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/hash_test.cc") leveldb_test("util/hash_test.cc")
leveldb_test("${PROJECT_SOURCE_DIR}/util/logging_test.cc") leveldb_test("util/logging_test.cc")
# TODO(costan): This test also uses # TODO(costan): This test also uses
# "${PROJECT_SOURCE_DIR}/util/env_posix_test_helper.h" # "util/env_{posix|windows}_test_helper.h"
leveldb_test("${PROJECT_SOURCE_DIR}/util/env_posix_test.cc") if (WIN32)
leveldb_test("util/env_windows_test.cc")
else (WIN32)
leveldb_test("util/env_posix_test.cc")
endif (WIN32)
endif(NOT BUILD_SHARED_LIBS) endif(NOT BUILD_SHARED_LIBS)
endif(LEVELDB_BUILD_TESTS) endif(LEVELDB_BUILD_TESTS)
@ -326,19 +397,17 @@ if(LEVELDB_BUILD_BENCHMARKS)
target_sources("${bench_target_name}" target_sources("${bench_target_name}"
PRIVATE PRIVATE
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
"${PROJECT_SOURCE_DIR}/util/histogram.cc" "util/histogram.cc"
"${PROJECT_SOURCE_DIR}/util/histogram.h" "util/histogram.h"
"${PROJECT_SOURCE_DIR}/util/testharness.cc" "util/testutil.cc"
"${PROJECT_SOURCE_DIR}/util/testharness.h" "util/testutil.h"
"${PROJECT_SOURCE_DIR}/util/testutil.cc"
"${PROJECT_SOURCE_DIR}/util/testutil.h"
"${bench_file}" "${bench_file}"
) )
target_link_libraries("${bench_target_name}" leveldb) target_link_libraries("${bench_target_name}" leveldb gmock gtest)
target_compile_definitions("${bench_target_name}" target_compile_definitions("${bench_target_name}"
PRIVATE PRIVATE
LEVELDB_PLATFORM_POSIX=1 ${LEVELDB_PLATFORM_NAME}=1
) )
if (NOT HAVE_CXX17_HAS_INCLUDE) if (NOT HAVE_CXX17_HAS_INCLUDE)
target_compile_definitions("${bench_target_name}" target_compile_definitions("${bench_target_name}"
@ -349,12 +418,12 @@ if(LEVELDB_BUILD_BENCHMARKS)
endfunction(leveldb_benchmark) endfunction(leveldb_benchmark)
if(NOT BUILD_SHARED_LIBS) if(NOT BUILD_SHARED_LIBS)
leveldb_benchmark("${PROJECT_SOURCE_DIR}/db/db_bench.cc") leveldb_benchmark("benchmarks/db_bench.cc")
endif(NOT BUILD_SHARED_LIBS) endif(NOT BUILD_SHARED_LIBS)
check_library_exists(sqlite3 sqlite3_open "" HAVE_SQLITE3) check_library_exists(sqlite3 sqlite3_open "" HAVE_SQLITE3)
if(HAVE_SQLITE3) if(HAVE_SQLITE3)
leveldb_benchmark("${PROJECT_SOURCE_DIR}/doc/bench/db_bench_sqlite3.cc") leveldb_benchmark("benchmarks/db_bench_sqlite3.cc")
target_link_libraries(db_bench_sqlite3 sqlite3) target_link_libraries(db_bench_sqlite3 sqlite3)
endif(HAVE_SQLITE3) endif(HAVE_SQLITE3)
@ -374,13 +443,12 @@ int main() {
" HAVE_KYOTOCABINET) " HAVE_KYOTOCABINET)
set(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQURED_LIBRARIES}) set(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQURED_LIBRARIES})
if(HAVE_KYOTOCABINET) if(HAVE_KYOTOCABINET)
leveldb_benchmark("${PROJECT_SOURCE_DIR}/doc/bench/db_bench_tree_db.cc") leveldb_benchmark("benchmarks/db_bench_tree_db.cc")
target_link_libraries(db_bench_tree_db kyotocabinet) target_link_libraries(db_bench_tree_db kyotocabinet)
endif(HAVE_KYOTOCABINET) endif(HAVE_KYOTOCABINET)
endif(LEVELDB_BUILD_BENCHMARKS) endif(LEVELDB_BUILD_BENCHMARKS)
if(LEVELDB_INSTALL) if(LEVELDB_INSTALL)
include(GNUInstallDirs)
install(TARGETS leveldb install(TARGETS leveldb
EXPORT leveldbTargets EXPORT leveldbTargets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
@ -389,38 +457,43 @@ if(LEVELDB_INSTALL)
) )
install( install(
FILES FILES
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/c.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/c.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/cache.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/cache.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/comparator.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/comparator.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/db.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/db.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/dumpfile.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/dumpfile.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/env.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/env.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/export.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/export.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/filter_policy.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/filter_policy.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/iterator.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/iterator.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/options.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/options.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/slice.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/slice.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/status.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/status.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/table_builder.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/table_builder.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/table.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/table.h"
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h" "${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h"
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/leveldb DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/leveldb"
) )
include(CMakePackageConfigHelpers) include(CMakePackageConfigHelpers)
configure_package_config_file(
"cmake/${PROJECT_NAME}Config.cmake.in"
"${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake"
INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}"
)
write_basic_package_version_file( write_basic_package_version_file(
"${PROJECT_BINARY_DIR}/leveldbConfigVersion.cmake" "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake"
COMPATIBILITY SameMajorVersion COMPATIBILITY SameMajorVersion
) )
install( install(
EXPORT leveldbTargets EXPORT leveldbTargets
NAMESPACE leveldb:: NAMESPACE leveldb::
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/leveldb" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}"
) )
install( install(
FILES FILES
"${PROJECT_SOURCE_DIR}/cmake/leveldbConfig.cmake" "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake"
"${PROJECT_BINARY_DIR}/leveldbConfigVersion.cmake" "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/leveldb" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}"
) )
endif(LEVELDB_INSTALL) endif(LEVELDB_INSTALL)

View File

@ -1,6 +1,7 @@
**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.** **LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**
[![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb) [![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb)
[![Build status](https://ci.appveyor.com/api/projects/status/g2j5j4rfkda6eyw5/branch/master?svg=true)](https://ci.appveyor.com/project/pwnall/leveldb)
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
@ -26,10 +27,18 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
* Only a single process (possibly multi-threaded) can access a particular database at a time. * Only a single process (possibly multi-threaded) can access a particular database at a time.
* There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library. * There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library.
# Getting the Source
```bash
git clone --recurse-submodules https://github.com/google/leveldb.git
```
# Building # Building
This project supports [CMake](https://cmake.org/) out of the box. This project supports [CMake](https://cmake.org/) out of the box.
### Build for POSIX
Quick start: Quick start:
```bash ```bash
@ -37,6 +46,29 @@ mkdir -p build && cd build
cmake -DCMAKE_BUILD_TYPE=Release .. && cmake --build . cmake -DCMAKE_BUILD_TYPE=Release .. && cmake --build .
``` ```
### Building for Windows
First generate the Visual Studio 2017 project/solution files:
```cmd
mkdir build
cd build
cmake -G "Visual Studio 15" ..
```
The default default will build for x86. For 64-bit run:
```cmd
cmake -G "Visual Studio 15 Win64" ..
```
To compile the Windows solution from the command-line:
```cmd
devenv /build Debug leveldb.sln
```
or open leveldb.sln in Visual Studio and build from within.
Please see the CMake documentation and `CMakeLists.txt` for more advanced usage. Please see the CMake documentation and `CMakeLists.txt` for more advanced usage.
# Contributing to the leveldb Project # Contributing to the leveldb Project
@ -48,10 +80,10 @@ will be considered.
Contribution requirements: Contribution requirements:
1. **POSIX only**. We _generally_ will only accept changes that are both 1. **Tested platforms only**. We _generally_ will only accept changes for
compiled, and tested on a POSIX platform - usually Linux. Very small platforms that are compiled and tested. This means POSIX (for Linux and
changes will sometimes be accepted, but consider that more of an macOS) or Windows. Very small changes will sometimes be accepted, but
exception than the rule. consider that more of an exception than the rule.
2. **Stable API**. We strive very hard to maintain a stable API. Changes that 2. **Stable API**. We strive very hard to maintain a stable API. Changes that
require changes for projects using leveldb _might_ be rejected without require changes for projects using leveldb _might_ be rejected without
@ -60,6 +92,14 @@ Contribution requirements:
3. **Tests**: All changes must be accompanied by a new (or changed) test, or 3. **Tests**: All changes must be accompanied by a new (or changed) test, or
a sufficient explanation as to why a new (or changed) test is not required. a sufficient explanation as to why a new (or changed) test is not required.
4. **Consistent Style**: This project conforms to the
[Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html).
To ensure your changes are properly formatted please run:
```
clang-format -i --style=file <file>
```
## Submitting a Pull Request ## Submitting a Pull Request
Before any pull request will be accepted the author must first sign a Before any pull request will be accepted the author must first sign a
@ -155,37 +195,37 @@ uncompressed blocks in memory, the read performance improves again:
See [doc/index.md](doc/index.md) for more explanation. See See [doc/index.md](doc/index.md) for more explanation. See
[doc/impl.md](doc/impl.md) for a brief overview of the implementation. [doc/impl.md](doc/impl.md) for a brief overview of the implementation.
The public interface is in include/*.h. Callers should not include or The public interface is in include/leveldb/*.h. Callers should not include or
rely on the details of any other header files in this package. Those rely on the details of any other header files in this package. Those
internal APIs may be changed without warning. internal APIs may be changed without warning.
Guide to header files: Guide to header files:
* **include/db.h**: Main interface to the DB: Start here * **include/leveldb/db.h**: Main interface to the DB: Start here.
* **include/options.h**: Control over the behavior of an entire database, * **include/leveldb/options.h**: Control over the behavior of an entire database,
and also control over the behavior of individual reads and writes. and also control over the behavior of individual reads and writes.
* **include/comparator.h**: Abstraction for user-specified comparison function. * **include/leveldb/comparator.h**: Abstraction for user-specified comparison function.
If you want just bytewise comparison of keys, you can use the default If you want just bytewise comparison of keys, you can use the default
comparator, but clients can write their own comparator implementations if they comparator, but clients can write their own comparator implementations if they
want custom ordering (e.g. to handle different character encodings, etc.) want custom ordering (e.g. to handle different character encodings, etc.).
* **include/iterator.h**: Interface for iterating over data. You can get * **include/leveldb/iterator.h**: Interface for iterating over data. You can get
an iterator from a DB object. an iterator from a DB object.
* **include/write_batch.h**: Interface for atomically applying multiple * **include/leveldb/write_batch.h**: Interface for atomically applying multiple
updates to a database. updates to a database.
* **include/slice.h**: A simple module for maintaining a pointer and a * **include/leveldb/slice.h**: A simple module for maintaining a pointer and a
length into some other byte array. length into some other byte array.
* **include/status.h**: Status is returned from many of the public interfaces * **include/leveldb/status.h**: Status is returned from many of the public interfaces
and is used to report success and various kinds of errors. and is used to report success and various kinds of errors.
* **include/env.h**: * **include/leveldb/env.h**:
Abstraction of the OS environment. A posix implementation of this interface is Abstraction of the OS environment. A posix implementation of this interface is
in util/env_posix.cc in util/env_posix.cc.
* **include/table.h, include/table_builder.h**: Lower-level modules that most * **include/leveldb/table.h, include/leveldb/table_builder.h**: Lower-level modules that most
clients probably won't use directly clients probably won't use directly.

View File

@ -2,9 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <sys/types.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <sys/types.h>
#include "leveldb/cache.h" #include "leveldb/cache.h"
#include "leveldb/db.h" #include "leveldb/db.h"
#include "leveldb/env.h" #include "leveldb/env.h"
@ -34,7 +35,6 @@
// seekrandom -- N random seeks // seekrandom -- N random seeks
// open -- cost of opening a DB // open -- cost of opening a DB
// crc32c -- repeated crc32c of 4K of data // crc32c -- repeated crc32c of 4K of data
// acquireload -- load N*1000 times
// Meta operations: // Meta operations:
// compact -- Compact the entire DB // compact -- Compact the entire DB
// stats -- Print DB stats // stats -- Print DB stats
@ -56,9 +56,7 @@ static const char* FLAGS_benchmarks =
"fill100K," "fill100K,"
"crc32c," "crc32c,"
"snappycomp," "snappycomp,"
"snappyuncomp," "snappyuncomp,";
"acquireload,"
;
// Number of key/values to place in database // Number of key/values to place in database
static int FLAGS_num = 1000000; static int FLAGS_num = 1000000;
@ -189,14 +187,12 @@ class Stats {
void Start() { void Start() {
next_report_ = 100; next_report_ = 100;
last_op_finish_ = start_;
hist_.Clear(); hist_.Clear();
done_ = 0; done_ = 0;
bytes_ = 0; bytes_ = 0;
seconds_ = 0; seconds_ = 0;
start_ = g_env->NowMicros();
finish_ = start_;
message_.clear(); message_.clear();
start_ = finish_ = last_op_finish_ = g_env->NowMicros();
} }
void Merge(const Stats& other) { void Merge(const Stats& other) {
@ -216,9 +212,7 @@ class Stats {
seconds_ = (finish_ - start_) * 1e-6; seconds_ = (finish_ - start_) * 1e-6;
} }
void AddMessage(Slice msg) { void AddMessage(Slice msg) { AppendWithSpace(&message_, msg); }
AppendWithSpace(&message_, msg);
}
void FinishedSingleOp() { void FinishedSingleOp() {
if (FLAGS_histogram) { if (FLAGS_histogram) {
@ -234,21 +228,26 @@ class Stats {
done_++; done_++;
if (done_ >= next_report_) { if (done_ >= next_report_) {
if (next_report_ < 1000) next_report_ += 100; if (next_report_ < 1000)
else if (next_report_ < 5000) next_report_ += 500; next_report_ += 100;
else if (next_report_ < 10000) next_report_ += 1000; else if (next_report_ < 5000)
else if (next_report_ < 50000) next_report_ += 5000; next_report_ += 500;
else if (next_report_ < 100000) next_report_ += 10000; else if (next_report_ < 10000)
else if (next_report_ < 500000) next_report_ += 50000; next_report_ += 1000;
else next_report_ += 100000; else if (next_report_ < 50000)
next_report_ += 5000;
else if (next_report_ < 100000)
next_report_ += 10000;
else if (next_report_ < 500000)
next_report_ += 50000;
else
next_report_ += 100000;
fprintf(stderr, "... finished %d ops%30s\r", done_, ""); fprintf(stderr, "... finished %d ops%30s\r", done_, "");
fflush(stderr); fflush(stderr);
} }
} }
void AddBytes(int64_t n) { void AddBytes(int64_t n) { bytes_ += n; }
bytes_ += n;
}
void Report(const Slice& name) { void Report(const Slice& name) {
// Pretend at least one op was done in case we are running a benchmark // Pretend at least one op was done in case we are running a benchmark
@ -267,11 +266,8 @@ class Stats {
} }
AppendWithSpace(&extra, message_); AppendWithSpace(&extra, message_);
fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", name.ToString().c_str(),
name.ToString().c_str(), seconds_ * 1e6 / done_, (extra.empty() ? "" : " "), extra.c_str());
seconds_ * 1e6 / done_,
(extra.empty() ? "" : " "),
extra.c_str());
if (FLAGS_histogram) { if (FLAGS_histogram) {
fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
} }
@ -306,10 +302,7 @@ struct ThreadState {
Stats stats; Stats stats;
SharedState* shared; SharedState* shared;
ThreadState(int index) ThreadState(int index) : tid(index), rand(1000 + index), shared(nullptr) {}
: tid(index),
rand(1000 + index) {
}
}; };
} // namespace } // namespace
@ -335,20 +328,20 @@ class Benchmark {
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
fprintf(stdout, "Entries: %d\n", num_); fprintf(stdout, "Entries: %d\n", num_);
fprintf(stdout, "RawSize: %.1f MB (estimated)\n", fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
/ 1048576.0)); 1048576.0));
fprintf(stdout, "FileSize: %.1f MB (estimated)\n", fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) /
/ 1048576.0)); 1048576.0));
PrintWarnings(); PrintWarnings();
fprintf(stdout, "------------------------------------------------\n"); fprintf(stdout, "------------------------------------------------\n");
} }
void PrintWarnings() { void PrintWarnings() {
#if defined(__GNUC__) && !defined(__OPTIMIZE__) #if defined(__GNUC__) && !defined(__OPTIMIZE__)
fprintf(stdout, fprintf(
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" stdout,
); "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
#endif #endif
#ifndef NDEBUG #ifndef NDEBUG
fprintf(stdout, fprintf(stdout,
@ -366,8 +359,8 @@ class Benchmark {
} }
void PrintEnvironment() { void PrintEnvironment() {
fprintf(stderr, "LevelDB: version %d.%d\n", fprintf(stderr, "LevelDB: version %d.%d\n", kMajorVersion,
kMajorVersion, kMinorVersion); kMinorVersion);
#if defined(__linux) #if defined(__linux)
time_t now = time(nullptr); time_t now = time(nullptr);
@ -510,8 +503,6 @@ class Benchmark {
method = &Benchmark::Compact; method = &Benchmark::Compact;
} else if (name == Slice("crc32c")) { } else if (name == Slice("crc32c")) {
method = &Benchmark::Crc32c; method = &Benchmark::Crc32c;
} else if (name == Slice("acquireload")) {
method = &Benchmark::AcquireLoad;
} else if (name == Slice("snappycomp")) { } else if (name == Slice("snappycomp")) {
method = &Benchmark::SnappyCompress; method = &Benchmark::SnappyCompress;
} else if (name == Slice("snappyuncomp")) { } else if (name == Slice("snappyuncomp")) {
@ -523,7 +514,7 @@ class Benchmark {
} else if (name == Slice("sstables")) { } else if (name == Slice("sstables")) {
PrintStats("leveldb.sstables"); PrintStats("leveldb.sstables");
} else { } else {
if (name != Slice()) { // No error message for empty name if (!name.empty()) { // No error message for empty name
fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str()); fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
} }
} }
@ -639,22 +630,6 @@ class Benchmark {
thread->stats.AddMessage(label); thread->stats.AddMessage(label);
} }
void AcquireLoad(ThreadState* thread) {
int dummy;
port::AtomicPointer ap(&dummy);
int count = 0;
void *ptr = nullptr;
thread->stats.AddMessage("(each op is 1000 loads)");
while (count < 100000) {
for (int i = 0; i < 1000; i++) {
ptr = ap.Acquire_Load();
}
count++;
thread->stats.FinishedSingleOp();
}
if (ptr == nullptr) exit(1); // Disable unused variable warning.
}
void SnappyCompress(ThreadState* thread) { void SnappyCompress(ThreadState* thread) {
RandomGenerator gen; RandomGenerator gen;
Slice input = gen.Generate(Options().block_size); Slice input = gen.Generate(Options().block_size);
@ -729,13 +704,9 @@ class Benchmark {
} }
} }
void WriteSeq(ThreadState* thread) { void WriteSeq(ThreadState* thread) { DoWrite(thread, true); }
DoWrite(thread, true);
}
void WriteRandom(ThreadState* thread) { void WriteRandom(ThreadState* thread) { DoWrite(thread, false); }
DoWrite(thread, false);
}
void DoWrite(ThreadState* thread, bool seq) { void DoWrite(ThreadState* thread, bool seq) {
if (num_ != FLAGS_num) { if (num_ != FLAGS_num) {
@ -875,13 +846,9 @@ class Benchmark {
} }
} }
void DeleteSeq(ThreadState* thread) { void DeleteSeq(ThreadState* thread) { DoDelete(thread, true); }
DoDelete(thread, true);
}
void DeleteRandom(ThreadState* thread) { void DeleteRandom(ThreadState* thread) { DoDelete(thread, false); }
DoDelete(thread, false);
}
void ReadWhileWriting(ThreadState* thread) { void ReadWhileWriting(ThreadState* thread) {
if (thread->tid > 0) { if (thread->tid > 0) {
@ -913,9 +880,7 @@ class Benchmark {
} }
} }
void Compact(ThreadState* thread) { void Compact(ThreadState* thread) { db_->CompactRange(nullptr, nullptr); }
db_->CompactRange(nullptr, nullptr);
}
void PrintStats(const char* key) { void PrintStats(const char* key) {
std::string stats; std::string stats;

View File

@ -2,9 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <sqlite3.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <sqlite3.h>
#include "util/histogram.h" #include "util/histogram.h"
#include "util/random.h" #include "util/random.h"
#include "util/testutil.h" #include "util/testutil.h"
@ -38,8 +39,7 @@ static const char* FLAGS_benchmarks =
"fillrand100K," "fillrand100K,"
"fillseq100K," "fillseq100K,"
"readseq," "readseq,"
"readrand100K," "readrand100K,";
;
// Number of key/values to place in database // Number of key/values to place in database
static int FLAGS_num = 1000000; static int FLAGS_num = 1000000;
@ -69,6 +69,9 @@ static int FLAGS_num_pages = 4096;
// benchmark will fail. // benchmark will fail.
static bool FLAGS_use_existing_db = false; static bool FLAGS_use_existing_db = false;
// If true, the SQLite table has ROWIDs.
static bool FLAGS_use_rowids = false;
// If true, we allow batch writes to occur // If true, we allow batch writes to occur
static bool FLAGS_transaction = true; static bool FLAGS_transaction = true;
@ -78,8 +81,7 @@ static bool FLAGS_WAL_enabled = true;
// Use the db with the following name. // Use the db with the following name.
static const char* FLAGS_db = nullptr; static const char* FLAGS_db = nullptr;
inline inline static void ExecErrorCheck(int status, char* err_msg) {
static void ExecErrorCheck(int status, char *err_msg) {
if (status != SQLITE_OK) { if (status != SQLITE_OK) {
fprintf(stderr, "SQL error: %s\n", err_msg); fprintf(stderr, "SQL error: %s\n", err_msg);
sqlite3_free(err_msg); sqlite3_free(err_msg);
@ -87,24 +89,21 @@ static void ExecErrorCheck(int status, char *err_msg) {
} }
} }
inline inline static void StepErrorCheck(int status) {
static void StepErrorCheck(int status) {
if (status != SQLITE_DONE) { if (status != SQLITE_DONE) {
fprintf(stderr, "SQL step error: status = %d\n", status); fprintf(stderr, "SQL step error: status = %d\n", status);
exit(1); exit(1);
} }
} }
inline inline static void ErrorCheck(int status) {
static void ErrorCheck(int status) {
if (status != SQLITE_OK) { if (status != SQLITE_OK) {
fprintf(stderr, "sqlite3 error: status = %d\n", status); fprintf(stderr, "sqlite3 error: status = %d\n", status);
exit(1); exit(1);
} }
} }
inline inline static void WalCheckpoint(sqlite3* db_) {
static void WalCheckpoint(sqlite3* db_) {
// Flush all writes to disk // Flush all writes to disk
if (FLAGS_WAL_enabled) { if (FLAGS_WAL_enabled) {
sqlite3_wal_checkpoint_v2(db_, nullptr, SQLITE_CHECKPOINT_FULL, nullptr, sqlite3_wal_checkpoint_v2(db_, nullptr, SQLITE_CHECKPOINT_FULL, nullptr,
@ -186,17 +185,17 @@ class Benchmark {
fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size); fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size);
fprintf(stdout, "Entries: %d\n", num_); fprintf(stdout, "Entries: %d\n", num_);
fprintf(stdout, "RawSize: %.1f MB (estimated)\n", fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
/ 1048576.0)); 1048576.0));
PrintWarnings(); PrintWarnings();
fprintf(stdout, "------------------------------------------------\n"); fprintf(stdout, "------------------------------------------------\n");
} }
void PrintWarnings() { void PrintWarnings() {
#if defined(__GNUC__) && !defined(__OPTIMIZE__) #if defined(__GNUC__) && !defined(__OPTIMIZE__)
fprintf(stdout, fprintf(
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" stdout,
); "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
#endif #endif
#ifndef NDEBUG #ifndef NDEBUG
fprintf(stdout, fprintf(stdout,
@ -262,13 +261,20 @@ class Benchmark {
done_++; done_++;
if (done_ >= next_report_) { if (done_ >= next_report_) {
if (next_report_ < 1000) next_report_ += 100; if (next_report_ < 1000)
else if (next_report_ < 5000) next_report_ += 500; next_report_ += 100;
else if (next_report_ < 10000) next_report_ += 1000; else if (next_report_ < 5000)
else if (next_report_ < 50000) next_report_ += 5000; next_report_ += 500;
else if (next_report_ < 100000) next_report_ += 10000; else if (next_report_ < 10000)
else if (next_report_ < 500000) next_report_ += 50000; next_report_ += 1000;
else next_report_ += 100000; else if (next_report_ < 50000)
next_report_ += 5000;
else if (next_report_ < 100000)
next_report_ += 10000;
else if (next_report_ < 500000)
next_report_ += 50000;
else
next_report_ += 100000;
fprintf(stderr, "... finished %d ops%30s\r", done_, ""); fprintf(stderr, "... finished %d ops%30s\r", done_, "");
fflush(stderr); fflush(stderr);
} }
@ -292,10 +298,8 @@ class Benchmark {
} }
} }
fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", name.ToString().c_str(),
name.ToString().c_str(), (finish - start_) * 1e6 / done_, (message_.empty() ? "" : " "),
(finish - start_) * 1e6 / done_,
(message_.empty() ? "" : " "),
message_.c_str()); message_.c_str());
if (FLAGS_histogram) { if (FLAGS_histogram) {
fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
@ -304,14 +308,8 @@ class Benchmark {
} }
public: public:
enum Order { enum Order { SEQUENTIAL, RANDOM };
SEQUENTIAL, enum DBState { FRESH, EXISTING };
RANDOM
};
enum DBState {
FRESH,
EXISTING
};
Benchmark() Benchmark()
: db_(nullptr), : db_(nullptr),
@ -426,10 +424,8 @@ class Benchmark {
// Open database // Open database
std::string tmp_dir; std::string tmp_dir;
Env::Default()->GetTestDirectory(&tmp_dir); Env::Default()->GetTestDirectory(&tmp_dir);
snprintf(file_name, sizeof(file_name), snprintf(file_name, sizeof(file_name), "%s/dbbench_sqlite3-%d.db",
"%s/dbbench_sqlite3-%d.db", tmp_dir.c_str(), db_num_);
tmp_dir.c_str(),
db_num_);
status = sqlite3_open(file_name, &db_); status = sqlite3_open(file_name, &db_);
if (status) { if (status) {
fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_)); fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_));
@ -460,8 +456,8 @@ class Benchmark {
std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096"; std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096";
status = sqlite3_exec(db_, WAL_stmt.c_str(), nullptr, nullptr, &err_msg); status = sqlite3_exec(db_, WAL_stmt.c_str(), nullptr, nullptr, &err_msg);
ExecErrorCheck(status, err_msg); ExecErrorCheck(status, err_msg);
status = sqlite3_exec(db_, WAL_checkpoint.c_str(), nullptr, nullptr, status =
&err_msg); sqlite3_exec(db_, WAL_checkpoint.c_str(), nullptr, nullptr, &err_msg);
ExecErrorCheck(status, err_msg); ExecErrorCheck(status, err_msg);
} }
@ -469,17 +465,18 @@ class Benchmark {
std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE"; std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE";
std::string create_stmt = std::string create_stmt =
"CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))"; "CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))";
if (!FLAGS_use_rowids) create_stmt += " WITHOUT ROWID";
std::string stmt_array[] = {locking_stmt, create_stmt}; std::string stmt_array[] = {locking_stmt, create_stmt};
int stmt_array_length = sizeof(stmt_array) / sizeof(std::string); int stmt_array_length = sizeof(stmt_array) / sizeof(std::string);
for (int i = 0; i < stmt_array_length; i++) { for (int i = 0; i < stmt_array_length; i++) {
status = sqlite3_exec(db_, stmt_array[i].c_str(), nullptr, nullptr, status =
&err_msg); sqlite3_exec(db_, stmt_array[i].c_str(), nullptr, nullptr, &err_msg);
ExecErrorCheck(status, err_msg); ExecErrorCheck(status, err_msg);
} }
} }
void Write(bool write_sync, Order order, DBState state, void Write(bool write_sync, Order order, DBState state, int num_entries,
int num_entries, int value_size, int entries_per_batch) { int value_size, int entries_per_batch) {
// Create new database if state == FRESH // Create new database if state == FRESH
if (state == FRESH) { if (state == FRESH) {
if (FLAGS_use_existing_db) { if (FLAGS_use_existing_db) {
@ -507,20 +504,20 @@ class Benchmark {
std::string end_trans_str = "END TRANSACTION;"; std::string end_trans_str = "END TRANSACTION;";
// Check for synchronous flag in options // Check for synchronous flag in options
std::string sync_stmt = (write_sync) ? "PRAGMA synchronous = FULL" : std::string sync_stmt =
"PRAGMA synchronous = OFF"; (write_sync) ? "PRAGMA synchronous = FULL" : "PRAGMA synchronous = OFF";
status = sqlite3_exec(db_, sync_stmt.c_str(), nullptr, nullptr, &err_msg); status = sqlite3_exec(db_, sync_stmt.c_str(), nullptr, nullptr, &err_msg);
ExecErrorCheck(status, err_msg); ExecErrorCheck(status, err_msg);
// Preparing sqlite3 statements // Preparing sqlite3 statements
status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1, status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1, &replace_stmt,
&replace_stmt, nullptr); nullptr);
ErrorCheck(status); ErrorCheck(status);
status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1, status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
&begin_trans_stmt, nullptr); &begin_trans_stmt, nullptr);
ErrorCheck(status); ErrorCheck(status);
status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt,
&end_trans_stmt, nullptr); nullptr);
ErrorCheck(status); ErrorCheck(status);
bool transaction = (entries_per_batch > 1); bool transaction = (entries_per_batch > 1);
@ -538,16 +535,16 @@ class Benchmark {
const char* value = gen_.Generate(value_size).data(); const char* value = gen_.Generate(value_size).data();
// Create values for key-value pair // Create values for key-value pair
const int k = (order == SEQUENTIAL) ? i + j : const int k =
(rand_.Next() % num_entries); (order == SEQUENTIAL) ? i + j : (rand_.Next() % num_entries);
char key[100]; char key[100];
snprintf(key, sizeof(key), "%016d", k); snprintf(key, sizeof(key), "%016d", k);
// Bind KV values into replace_stmt // Bind KV values into replace_stmt
status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC); status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC);
ErrorCheck(status); ErrorCheck(status);
status = sqlite3_bind_blob(replace_stmt, 2, value, status = sqlite3_bind_blob(replace_stmt, 2, value, value_size,
value_size, SQLITE_STATIC); SQLITE_STATIC);
ErrorCheck(status); ErrorCheck(status);
// Execute replace_stmt // Execute replace_stmt
@ -593,8 +590,8 @@ class Benchmark {
status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1, status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
&begin_trans_stmt, nullptr); &begin_trans_stmt, nullptr);
ErrorCheck(status); ErrorCheck(status);
status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt,
&end_trans_stmt, nullptr); nullptr);
ErrorCheck(status); ErrorCheck(status);
status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, nullptr); status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, nullptr);
ErrorCheck(status); ErrorCheck(status);
@ -621,7 +618,8 @@ class Benchmark {
ErrorCheck(status); ErrorCheck(status);
// Execute read statement // Execute read statement
while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {} while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {
}
StepErrorCheck(status); StepErrorCheck(status);
// Reset SQLite statement for another use // Reset SQLite statement for another use
@ -664,7 +662,6 @@ class Benchmark {
status = sqlite3_finalize(pStmt); status = sqlite3_finalize(pStmt);
ErrorCheck(status); ErrorCheck(status);
} }
}; };
} // namespace leveldb } // namespace leveldb
@ -685,6 +682,9 @@ int main(int argc, char** argv) {
} else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) { (n == 0 || n == 1)) {
FLAGS_use_existing_db = n; FLAGS_use_existing_db = n;
} else if (sscanf(argv[i], "--use_rowids=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
FLAGS_use_rowids = n;
} else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
FLAGS_num = n; FLAGS_num = n;
} else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {

View File

@ -2,9 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <kcpolydb.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <kcpolydb.h>
#include "util/histogram.h" #include "util/histogram.h"
#include "util/random.h" #include "util/random.h"
#include "util/testutil.h" #include "util/testutil.h"
@ -34,8 +35,7 @@ static const char* FLAGS_benchmarks =
"fillrand100K," "fillrand100K,"
"fillseq100K," "fillseq100K,"
"readseq100K," "readseq100K,"
"readrand100K," "readrand100K,";
;
// Number of key/values to place in database // Number of key/values to place in database
static int FLAGS_num = 1000000; static int FLAGS_num = 1000000;
@ -71,9 +71,7 @@ static bool FLAGS_compression = true;
// Use the db with the following name. // Use the db with the following name.
static const char* FLAGS_db = nullptr; static const char* FLAGS_db = nullptr;
inline inline static void DBSynchronize(kyotocabinet::TreeDB* db_) {
static void DBSynchronize(kyotocabinet::TreeDB* db_)
{
// Synchronize will flush writes to disk // Synchronize will flush writes to disk
if (!db_->synchronize()) { if (!db_->synchronize()) {
fprintf(stderr, "synchronize error: %s\n", db_->error().name()); fprintf(stderr, "synchronize error: %s\n", db_->error().name());
@ -157,20 +155,20 @@ class Benchmark {
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
fprintf(stdout, "Entries: %d\n", num_); fprintf(stdout, "Entries: %d\n", num_);
fprintf(stdout, "RawSize: %.1f MB (estimated)\n", fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
/ 1048576.0)); 1048576.0));
fprintf(stdout, "FileSize: %.1f MB (estimated)\n", fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) /
/ 1048576.0)); 1048576.0));
PrintWarnings(); PrintWarnings();
fprintf(stdout, "------------------------------------------------\n"); fprintf(stdout, "------------------------------------------------\n");
} }
void PrintWarnings() { void PrintWarnings() {
#if defined(__GNUC__) && !defined(__OPTIMIZE__) #if defined(__GNUC__) && !defined(__OPTIMIZE__)
fprintf(stdout, fprintf(
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" stdout,
); "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
#endif #endif
#ifndef NDEBUG #ifndef NDEBUG
fprintf(stdout, fprintf(stdout,
@ -237,13 +235,20 @@ class Benchmark {
done_++; done_++;
if (done_ >= next_report_) { if (done_ >= next_report_) {
if (next_report_ < 1000) next_report_ += 100; if (next_report_ < 1000)
else if (next_report_ < 5000) next_report_ += 500; next_report_ += 100;
else if (next_report_ < 10000) next_report_ += 1000; else if (next_report_ < 5000)
else if (next_report_ < 50000) next_report_ += 5000; next_report_ += 500;
else if (next_report_ < 100000) next_report_ += 10000; else if (next_report_ < 10000)
else if (next_report_ < 500000) next_report_ += 50000; next_report_ += 1000;
else next_report_ += 100000; else if (next_report_ < 50000)
next_report_ += 5000;
else if (next_report_ < 100000)
next_report_ += 10000;
else if (next_report_ < 500000)
next_report_ += 50000;
else
next_report_ += 100000;
fprintf(stderr, "... finished %d ops%30s\r", done_, ""); fprintf(stderr, "... finished %d ops%30s\r", done_, "");
fflush(stderr); fflush(stderr);
} }
@ -267,10 +272,8 @@ class Benchmark {
} }
} }
fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", name.ToString().c_str(),
name.ToString().c_str(), (finish - start_) * 1e6 / done_, (message_.empty() ? "" : " "),
(finish - start_) * 1e6 / done_,
(message_.empty() ? "" : " "),
message_.c_str()); message_.c_str());
if (FLAGS_histogram) { if (FLAGS_histogram) {
fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
@ -279,14 +282,8 @@ class Benchmark {
} }
public: public:
enum Order { enum Order { SEQUENTIAL, RANDOM };
SEQUENTIAL, enum DBState { FRESH, EXISTING };
RANDOM
};
enum DBState {
FRESH,
EXISTING
};
Benchmark() Benchmark()
: db_(nullptr), : db_(nullptr),
@ -395,16 +392,14 @@ class Benchmark {
db_num_++; db_num_++;
std::string test_dir; std::string test_dir;
Env::Default()->GetTestDirectory(&test_dir); Env::Default()->GetTestDirectory(&test_dir);
snprintf(file_name, sizeof(file_name), snprintf(file_name, sizeof(file_name), "%s/dbbench_polyDB-%d.kct",
"%s/dbbench_polyDB-%d.kct", test_dir.c_str(), db_num_);
test_dir.c_str(),
db_num_);
// Create tuning options and open the database // Create tuning options and open the database
int open_options = kyotocabinet::PolyDB::OWRITER | int open_options =
kyotocabinet::PolyDB::OCREATE; kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE;
int tune_options = kyotocabinet::TreeDB::TSMALL | int tune_options =
kyotocabinet::TreeDB::TLINEAR; kyotocabinet::TreeDB::TSMALL | kyotocabinet::TreeDB::TLINEAR;
if (FLAGS_compression) { if (FLAGS_compression) {
tune_options |= kyotocabinet::TreeDB::TCOMPRESS; tune_options |= kyotocabinet::TreeDB::TCOMPRESS;
db_->tune_compressor(&comp_); db_->tune_compressor(&comp_);
@ -421,8 +416,8 @@ class Benchmark {
} }
} }
void Write(bool sync, Order order, DBState state, void Write(bool sync, Order order, DBState state, int num_entries,
int num_entries, int value_size, int entries_per_batch) { int value_size, int entries_per_batch) {
// Create new database if state == FRESH // Create new database if state == FRESH
if (state == FRESH) { if (state == FRESH) {
if (FLAGS_use_existing_db) { if (FLAGS_use_existing_db) {
@ -442,8 +437,7 @@ class Benchmark {
} }
// Write to database // Write to database
for (int i = 0; i < num_entries; i++) for (int i = 0; i < num_entries; i++) {
{
const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % num_entries); const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % num_entries);
char key[100]; char key[100];
snprintf(key, sizeof(key), "%016d", k); snprintf(key, sizeof(key), "%016d", k);

View File

@ -1 +0,0 @@
include("${CMAKE_CURRENT_LIST_DIR}/leveldbTargets.cmake")

View File

@ -0,0 +1,9 @@
# Copyright 2019 The LevelDB Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file. See the AUTHORS file for names of contributors.
@PACKAGE_INIT@
include("${CMAKE_CURRENT_LIST_DIR}/leveldbTargets.cmake")
check_required_components(leveldb)

View File

@ -2,29 +2,24 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "leveldb/db.h" #include "gtest/gtest.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "leveldb/cache.h" #include "leveldb/cache.h"
#include "util/testharness.h" #include "leveldb/db.h"
#include "util/testutil.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
class AutoCompactTest { class AutoCompactTest : public testing::Test {
public: public:
std::string dbname_;
Cache* tiny_cache_;
Options options_;
DB* db_;
AutoCompactTest() { AutoCompactTest() {
dbname_ = test::TmpDir() + "/autocompact_test"; dbname_ = testing::TempDir() + "autocompact_test";
tiny_cache_ = NewLRUCache(100); tiny_cache_ = NewLRUCache(100);
options_.block_cache = tiny_cache_; options_.block_cache = tiny_cache_;
DestroyDB(dbname_, options_); DestroyDB(dbname_, options_);
options_.create_if_missing = true; options_.create_if_missing = true;
options_.compression = kNoCompression; options_.compression = kNoCompression;
ASSERT_OK(DB::Open(options_, dbname_, &db_)); EXPECT_LEVELDB_OK(DB::Open(options_, dbname_, &db_));
} }
~AutoCompactTest() { ~AutoCompactTest() {
@ -47,6 +42,12 @@ class AutoCompactTest {
} }
void DoReads(int n); void DoReads(int n);
private:
std::string dbname_;
Cache* tiny_cache_;
Options options_;
DB* db_;
}; };
static const int kValueSize = 200 * 1024; static const int kValueSize = 200 * 1024;
@ -61,15 +62,15 @@ void AutoCompactTest::DoReads(int n) {
// Fill database // Fill database
for (int i = 0; i < kCount; i++) { for (int i = 0; i < kCount; i++) {
ASSERT_OK(db_->Put(WriteOptions(), Key(i), value)); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), Key(i), value));
} }
ASSERT_OK(dbi->TEST_CompactMemTable()); ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable());
// Delete everything // Delete everything
for (int i = 0; i < kCount; i++) { for (int i = 0; i < kCount; i++) {
ASSERT_OK(db_->Delete(WriteOptions(), Key(i))); ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), Key(i)));
} }
ASSERT_OK(dbi->TEST_CompactMemTable()); ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable());
// Get initial measurement of the space we will be reading. // Get initial measurement of the space we will be reading.
const int64_t initial_size = Size(Key(0), Key(n)); const int64_t initial_size = Size(Key(0), Key(n));
@ -81,16 +82,15 @@ void AutoCompactTest::DoReads(int n) {
ASSERT_LT(read, 100) << "Taking too long to compact"; ASSERT_LT(read, 100) << "Taking too long to compact";
Iterator* iter = db_->NewIterator(ReadOptions()); Iterator* iter = db_->NewIterator(ReadOptions());
for (iter->SeekToFirst(); for (iter->SeekToFirst();
iter->Valid() && iter->key().ToString() < limit_key; iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) {
iter->Next()) {
// Drop data // Drop data
} }
delete iter; delete iter;
// Wait a little bit to allow any triggered compactions to complete. // Wait a little bit to allow any triggered compactions to complete.
Env::Default()->SleepForMicroseconds(1000000); Env::Default()->SleepForMicroseconds(1000000);
uint64_t size = Size(Key(0), Key(n)); uint64_t size = Size(Key(0), Key(n));
fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1,
read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0); size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0);
if (size <= initial_size / 10) { if (size <= initial_size / 10) {
break; break;
} }
@ -103,16 +103,13 @@ void AutoCompactTest::DoReads(int n) {
ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576); ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576);
} }
TEST(AutoCompactTest, ReadAll) { TEST_F(AutoCompactTest, ReadAll) { DoReads(kCount); }
DoReads(kCount);
}
TEST(AutoCompactTest, ReadHalf) { TEST_F(AutoCompactTest, ReadHalf) { DoReads(kCount / 2); }
DoReads(kCount/2);
}
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -4,8 +4,8 @@
#include "db/builder.h" #include "db/builder.h"
#include "db/filename.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/filename.h"
#include "db/table_cache.h" #include "db/table_cache.h"
#include "db/version_edit.h" #include "db/version_edit.h"
#include "leveldb/db.h" #include "leveldb/db.h"
@ -14,12 +14,8 @@
namespace leveldb { namespace leveldb {
Status BuildTable(const std::string& dbname, Status BuildTable(const std::string& dbname, Env* env, const Options& options,
Env* env, TableCache* table_cache, Iterator* iter, FileMetaData* meta) {
const Options& options,
TableCache* table_cache,
Iterator* iter,
FileMetaData* meta) {
Status s; Status s;
meta->file_size = 0; meta->file_size = 0;
iter->SeekToFirst(); iter->SeekToFirst();
@ -60,8 +56,7 @@ Status BuildTable(const std::string& dbname,
if (s.ok()) { if (s.ok()) {
// Verify that the table is usable // Verify that the table is usable
Iterator* it = table_cache->NewIterator(ReadOptions(), Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number,
meta->number,
meta->file_size); meta->file_size);
s = it->status(); s = it->status();
delete it; delete it;

View File

@ -22,12 +22,8 @@ class VersionEdit;
// *meta will be filled with metadata about the generated table. // *meta will be filled with metadata about the generated table.
// If no data is present in *iter, meta->file_size will be set to // If no data is present in *iter, meta->file_size will be set to
// zero, and no Table file will be produced. // zero, and no Table file will be produced.
Status BuildTable(const std::string& dbname, Status BuildTable(const std::string& dbname, Env* env, const Options& options,
Env* env, TableCache* table_cache, Iterator* iter, FileMetaData* meta);
const Options& options,
TableCache* table_cache,
Iterator* iter,
FileMetaData* meta);
} // namespace leveldb } // namespace leveldb

328
db/c.cc
View File

@ -4,7 +4,9 @@
#include "leveldb/c.h" #include "leveldb/c.h"
#include <stdlib.h> #include <cstdint>
#include <cstdlib>
#include "leveldb/cache.h" #include "leveldb/cache.h"
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
#include "leveldb/db.h" #include "leveldb/db.h"
@ -42,69 +44,72 @@ using leveldb::WriteOptions;
extern "C" { extern "C" {
struct leveldb_t { DB* rep; }; struct leveldb_t {
struct leveldb_iterator_t { Iterator* rep; }; DB* rep;
struct leveldb_writebatch_t { WriteBatch rep; }; };
struct leveldb_snapshot_t { const Snapshot* rep; }; struct leveldb_iterator_t {
struct leveldb_readoptions_t { ReadOptions rep; }; Iterator* rep;
struct leveldb_writeoptions_t { WriteOptions rep; }; };
struct leveldb_options_t { Options rep; }; struct leveldb_writebatch_t {
struct leveldb_cache_t { Cache* rep; }; WriteBatch rep;
struct leveldb_seqfile_t { SequentialFile* rep; }; };
struct leveldb_randomfile_t { RandomAccessFile* rep; }; struct leveldb_snapshot_t {
struct leveldb_writablefile_t { WritableFile* rep; }; const Snapshot* rep;
struct leveldb_logger_t { Logger* rep; }; };
struct leveldb_filelock_t { FileLock* rep; }; struct leveldb_readoptions_t {
ReadOptions rep;
};
struct leveldb_writeoptions_t {
WriteOptions rep;
};
struct leveldb_options_t {
Options rep;
};
struct leveldb_cache_t {
Cache* rep;
};
struct leveldb_seqfile_t {
SequentialFile* rep;
};
struct leveldb_randomfile_t {
RandomAccessFile* rep;
};
struct leveldb_writablefile_t {
WritableFile* rep;
};
struct leveldb_logger_t {
Logger* rep;
};
struct leveldb_filelock_t {
FileLock* rep;
};
struct leveldb_comparator_t : public Comparator { struct leveldb_comparator_t : public Comparator {
void* state_; ~leveldb_comparator_t() override { (*destructor_)(state_); }
void (*destructor_)(void*);
int (*compare_)(
void*,
const char* a, size_t alen,
const char* b, size_t blen);
const char* (*name_)(void*);
virtual ~leveldb_comparator_t() { int Compare(const Slice& a, const Slice& b) const override {
(*destructor_)(state_);
}
virtual int Compare(const Slice& a, const Slice& b) const {
return (*compare_)(state_, a.data(), a.size(), b.data(), b.size()); return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
} }
virtual const char* Name() const { const char* Name() const override { return (*name_)(state_); }
return (*name_)(state_);
}
// No-ops since the C binding does not support key shortening methods. // No-ops since the C binding does not support key shortening methods.
virtual void FindShortestSeparator(std::string*, const Slice&) const { } void FindShortestSeparator(std::string*, const Slice&) const override {}
virtual void FindShortSuccessor(std::string* key) const { } void FindShortSuccessor(std::string* key) const override {}
void* state_;
void (*destructor_)(void*);
int (*compare_)(void*, const char* a, size_t alen, const char* b,
size_t blen);
const char* (*name_)(void*);
}; };
struct leveldb_filterpolicy_t : public FilterPolicy { struct leveldb_filterpolicy_t : public FilterPolicy {
void* state_; ~leveldb_filterpolicy_t() override { (*destructor_)(state_); }
void (*destructor_)(void*);
const char* (*name_)(void*);
char* (*create_)(
void*,
const char* const* key_array, const size_t* key_length_array,
int num_keys,
size_t* filter_length);
unsigned char (*key_match_)(
void*,
const char* key, size_t length,
const char* filter, size_t filter_length);
virtual ~leveldb_filterpolicy_t() { const char* Name() const override { return (*name_)(state_); }
(*destructor_)(state_);
}
virtual const char* Name() const { void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
return (*name_)(state_);
}
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
std::vector<const char*> key_pointers(n); std::vector<const char*> key_pointers(n);
std::vector<size_t> key_sizes(n); std::vector<size_t> key_sizes(n);
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
@ -117,10 +122,19 @@ struct leveldb_filterpolicy_t : public FilterPolicy {
free(filter); free(filter);
} }
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
return (*key_match_)(state_, key.data(), key.size(), return (*key_match_)(state_, key.data(), key.size(), filter.data(),
filter.data(), filter.size()); filter.size());
} }
void* state_;
void (*destructor_)(void*);
const char* (*name_)(void*);
char* (*create_)(void*, const char* const* key_array,
const size_t* key_length_array, int num_keys,
size_t* filter_length);
uint8_t (*key_match_)(void*, const char* key, size_t length,
const char* filter, size_t filter_length);
}; };
struct leveldb_env_t { struct leveldb_env_t {
@ -148,9 +162,7 @@ static char* CopyString(const std::string& str) {
return result; return result;
} }
leveldb_t* leveldb_open( leveldb_t* leveldb_open(const leveldb_options_t* options, const char* name,
const leveldb_options_t* options,
const char* name,
char** errptr) { char** errptr) {
DB* db; DB* db;
if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) { if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
@ -166,38 +178,25 @@ void leveldb_close(leveldb_t* db) {
delete db; delete db;
} }
void leveldb_put( void leveldb_put(leveldb_t* db, const leveldb_writeoptions_t* options,
leveldb_t* db, const char* key, size_t keylen, const char* val, size_t vallen,
const leveldb_writeoptions_t* options,
const char* key, size_t keylen,
const char* val, size_t vallen,
char** errptr) { char** errptr) {
SaveError(errptr, SaveError(errptr,
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen))); db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
} }
void leveldb_delete( void leveldb_delete(leveldb_t* db, const leveldb_writeoptions_t* options,
leveldb_t* db, const char* key, size_t keylen, char** errptr) {
const leveldb_writeoptions_t* options,
const char* key, size_t keylen,
char** errptr) {
SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen))); SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
} }
void leveldb_write(leveldb_t* db, const leveldb_writeoptions_t* options,
void leveldb_write( leveldb_writebatch_t* batch, char** errptr) {
leveldb_t* db,
const leveldb_writeoptions_t* options,
leveldb_writebatch_t* batch,
char** errptr) {
SaveError(errptr, db->rep->Write(options->rep, &batch->rep)); SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
} }
char* leveldb_get( char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options,
leveldb_t* db, const char* key, size_t keylen, size_t* vallen,
const leveldb_readoptions_t* options,
const char* key, size_t keylen,
size_t* vallen,
char** errptr) { char** errptr) {
char* result = nullptr; char* result = nullptr;
std::string tmp; std::string tmp;
@ -215,30 +214,25 @@ char* leveldb_get(
} }
leveldb_iterator_t* leveldb_create_iterator( leveldb_iterator_t* leveldb_create_iterator(
leveldb_t* db, leveldb_t* db, const leveldb_readoptions_t* options) {
const leveldb_readoptions_t* options) {
leveldb_iterator_t* result = new leveldb_iterator_t; leveldb_iterator_t* result = new leveldb_iterator_t;
result->rep = db->rep->NewIterator(options->rep); result->rep = db->rep->NewIterator(options->rep);
return result; return result;
} }
const leveldb_snapshot_t* leveldb_create_snapshot( const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db) {
leveldb_t* db) {
leveldb_snapshot_t* result = new leveldb_snapshot_t; leveldb_snapshot_t* result = new leveldb_snapshot_t;
result->rep = db->rep->GetSnapshot(); result->rep = db->rep->GetSnapshot();
return result; return result;
} }
void leveldb_release_snapshot( void leveldb_release_snapshot(leveldb_t* db,
leveldb_t* db,
const leveldb_snapshot_t* snapshot) { const leveldb_snapshot_t* snapshot) {
db->rep->ReleaseSnapshot(snapshot->rep); db->rep->ReleaseSnapshot(snapshot->rep);
delete snapshot; delete snapshot;
} }
char* leveldb_property_value( char* leveldb_property_value(leveldb_t* db, const char* propname) {
leveldb_t* db,
const char* propname) {
std::string tmp; std::string tmp;
if (db->rep->GetProperty(Slice(propname), &tmp)) { if (db->rep->GetProperty(Slice(propname), &tmp)) {
// We use strdup() since we expect human readable output. // We use strdup() since we expect human readable output.
@ -248,11 +242,11 @@ char* leveldb_property_value(
} }
} }
void leveldb_approximate_sizes( void leveldb_approximate_sizes(leveldb_t* db, int num_ranges,
leveldb_t* db, const char* const* range_start_key,
int num_ranges, const size_t* range_start_key_len,
const char* const* range_start_key, const size_t* range_start_key_len, const char* const* range_limit_key,
const char* const* range_limit_key, const size_t* range_limit_key_len, const size_t* range_limit_key_len,
uint64_t* sizes) { uint64_t* sizes) {
Range* ranges = new Range[num_ranges]; Range* ranges = new Range[num_ranges];
for (int i = 0; i < num_ranges; i++) { for (int i = 0; i < num_ranges; i++) {
@ -263,10 +257,9 @@ void leveldb_approximate_sizes(
delete[] ranges; delete[] ranges;
} }
void leveldb_compact_range( void leveldb_compact_range(leveldb_t* db, const char* start_key,
leveldb_t* db, size_t start_key_len, const char* limit_key,
const char* start_key, size_t start_key_len, size_t limit_key_len) {
const char* limit_key, size_t limit_key_len) {
Slice a, b; Slice a, b;
db->rep->CompactRange( db->rep->CompactRange(
// Pass null Slice if corresponding "const char*" is null // Pass null Slice if corresponding "const char*" is null
@ -274,16 +267,12 @@ void leveldb_compact_range(
(limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
} }
void leveldb_destroy_db( void leveldb_destroy_db(const leveldb_options_t* options, const char* name,
const leveldb_options_t* options,
const char* name,
char** errptr) { char** errptr) {
SaveError(errptr, DestroyDB(name, options->rep)); SaveError(errptr, DestroyDB(name, options->rep));
} }
void leveldb_repair_db( void leveldb_repair_db(const leveldb_options_t* options, const char* name,
const leveldb_options_t* options,
const char* name,
char** errptr) { char** errptr) {
SaveError(errptr, RepairDB(name, options->rep)); SaveError(errptr, RepairDB(name, options->rep));
} }
@ -293,7 +282,7 @@ void leveldb_iter_destroy(leveldb_iterator_t* iter) {
delete iter; delete iter;
} }
unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) { uint8_t leveldb_iter_valid(const leveldb_iterator_t* iter) {
return iter->rep->Valid(); return iter->rep->Valid();
} }
@ -309,13 +298,9 @@ void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) {
iter->rep->Seek(Slice(k, klen)); iter->rep->Seek(Slice(k, klen));
} }
void leveldb_iter_next(leveldb_iterator_t* iter) { void leveldb_iter_next(leveldb_iterator_t* iter) { iter->rep->Next(); }
iter->rep->Next();
}
void leveldb_iter_prev(leveldb_iterator_t* iter) { void leveldb_iter_prev(leveldb_iterator_t* iter) { iter->rep->Prev(); }
iter->rep->Prev();
}
const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) { const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) {
Slice s = iter->rep->key(); Slice s = iter->rep->key();
@ -337,41 +322,34 @@ leveldb_writebatch_t* leveldb_writebatch_create() {
return new leveldb_writebatch_t; return new leveldb_writebatch_t;
} }
void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { delete b; }
delete b;
}
void leveldb_writebatch_clear(leveldb_writebatch_t* b) { void leveldb_writebatch_clear(leveldb_writebatch_t* b) { b->rep.Clear(); }
b->rep.Clear();
}
void leveldb_writebatch_put( void leveldb_writebatch_put(leveldb_writebatch_t* b, const char* key,
leveldb_writebatch_t* b, size_t klen, const char* val, size_t vlen) {
const char* key, size_t klen,
const char* val, size_t vlen) {
b->rep.Put(Slice(key, klen), Slice(val, vlen)); b->rep.Put(Slice(key, klen), Slice(val, vlen));
} }
void leveldb_writebatch_delete( void leveldb_writebatch_delete(leveldb_writebatch_t* b, const char* key,
leveldb_writebatch_t* b, size_t klen) {
const char* key, size_t klen) {
b->rep.Delete(Slice(key, klen)); b->rep.Delete(Slice(key, klen));
} }
void leveldb_writebatch_iterate( void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state,
leveldb_writebatch_t* b, void (*put)(void*, const char* k, size_t klen,
void* state, const char* v, size_t vlen),
void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), void (*deleted)(void*, const char* k,
void (*deleted)(void*, const char* k, size_t klen)) { size_t klen)) {
class H : public WriteBatch::Handler { class H : public WriteBatch::Handler {
public: public:
void* state_; void* state_;
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen); void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
void (*deleted_)(void*, const char* k, size_t klen); void (*deleted_)(void*, const char* k, size_t klen);
virtual void Put(const Slice& key, const Slice& value) { void Put(const Slice& key, const Slice& value) override {
(*put_)(state_, key.data(), key.size(), value.data(), value.size()); (*put_)(state_, key.data(), key.size(), value.data(), value.size());
} }
virtual void Delete(const Slice& key) { void Delete(const Slice& key) override {
(*deleted_)(state_, key.data(), key.size()); (*deleted_)(state_, key.data(), key.size());
} }
}; };
@ -382,38 +360,34 @@ void leveldb_writebatch_iterate(
b->rep.Iterate(&handler); b->rep.Iterate(&handler);
} }
leveldb_options_t* leveldb_options_create() { void leveldb_writebatch_append(leveldb_writebatch_t* destination,
return new leveldb_options_t; const leveldb_writebatch_t* source) {
destination->rep.Append(source->rep);
} }
void leveldb_options_destroy(leveldb_options_t* options) { leveldb_options_t* leveldb_options_create() { return new leveldb_options_t; }
delete options;
}
void leveldb_options_set_comparator( void leveldb_options_destroy(leveldb_options_t* options) { delete options; }
leveldb_options_t* opt,
void leveldb_options_set_comparator(leveldb_options_t* opt,
leveldb_comparator_t* cmp) { leveldb_comparator_t* cmp) {
opt->rep.comparator = cmp; opt->rep.comparator = cmp;
} }
void leveldb_options_set_filter_policy( void leveldb_options_set_filter_policy(leveldb_options_t* opt,
leveldb_options_t* opt,
leveldb_filterpolicy_t* policy) { leveldb_filterpolicy_t* policy) {
opt->rep.filter_policy = policy; opt->rep.filter_policy = policy;
} }
void leveldb_options_set_create_if_missing( void leveldb_options_set_create_if_missing(leveldb_options_t* opt, uint8_t v) {
leveldb_options_t* opt, unsigned char v) {
opt->rep.create_if_missing = v; opt->rep.create_if_missing = v;
} }
void leveldb_options_set_error_if_exists( void leveldb_options_set_error_if_exists(leveldb_options_t* opt, uint8_t v) {
leveldb_options_t* opt, unsigned char v) {
opt->rep.error_if_exists = v; opt->rep.error_if_exists = v;
} }
void leveldb_options_set_paranoid_checks( void leveldb_options_set_paranoid_checks(leveldb_options_t* opt, uint8_t v) {
leveldb_options_t* opt, unsigned char v) {
opt->rep.paranoid_checks = v; opt->rep.paranoid_checks = v;
} }
@ -454,12 +428,9 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
} }
leveldb_comparator_t* leveldb_comparator_create( leveldb_comparator_t* leveldb_comparator_create(
void* state, void* state, void (*destructor)(void*),
void (*destructor)(void*), int (*compare)(void*, const char* a, size_t alen, const char* b,
int (*compare)( size_t blen),
void*,
const char* a, size_t alen,
const char* b, size_t blen),
const char* (*name)(void*)) { const char* (*name)(void*)) {
leveldb_comparator_t* result = new leveldb_comparator_t; leveldb_comparator_t* result = new leveldb_comparator_t;
result->state_ = state; result->state_ = state;
@ -469,21 +440,14 @@ leveldb_comparator_t* leveldb_comparator_create(
return result; return result;
} }
void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { delete cmp; }
delete cmp;
}
leveldb_filterpolicy_t* leveldb_filterpolicy_create( leveldb_filterpolicy_t* leveldb_filterpolicy_create(
void* state, void* state, void (*destructor)(void*),
void (*destructor)(void*), char* (*create_filter)(void*, const char* const* key_array,
char* (*create_filter)( const size_t* key_length_array, int num_keys,
void*,
const char* const* key_array, const size_t* key_length_array,
int num_keys,
size_t* filter_length), size_t* filter_length),
unsigned char (*key_may_match)( uint8_t (*key_may_match)(void*, const char* key, size_t length,
void*,
const char* key, size_t length,
const char* filter, size_t filter_length), const char* filter, size_t filter_length),
const char* (*name)(void*)) { const char* (*name)(void*)) {
leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t; leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t;
@ -504,7 +468,8 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
// they delegate to a NewBloomFilterPolicy() instead of user // they delegate to a NewBloomFilterPolicy() instead of user
// supplied C functions. // supplied C functions.
struct Wrapper : public leveldb_filterpolicy_t { struct Wrapper : public leveldb_filterpolicy_t {
const FilterPolicy* rep_; static void DoNothing(void*) {}
~Wrapper() { delete rep_; } ~Wrapper() { delete rep_; }
const char* Name() const { return rep_->Name(); } const char* Name() const { return rep_->Name(); }
void CreateFilter(const Slice* keys, int n, std::string* dst) const { void CreateFilter(const Slice* keys, int n, std::string* dst) const {
@ -513,7 +478,8 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
bool KeyMayMatch(const Slice& key, const Slice& filter) const { bool KeyMayMatch(const Slice& key, const Slice& filter) const {
return rep_->KeyMayMatch(key, filter); return rep_->KeyMayMatch(key, filter);
} }
static void DoNothing(void*) { }
const FilterPolicy* rep_;
}; };
Wrapper* wrapper = new Wrapper; Wrapper* wrapper = new Wrapper;
wrapper->rep_ = NewBloomFilterPolicy(bits_per_key); wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
@ -526,23 +492,18 @@ leveldb_readoptions_t* leveldb_readoptions_create() {
return new leveldb_readoptions_t; return new leveldb_readoptions_t;
} }
void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { delete opt; }
delete opt;
}
void leveldb_readoptions_set_verify_checksums( void leveldb_readoptions_set_verify_checksums(leveldb_readoptions_t* opt,
leveldb_readoptions_t* opt, uint8_t v) {
unsigned char v) {
opt->rep.verify_checksums = v; opt->rep.verify_checksums = v;
} }
void leveldb_readoptions_set_fill_cache( void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t* opt, uint8_t v) {
leveldb_readoptions_t* opt, unsigned char v) {
opt->rep.fill_cache = v; opt->rep.fill_cache = v;
} }
void leveldb_readoptions_set_snapshot( void leveldb_readoptions_set_snapshot(leveldb_readoptions_t* opt,
leveldb_readoptions_t* opt,
const leveldb_snapshot_t* snap) { const leveldb_snapshot_t* snap) {
opt->rep.snapshot = (snap ? snap->rep : nullptr); opt->rep.snapshot = (snap ? snap->rep : nullptr);
} }
@ -551,12 +512,9 @@ leveldb_writeoptions_t* leveldb_writeoptions_create() {
return new leveldb_writeoptions_t; return new leveldb_writeoptions_t;
} }
void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { delete opt; }
delete opt;
}
void leveldb_writeoptions_set_sync( void leveldb_writeoptions_set_sync(leveldb_writeoptions_t* opt, uint8_t v) {
leveldb_writeoptions_t* opt, unsigned char v) {
opt->rep.sync = v; opt->rep.sync = v;
} }
@ -595,16 +553,10 @@ char* leveldb_env_get_test_directory(leveldb_env_t* env) {
return buffer; return buffer;
} }
void leveldb_free(void* ptr) { void leveldb_free(void* ptr) { free(ptr); }
free(ptr);
}
int leveldb_major_version() { int leveldb_major_version() { return kMajorVersion; }
return kMajorVersion;
}
int leveldb_minor_version() { int leveldb_minor_version() { return kMinorVersion; }
return kMinorVersion;
}
} // end extern "C" } // end extern "C"

View File

@ -120,7 +120,7 @@ static const char* CmpName(void* arg) {
} }
// Custom filter policy // Custom filter policy
static unsigned char fake_filter_result = 1; static uint8_t fake_filter_result = 1;
static void FilterDestroy(void* arg) { } static void FilterDestroy(void* arg) { }
static const char* FilterName(void* arg) { static const char* FilterName(void* arg) {
return "TestFilter"; return "TestFilter";
@ -135,9 +135,7 @@ static char* FilterCreate(
memcpy(result, "fake", 4); memcpy(result, "fake", 4);
return result; return result;
} }
unsigned char FilterKeyMatch( uint8_t FilterKeyMatch(void* arg, const char* key, size_t length,
void* arg,
const char* key, size_t length,
const char* filter, size_t filter_length) { const char* filter, size_t filter_length) {
CheckCondition(filter_length == 4); CheckCondition(filter_length == 4);
CheckCondition(memcmp(filter, "fake", 4) == 0); CheckCondition(memcmp(filter, "fake", 4) == 0);
@ -228,12 +226,18 @@ int main(int argc, char** argv) {
leveldb_writebatch_clear(wb); leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_writebatch_t* wb2 = leveldb_writebatch_create();
leveldb_writebatch_delete(wb2, "bar", 3);
leveldb_writebatch_append(wb, wb2);
leveldb_writebatch_destroy(wb2);
leveldb_write(db, woptions, wb, &err); leveldb_write(db, woptions, wb, &err);
CheckNoError(err); CheckNoError(err);
CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c"); CheckGet(db, roptions, "box", "c");
int pos = 0; int pos = 0;
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3); CheckCondition(pos == 3);

View File

@ -2,44 +2,34 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "leveldb/db.h"
#include <errno.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include "leveldb/cache.h"
#include "leveldb/env.h" #include "gtest/gtest.h"
#include "leveldb/table.h"
#include "leveldb/write_batch.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/filename.h" #include "db/filename.h"
#include "db/log_format.h" #include "db/log_format.h"
#include "db/version_set.h" #include "db/version_set.h"
#include "leveldb/cache.h"
#include "leveldb/db.h"
#include "leveldb/table.h"
#include "leveldb/write_batch.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
static const int kValueSize = 1000; static const int kValueSize = 1000;
class CorruptionTest { class CorruptionTest : public testing::Test {
public: public:
test::ErrorEnv env_; CorruptionTest()
std::string dbname_; : db_(nullptr),
Cache* tiny_cache_; dbname_("/memenv/corruption_test"),
Options options_; tiny_cache_(NewLRUCache(100)) {
DB* db_;
CorruptionTest() {
tiny_cache_ = NewLRUCache(100);
options_.env = &env_; options_.env = &env_;
options_.block_cache = tiny_cache_; options_.block_cache = tiny_cache_;
dbname_ = test::TmpDir() + "/corruption_test";
DestroyDB(dbname_, options_); DestroyDB(dbname_, options_);
db_ = nullptr;
options_.create_if_missing = true; options_.create_if_missing = true;
Reopen(); Reopen();
options_.create_if_missing = false; options_.create_if_missing = false;
@ -47,7 +37,6 @@ class CorruptionTest {
~CorruptionTest() { ~CorruptionTest() {
delete db_; delete db_;
DestroyDB(dbname_, Options());
delete tiny_cache_; delete tiny_cache_;
} }
@ -57,14 +46,12 @@ class CorruptionTest {
return DB::Open(options_, dbname_, &db_); return DB::Open(options_, dbname_, &db_);
} }
void Reopen() { void Reopen() { ASSERT_LEVELDB_OK(TryReopen()); }
ASSERT_OK(TryReopen());
}
void RepairDB() { void RepairDB() {
delete db_; delete db_;
db_ = nullptr; db_ = nullptr;
ASSERT_OK(::leveldb::RepairDB(dbname_, options_)); ASSERT_LEVELDB_OK(::leveldb::RepairDB(dbname_, options_));
} }
void Build(int n) { void Build(int n) {
@ -81,7 +68,7 @@ class CorruptionTest {
if (i == n - 1) { if (i == n - 1) {
options.sync = true; options.sync = true;
} }
ASSERT_OK(db_->Write(options, &batch)); ASSERT_LEVELDB_OK(db_->Write(options, &batch));
} }
} }
@ -100,8 +87,7 @@ class CorruptionTest {
// Ignore boundary keys. // Ignore boundary keys.
continue; continue;
} }
if (!ConsumeDecimalNumber(&in, &key) || if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
!in.empty() ||
key < next_expected) { key < next_expected) {
bad_keys++; bad_keys++;
continue; continue;
@ -126,14 +112,13 @@ class CorruptionTest {
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
// Pick file to corrupt // Pick file to corrupt
std::vector<std::string> filenames; std::vector<std::string> filenames;
ASSERT_OK(env_.GetChildren(dbname_, &filenames)); ASSERT_LEVELDB_OK(env_.target()->GetChildren(dbname_, &filenames));
uint64_t number; uint64_t number;
FileType type; FileType type;
std::string fname; std::string fname;
int picked_number = -1; int picked_number = -1;
for (size_t i = 0; i < filenames.size(); i++) { for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) && if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
type == filetype &&
int(number) > picked_number) { // Pick latest file int(number) > picked_number) { // Pick latest file
fname = dbname_ + "/" + filenames[i]; fname = dbname_ + "/" + filenames[i];
picked_number = number; picked_number = number;
@ -141,35 +126,32 @@ class CorruptionTest {
} }
ASSERT_TRUE(!fname.empty()) << filetype; ASSERT_TRUE(!fname.empty()) << filetype;
struct stat sbuf; uint64_t file_size;
if (stat(fname.c_str(), &sbuf) != 0) { ASSERT_LEVELDB_OK(env_.target()->GetFileSize(fname, &file_size));
const char* msg = strerror(errno);
ASSERT_TRUE(false) << fname << ": " << msg;
}
if (offset < 0) { if (offset < 0) {
// Relative to end of file; make it absolute // Relative to end of file; make it absolute
if (-offset > sbuf.st_size) { if (-offset > file_size) {
offset = 0; offset = 0;
} else { } else {
offset = sbuf.st_size + offset; offset = file_size + offset;
} }
} }
if (offset > sbuf.st_size) { if (offset > file_size) {
offset = sbuf.st_size; offset = file_size;
} }
if (offset + bytes_to_corrupt > sbuf.st_size) { if (offset + bytes_to_corrupt > file_size) {
bytes_to_corrupt = sbuf.st_size - offset; bytes_to_corrupt = file_size - offset;
} }
// Do it // Do it
std::string contents; std::string contents;
Status s = ReadFileToString(Env::Default(), fname, &contents); Status s = ReadFileToString(env_.target(), fname, &contents);
ASSERT_TRUE(s.ok()) << s.ToString(); ASSERT_TRUE(s.ok()) << s.ToString();
for (int i = 0; i < bytes_to_corrupt; i++) { for (int i = 0; i < bytes_to_corrupt; i++) {
contents[i + offset] ^= 0x80; contents[i + offset] ^= 0x80;
} }
s = WriteStringToFile(Env::Default(), contents, fname); s = WriteStringToFile(env_.target(), contents, fname);
ASSERT_TRUE(s.ok()) << s.ToString(); ASSERT_TRUE(s.ok()) << s.ToString();
} }
@ -197,9 +179,17 @@ class CorruptionTest {
Random r(k); Random r(k);
return test::RandomString(&r, kValueSize, storage); return test::RandomString(&r, kValueSize, storage);
} }
test::ErrorEnv env_;
Options options_;
DB* db_;
private:
std::string dbname_;
Cache* tiny_cache_;
}; };
TEST(CorruptionTest, Recovery) { TEST_F(CorruptionTest, Recovery) {
Build(100); Build(100);
Check(100, 100); Check(100, 100);
Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
@ -210,13 +200,13 @@ TEST(CorruptionTest, Recovery) {
Check(36, 36); Check(36, 36);
} }
TEST(CorruptionTest, RecoverWriteError) { TEST_F(CorruptionTest, RecoverWriteError) {
env_.writable_file_error_ = true; env_.writable_file_error_ = true;
Status s = TryReopen(); Status s = TryReopen();
ASSERT_TRUE(!s.ok()); ASSERT_TRUE(!s.ok());
} }
TEST(CorruptionTest, NewFileErrorDuringWrite) { TEST_F(CorruptionTest, NewFileErrorDuringWrite) {
// Do enough writing to force minor compaction // Do enough writing to force minor compaction
env_.writable_file_error_ = true; env_.writable_file_error_ = true;
const int num = 3 + (Options().write_buffer_size / kValueSize); const int num = 3 + (Options().write_buffer_size / kValueSize);
@ -233,7 +223,7 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) {
Reopen(); Reopen();
} }
TEST(CorruptionTest, TableFile) { TEST_F(CorruptionTest, TableFile) {
Build(100); Build(100);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable(); dbi->TEST_CompactMemTable();
@ -244,7 +234,7 @@ TEST(CorruptionTest, TableFile) {
Check(90, 99); Check(90, 99);
} }
TEST(CorruptionTest, TableFileRepair) { TEST_F(CorruptionTest, TableFileRepair) {
options_.block_size = 2 * kValueSize; // Limit scope of corruption options_.block_size = 2 * kValueSize; // Limit scope of corruption
options_.paranoid_checks = true; options_.paranoid_checks = true;
Reopen(); Reopen();
@ -260,7 +250,7 @@ TEST(CorruptionTest, TableFileRepair) {
Check(95, 99); Check(95, 99);
} }
TEST(CorruptionTest, TableFileIndexData) { TEST_F(CorruptionTest, TableFileIndexData) {
Build(10000); // Enough to build multiple Tables Build(10000); // Enough to build multiple Tables
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable(); dbi->TEST_CompactMemTable();
@ -270,36 +260,36 @@ TEST(CorruptionTest, TableFileIndexData) {
Check(5000, 9999); Check(5000, 9999);
} }
TEST(CorruptionTest, MissingDescriptor) { TEST_F(CorruptionTest, MissingDescriptor) {
Build(1000); Build(1000);
RepairDB(); RepairDB();
Reopen(); Reopen();
Check(1000, 1000); Check(1000, 1000);
} }
TEST(CorruptionTest, SequenceNumberRecovery) { TEST_F(CorruptionTest, SequenceNumberRecovery) {
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1")); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v1"));
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2")); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v2"));
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3")); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v3"));
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4")); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v4"));
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5")); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v5"));
RepairDB(); RepairDB();
Reopen(); Reopen();
std::string v; std::string v;
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
ASSERT_EQ("v5", v); ASSERT_EQ("v5", v);
// Write something. If sequence number was not recovered properly, // Write something. If sequence number was not recovered properly,
// it will be hidden by an earlier write. // it will be hidden by an earlier write.
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6")); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v6"));
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
ASSERT_EQ("v6", v); ASSERT_EQ("v6", v);
Reopen(); Reopen();
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
ASSERT_EQ("v6", v); ASSERT_EQ("v6", v);
} }
TEST(CorruptionTest, CorruptedDescriptor) { TEST_F(CorruptionTest, CorruptedDescriptor) {
ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello")); ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "hello"));
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable(); dbi->TEST_CompactMemTable();
dbi->TEST_CompactRange(0, nullptr, nullptr); dbi->TEST_CompactRange(0, nullptr, nullptr);
@ -311,11 +301,11 @@ TEST(CorruptionTest, CorruptedDescriptor) {
RepairDB(); RepairDB();
Reopen(); Reopen();
std::string v; std::string v;
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
ASSERT_EQ("hello", v); ASSERT_EQ("hello", v);
} }
TEST(CorruptionTest, CompactionInputError) { TEST_F(CorruptionTest, CompactionInputError) {
Build(10); Build(10);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable(); dbi->TEST_CompactMemTable();
@ -330,7 +320,7 @@ TEST(CorruptionTest, CompactionInputError) {
Check(10000, 10000); Check(10000, 10000);
} }
TEST(CorruptionTest, CompactionInputErrorParanoid) { TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
options_.paranoid_checks = true; options_.paranoid_checks = true;
options_.write_buffer_size = 512 << 10; options_.write_buffer_size = 512 << 10;
Reopen(); Reopen();
@ -351,24 +341,26 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db"; ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
} }
TEST(CorruptionTest, UnrelatedKeys) { TEST_F(CorruptionTest, UnrelatedKeys) {
Build(10); Build(10);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable(); dbi->TEST_CompactMemTable();
Corrupt(kTableFile, 100, 1); Corrupt(kTableFile, 100, 1);
std::string tmp1, tmp2; std::string tmp1, tmp2;
ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2))); ASSERT_LEVELDB_OK(
db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
std::string v; std::string v;
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
ASSERT_EQ(Value(1000, &tmp2).ToString(), v); ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
dbi->TEST_CompactMemTable(); dbi->TEST_CompactMemTable();
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
ASSERT_EQ(Value(1000, &tmp2).ToString(), v); ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
} }
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -8,6 +8,7 @@
#include <stdio.h> #include <stdio.h>
#include <algorithm> #include <algorithm>
#include <atomic>
#include <set> #include <set>
#include <string> #include <string>
#include <vector> #include <vector>
@ -41,16 +42,33 @@ const int kNumNonTableCacheFiles = 10;
// Information kept for every waiting writer // Information kept for every waiting writer
struct DBImpl::Writer { struct DBImpl::Writer {
explicit Writer(port::Mutex* mu)
: batch(nullptr), sync(false), done(false), cv(mu) {}
Status status; Status status;
WriteBatch* batch; WriteBatch* batch;
bool sync; bool sync;
bool done; bool done;
port::CondVar cv; port::CondVar cv;
explicit Writer(port::Mutex* mu) : cv(mu) { }
}; };
struct DBImpl::CompactionState { struct DBImpl::CompactionState {
// Files produced by compaction
struct Output {
uint64_t number;
uint64_t file_size;
InternalKey smallest, largest;
};
Output* current_output() { return &outputs[outputs.size() - 1]; }
explicit CompactionState(Compaction* c)
: compaction(c),
smallest_snapshot(0),
outfile(nullptr),
builder(nullptr),
total_bytes(0) {}
Compaction* const compaction; Compaction* const compaction;
// Sequence numbers < smallest_snapshot are not significant since we // Sequence numbers < smallest_snapshot are not significant since we
@ -59,12 +77,6 @@ struct DBImpl::CompactionState {
// we can drop all entries for the same key with sequence numbers < S. // we can drop all entries for the same key with sequence numbers < S.
SequenceNumber smallest_snapshot; SequenceNumber smallest_snapshot;
// Files produced by compaction
struct Output {
uint64_t number;
uint64_t file_size;
InternalKey smallest, largest;
};
std::vector<Output> outputs; std::vector<Output> outputs;
// State kept for output being generated // State kept for output being generated
@ -72,15 +84,6 @@ struct DBImpl::CompactionState {
TableBuilder* builder; TableBuilder* builder;
uint64_t total_bytes; uint64_t total_bytes;
Output* current_output() { return &outputs[outputs.size()-1]; }
explicit CompactionState(Compaction* c)
: compaction(c),
outfile(nullptr),
builder(nullptr),
total_bytes(0) {
}
}; };
// Fix user-supplied options to be reasonable // Fix user-supplied options to be reasonable
@ -132,10 +135,11 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname)
dbname_(dbname), dbname_(dbname),
table_cache_(new TableCache(dbname_, options_, TableCacheSize(options_))), table_cache_(new TableCache(dbname_, options_, TableCacheSize(options_))),
db_lock_(nullptr), db_lock_(nullptr),
shutting_down_(nullptr), shutting_down_(false),
background_work_finished_signal_(&mutex_), background_work_finished_signal_(&mutex_),
mem_(nullptr), mem_(nullptr),
imm_(nullptr), imm_(nullptr),
has_imm_(false),
logfile_(nullptr), logfile_(nullptr),
logfile_number_(0), logfile_number_(0),
log_(nullptr), log_(nullptr),
@ -144,14 +148,12 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname)
background_compaction_scheduled_(false), background_compaction_scheduled_(false),
manual_compaction_(nullptr), manual_compaction_(nullptr),
versions_(new VersionSet(dbname_, &options_, table_cache_, versions_(new VersionSet(dbname_, &options_, table_cache_,
&internal_comparator_)) { &internal_comparator_)) {}
has_imm_.Release_Store(nullptr);
}
DBImpl::~DBImpl() { DBImpl::~DBImpl() {
// Wait for background work to finish // Wait for background work to finish.
mutex_.Lock(); mutex_.Lock();
shutting_down_.Release_Store(this); // Any non-null value is ok shutting_down_.store(true, std::memory_order_release);
while (background_compaction_scheduled_) { while (background_compaction_scheduled_) {
background_work_finished_signal_.Wait(); background_work_finished_signal_.Wait();
} }
@ -235,8 +237,9 @@ void DBImpl::DeleteObsoleteFiles() {
env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose
uint64_t number; uint64_t number;
FileType type; FileType type;
for (size_t i = 0; i < filenames.size(); i++) { std::vector<std::string> files_to_delete;
if (ParseFileName(filenames[i], &number, &type)) { for (std::string& filename : filenames) {
if (ParseFileName(filename, &number, &type)) {
bool keep = true; bool keep = true;
switch (type) { switch (type) {
case kLogFile: case kLogFile:
@ -264,16 +267,24 @@ void DBImpl::DeleteObsoleteFiles() {
} }
if (!keep) { if (!keep) {
files_to_delete.push_back(std::move(filename));
if (type == kTableFile) { if (type == kTableFile) {
table_cache_->Evict(number); table_cache_->Evict(number);
} }
Log(options_.info_log, "Delete type=%d #%lld\n", Log(options_.info_log, "Delete type=%d #%lld\n", static_cast<int>(type),
static_cast<int>(type),
static_cast<unsigned long long>(number)); static_cast<unsigned long long>(number));
env_->DeleteFile(dbname_ + "/" + filenames[i]);
} }
} }
} }
// While deleting all files unblock other threads. All files being deleted
// have unique names which will not collide with newly created files and
// are therefore safe to delete while allowing other threads to proceed.
mutex_.Unlock();
for (const std::string& filename : files_to_delete) {
env_->DeleteFile(dbname_ + "/" + filename);
}
mutex_.Lock();
} }
Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) { Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) {
@ -301,8 +312,8 @@ Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) {
} }
} else { } else {
if (options_.error_if_exists) { if (options_.error_if_exists) {
return Status::InvalidArgument( return Status::InvalidArgument(dbname_,
dbname_, "exists (error_if_exists is true)"); "exists (error_if_exists is true)");
} }
} }
@ -375,10 +386,10 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
Logger* info_log; Logger* info_log;
const char* fname; const char* fname;
Status* status; // null if options_.paranoid_checks==false Status* status; // null if options_.paranoid_checks==false
virtual void Corruption(size_t bytes, const Status& s) { void Corruption(size_t bytes, const Status& s) override {
Log(info_log, "%s%s: dropping %d bytes; %s", Log(info_log, "%s%s: dropping %d bytes; %s",
(this->status == nullptr ? "(ignoring error) " : ""), (this->status == nullptr ? "(ignoring error) " : ""), fname,
fname, static_cast<int>(bytes), s.ToString().c_str()); static_cast<int>(bytes), s.ToString().c_str());
if (this->status != nullptr && this->status->ok()) *this->status = s; if (this->status != nullptr && this->status->ok()) *this->status = s;
} }
}; };
@ -404,8 +415,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
// paranoid_checks==false so that corruptions cause entire commits // paranoid_checks==false so that corruptions cause entire commits
// to be skipped instead of propagating bad information (like overly // to be skipped instead of propagating bad information (like overly
// large sequence numbers). // large sequence numbers).
log::Reader reader(file, &reporter, true/*checksum*/, log::Reader reader(file, &reporter, true /*checksum*/, 0 /*initial_offset*/);
0/*initial_offset*/);
Log(options_.info_log, "Recovering log #%llu", Log(options_.info_log, "Recovering log #%llu",
(unsigned long long)log_number); (unsigned long long)log_number);
@ -415,11 +425,10 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
WriteBatch batch; WriteBatch batch;
int compactions = 0; int compactions = 0;
MemTable* mem = nullptr; MemTable* mem = nullptr;
while (reader.ReadRecord(&record, &scratch) && while (reader.ReadRecord(&record, &scratch) && status.ok()) {
status.ok()) {
if (record.size() < 12) { if (record.size() < 12) {
reporter.Corruption( reporter.Corruption(record.size(),
record.size(), Status::Corruption("log record too small")); Status::Corruption("log record too small"));
continue; continue;
} }
WriteBatchInternal::SetContents(&batch, record); WriteBatchInternal::SetContents(&batch, record);
@ -433,8 +442,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
if (!status.ok()) { if (!status.ok()) {
break; break;
} }
const SequenceNumber last_seq = const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) +
WriteBatchInternal::Sequence(&batch) +
WriteBatchInternal::Count(&batch) - 1; WriteBatchInternal::Count(&batch) - 1;
if (last_seq > *max_sequence) { if (last_seq > *max_sequence) {
*max_sequence = last_seq; *max_sequence = last_seq;
@ -509,13 +517,11 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
} }
Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s", Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s",
(unsigned long long) meta.number, (unsigned long long)meta.number, (unsigned long long)meta.file_size,
(unsigned long long) meta.file_size,
s.ToString().c_str()); s.ToString().c_str());
delete iter; delete iter;
pending_outputs_.erase(meta.number); pending_outputs_.erase(meta.number);
// Note that if file_size is zero, the file has been deleted and // Note that if file_size is zero, the file has been deleted and
// should not be added to the manifest. // should not be added to the manifest.
int level = 0; int level = 0;
@ -525,8 +531,8 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
if (base != nullptr) { if (base != nullptr) {
level = base->PickLevelForMemTableOutput(min_user_key, max_user_key); level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
} }
edit->AddFile(level, meta.number, meta.file_size, edit->AddFile(level, meta.number, meta.file_size, meta.smallest,
meta.smallest, meta.largest); meta.largest);
} }
CompactionStats stats; CompactionStats stats;
@ -547,7 +553,7 @@ void DBImpl::CompactMemTable() {
Status s = WriteLevel0Table(imm_, &edit, base); Status s = WriteLevel0Table(imm_, &edit, base);
base->Unref(); base->Unref();
if (s.ok() && shutting_down_.Acquire_Load()) { if (s.ok() && shutting_down_.load(std::memory_order_acquire)) {
s = Status::IOError("Deleting DB during memtable compaction"); s = Status::IOError("Deleting DB during memtable compaction");
} }
@ -562,7 +568,7 @@ void DBImpl::CompactMemTable() {
// Commit to the new state // Commit to the new state
imm_->Unref(); imm_->Unref();
imm_ = nullptr; imm_ = nullptr;
has_imm_.Release_Store(nullptr); has_imm_.store(false, std::memory_order_release);
DeleteObsoleteFiles(); DeleteObsoleteFiles();
} else { } else {
RecordBackgroundError(s); RecordBackgroundError(s);
@ -610,7 +616,8 @@ void DBImpl::TEST_CompactRange(int level, const Slice* begin,
} }
MutexLock l(&mutex_); MutexLock l(&mutex_);
while (!manual.done && !shutting_down_.Acquire_Load() && bg_error_.ok()) { while (!manual.done && !shutting_down_.load(std::memory_order_acquire) &&
bg_error_.ok()) {
if (manual_compaction_ == nullptr) { // Idle if (manual_compaction_ == nullptr) { // Idle
manual_compaction_ = &manual; manual_compaction_ = &manual;
MaybeScheduleCompaction(); MaybeScheduleCompaction();
@ -652,12 +659,11 @@ void DBImpl::MaybeScheduleCompaction() {
mutex_.AssertHeld(); mutex_.AssertHeld();
if (background_compaction_scheduled_) { if (background_compaction_scheduled_) {
// Already scheduled // Already scheduled
} else if (shutting_down_.Acquire_Load()) { } else if (shutting_down_.load(std::memory_order_acquire)) {
// DB is being deleted; no more background compactions // DB is being deleted; no more background compactions
} else if (!bg_error_.ok()) { } else if (!bg_error_.ok()) {
// Already got an error; no more changes // Already got an error; no more changes
} else if (imm_ == nullptr && } else if (imm_ == nullptr && manual_compaction_ == nullptr &&
manual_compaction_ == nullptr &&
!versions_->NeedsCompaction()) { !versions_->NeedsCompaction()) {
// No work to be done // No work to be done
} else { } else {
@ -673,7 +679,7 @@ void DBImpl::BGWork(void* db) {
void DBImpl::BackgroundCall() { void DBImpl::BackgroundCall() {
MutexLock l(&mutex_); MutexLock l(&mutex_);
assert(background_compaction_scheduled_); assert(background_compaction_scheduled_);
if (shutting_down_.Acquire_Load()) { if (shutting_down_.load(std::memory_order_acquire)) {
// No more background work when shutting down. // No more background work when shutting down.
} else if (!bg_error_.ok()) { } else if (!bg_error_.ok()) {
// No more background work after a background error. // No more background work after a background error.
@ -709,8 +715,7 @@ void DBImpl::BackgroundCompaction() {
} }
Log(options_.info_log, Log(options_.info_log,
"Manual compaction at level-%d from %s .. %s; will stop at %s\n", "Manual compaction at level-%d from %s .. %s; will stop at %s\n",
m->level, m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
(m->begin ? m->begin->DebugString().c_str() : "(begin)"),
(m->end ? m->end->DebugString().c_str() : "(end)"), (m->end ? m->end->DebugString().c_str() : "(end)"),
(m->done ? "(end)" : manual_end.DebugString().c_str())); (m->done ? "(end)" : manual_end.DebugString().c_str()));
} else { } else {
@ -725,19 +730,17 @@ void DBImpl::BackgroundCompaction() {
assert(c->num_input_files(0) == 1); assert(c->num_input_files(0) == 1);
FileMetaData* f = c->input(0, 0); FileMetaData* f = c->input(0, 0);
c->edit()->DeleteFile(c->level(), f->number); c->edit()->DeleteFile(c->level(), f->number);
c->edit()->AddFile(c->level() + 1, f->number, f->file_size, c->edit()->AddFile(c->level() + 1, f->number, f->file_size, f->smallest,
f->smallest, f->largest); f->largest);
status = versions_->LogAndApply(c->edit(), &mutex_); status = versions_->LogAndApply(c->edit(), &mutex_);
if (!status.ok()) { if (!status.ok()) {
RecordBackgroundError(status); RecordBackgroundError(status);
} }
VersionSet::LevelSummaryStorage tmp; VersionSet::LevelSummaryStorage tmp;
Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n", Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n",
static_cast<unsigned long long>(f->number), static_cast<unsigned long long>(f->number), c->level() + 1,
c->level() + 1,
static_cast<unsigned long long>(f->file_size), static_cast<unsigned long long>(f->file_size),
status.ToString().c_str(), status.ToString().c_str(), versions_->LevelSummary(&tmp));
versions_->LevelSummary(&tmp));
} else { } else {
CompactionState* compact = new CompactionState(c); CompactionState* compact = new CompactionState(c);
status = DoCompactionWork(compact); status = DoCompactionWork(compact);
@ -752,11 +755,10 @@ void DBImpl::BackgroundCompaction() {
if (status.ok()) { if (status.ok()) {
// Done // Done
} else if (shutting_down_.Acquire_Load()) { } else if (shutting_down_.load(std::memory_order_acquire)) {
// Ignore compaction errors found during shutting down // Ignore compaction errors found during shutting down
} else { } else {
Log(options_.info_log, Log(options_.info_log, "Compaction error: %s", status.ToString().c_str());
"Compaction error: %s", status.ToString().c_str());
} }
if (is_manual) { if (is_manual) {
@ -851,16 +853,13 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
if (s.ok() && current_entries > 0) { if (s.ok() && current_entries > 0) {
// Verify that the table is usable // Verify that the table is usable
Iterator* iter = table_cache_->NewIterator(ReadOptions(), Iterator* iter =
output_number, table_cache_->NewIterator(ReadOptions(), output_number, current_bytes);
current_bytes);
s = iter->status(); s = iter->status();
delete iter; delete iter;
if (s.ok()) { if (s.ok()) {
Log(options_.info_log, Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes",
"Generated table #%llu@%d: %lld keys, %lld bytes", (unsigned long long)output_number, compact->compaction->level(),
(unsigned long long) output_number,
compact->compaction->level(),
(unsigned long long)current_entries, (unsigned long long)current_entries,
(unsigned long long)current_bytes); (unsigned long long)current_bytes);
} }
@ -868,14 +867,11 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
return s; return s;
} }
Status DBImpl::InstallCompactionResults(CompactionState* compact) { Status DBImpl::InstallCompactionResults(CompactionState* compact) {
mutex_.AssertHeld(); mutex_.AssertHeld();
Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes",
compact->compaction->num_input_files(0), compact->compaction->num_input_files(0), compact->compaction->level(),
compact->compaction->level(), compact->compaction->num_input_files(1), compact->compaction->level() + 1,
compact->compaction->num_input_files(1),
compact->compaction->level() + 1,
static_cast<long long>(compact->total_bytes)); static_cast<long long>(compact->total_bytes));
// Add compaction outputs // Add compaction outputs
@ -883,9 +879,8 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) {
const int level = compact->compaction->level(); const int level = compact->compaction->level();
for (size_t i = 0; i < compact->outputs.size(); i++) { for (size_t i = 0; i < compact->outputs.size(); i++) {
const CompactionState::Output& out = compact->outputs[i]; const CompactionState::Output& out = compact->outputs[i];
compact->compaction->edit()->AddFile( compact->compaction->edit()->AddFile(level + 1, out.number, out.file_size,
level + 1, out.smallest, out.largest);
out.number, out.file_size, out.smallest, out.largest);
} }
return versions_->LogAndApply(compact->compaction->edit(), &mutex_); return versions_->LogAndApply(compact->compaction->edit(), &mutex_);
} }
@ -895,8 +890,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
int64_t imm_micros = 0; // Micros spent doing imm_ compactions int64_t imm_micros = 0; // Micros spent doing imm_ compactions
Log(options_.info_log, "Compacting %d@%d + %d@%d files", Log(options_.info_log, "Compacting %d@%d + %d@%d files",
compact->compaction->num_input_files(0), compact->compaction->num_input_files(0), compact->compaction->level(),
compact->compaction->level(),
compact->compaction->num_input_files(1), compact->compaction->num_input_files(1),
compact->compaction->level() + 1); compact->compaction->level() + 1);
@ -909,19 +903,20 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
compact->smallest_snapshot = snapshots_.oldest()->sequence_number(); compact->smallest_snapshot = snapshots_.oldest()->sequence_number();
} }
Iterator* input = versions_->MakeInputIterator(compact->compaction);
// Release mutex while we're actually doing the compaction work // Release mutex while we're actually doing the compaction work
mutex_.Unlock(); mutex_.Unlock();
Iterator* input = versions_->MakeInputIterator(compact->compaction);
input->SeekToFirst(); input->SeekToFirst();
Status status; Status status;
ParsedInternalKey ikey; ParsedInternalKey ikey;
std::string current_user_key; std::string current_user_key;
bool has_current_user_key = false; bool has_current_user_key = false;
SequenceNumber last_sequence_for_key = kMaxSequenceNumber; SequenceNumber last_sequence_for_key = kMaxSequenceNumber;
for (; input->Valid() && !shutting_down_.Acquire_Load(); ) { while (input->Valid() && !shutting_down_.load(std::memory_order_acquire)) {
// Prioritize immutable compaction work // Prioritize immutable compaction work
if (has_imm_.NoBarrier_Load() != nullptr) { if (has_imm_.load(std::memory_order_relaxed)) {
const uint64_t imm_start = env_->NowMicros(); const uint64_t imm_start = env_->NowMicros();
mutex_.Lock(); mutex_.Lock();
if (imm_ != nullptr) { if (imm_ != nullptr) {
@ -951,8 +946,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
last_sequence_for_key = kMaxSequenceNumber; last_sequence_for_key = kMaxSequenceNumber;
} else { } else {
if (!has_current_user_key || if (!has_current_user_key ||
user_comparator()->Compare(ikey.user_key, user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) !=
Slice(current_user_key)) != 0) { 0) {
// First occurrence of this user key // First occurrence of this user key
current_user_key.assign(ikey.user_key.data(), ikey.user_key.size()); current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
has_current_user_key = true; has_current_user_key = true;
@ -1014,7 +1009,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
input->Next(); input->Next();
} }
if (status.ok() && shutting_down_.Acquire_Load()) { if (status.ok() && shutting_down_.load(std::memory_order_acquire)) {
status = Status::IOError("Deleting DB during compaction"); status = Status::IOError("Deleting DB during compaction");
} }
if (status.ok() && compact->builder != nullptr) { if (status.ok() && compact->builder != nullptr) {
@ -1047,8 +1042,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
RecordBackgroundError(status); RecordBackgroundError(status);
} }
VersionSet::LevelSummaryStorage tmp; VersionSet::LevelSummaryStorage tmp;
Log(options_.info_log, Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp));
"compacted to: %s", versions_->LevelSummary(&tmp));
return status; return status;
} }
@ -1114,8 +1108,7 @@ int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
return versions_->MaxNextLevelOverlappingBytes(); return versions_->MaxNextLevelOverlappingBytes();
} }
Status DBImpl::Get(const ReadOptions& options, Status DBImpl::Get(const ReadOptions& options, const Slice& key,
const Slice& key,
std::string* value) { std::string* value) {
Status s; Status s;
MutexLock l(&mutex_); MutexLock l(&mutex_);
@ -1166,10 +1159,10 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options) {
SequenceNumber latest_snapshot; SequenceNumber latest_snapshot;
uint32_t seed; uint32_t seed;
Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed); Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed);
return NewDBIterator( return NewDBIterator(this, user_comparator(), iter,
this, user_comparator(), iter,
(options.snapshot != nullptr (options.snapshot != nullptr
? static_cast<const SnapshotImpl*>(options.snapshot)->sequence_number() ? static_cast<const SnapshotImpl*>(options.snapshot)
->sequence_number()
: latest_snapshot), : latest_snapshot),
seed); seed);
} }
@ -1200,9 +1193,9 @@ Status DBImpl::Delete(const WriteOptions& options, const Slice& key) {
return DB::Delete(options, key); return DB::Delete(options, key);
} }
Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
Writer w(&mutex_); Writer w(&mutex_);
w.batch = my_batch; w.batch = updates;
w.sync = options.sync; w.sync = options.sync;
w.done = false; w.done = false;
@ -1216,13 +1209,13 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
} }
// May temporarily unlock and wait. // May temporarily unlock and wait.
Status status = MakeRoomForWrite(my_batch == nullptr); Status status = MakeRoomForWrite(updates == nullptr);
uint64_t last_sequence = versions_->LastSequence(); uint64_t last_sequence = versions_->LastSequence();
Writer* last_writer = &w; Writer* last_writer = &w;
if (status.ok() && my_batch != nullptr) { // nullptr batch is for compactions if (status.ok() && updates != nullptr) { // nullptr batch is for compactions
WriteBatch* updates = BuildBatchGroup(&last_writer); WriteBatch* write_batch = BuildBatchGroup(&last_writer);
WriteBatchInternal::SetSequence(updates, last_sequence + 1); WriteBatchInternal::SetSequence(write_batch, last_sequence + 1);
last_sequence += WriteBatchInternal::Count(updates); last_sequence += WriteBatchInternal::Count(write_batch);
// Add to log and apply to memtable. We can release the lock // Add to log and apply to memtable. We can release the lock
// during this phase since &w is currently responsible for logging // during this phase since &w is currently responsible for logging
@ -1230,7 +1223,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
// into mem_. // into mem_.
{ {
mutex_.Unlock(); mutex_.Unlock();
status = log_->AddRecord(WriteBatchInternal::Contents(updates)); status = log_->AddRecord(WriteBatchInternal::Contents(write_batch));
bool sync_error = false; bool sync_error = false;
if (status.ok() && options.sync) { if (status.ok() && options.sync) {
status = logfile_->Sync(); status = logfile_->Sync();
@ -1239,7 +1232,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
} }
} }
if (status.ok()) { if (status.ok()) {
status = WriteBatchInternal::InsertInto(updates, mem_); status = WriteBatchInternal::InsertInto(write_batch, mem_);
} }
mutex_.Lock(); mutex_.Lock();
if (sync_error) { if (sync_error) {
@ -1249,7 +1242,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
RecordBackgroundError(status); RecordBackgroundError(status);
} }
} }
if (updates == tmp_batch_) tmp_batch_->Clear(); if (write_batch == tmp_batch_) tmp_batch_->Clear();
versions_->SetLastSequence(last_sequence); versions_->SetLastSequence(last_sequence);
} }
@ -1335,9 +1328,8 @@ Status DBImpl::MakeRoomForWrite(bool force) {
// Yield previous error // Yield previous error
s = bg_error_; s = bg_error_;
break; break;
} else if ( } else if (allow_delay && versions_->NumLevelFiles(0) >=
allow_delay && config::kL0_SlowdownWritesTrigger) {
versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger) {
// We are getting close to hitting a hard limit on the number of // We are getting close to hitting a hard limit on the number of
// L0 files. Rather than delaying a single write by several // L0 files. Rather than delaying a single write by several
// seconds when we hit the hard limit, start delaying each // seconds when we hit the hard limit, start delaying each
@ -1378,7 +1370,7 @@ Status DBImpl::MakeRoomForWrite(bool force) {
logfile_number_ = new_log_number; logfile_number_ = new_log_number;
log_ = new log::Writer(lfile); log_ = new log::Writer(lfile);
imm_ = mem_; imm_ = mem_;
has_imm_.Release_Store(imm_); has_imm_.store(true, std::memory_order_release);
mem_ = new MemTable(internal_comparator_); mem_ = new MemTable(internal_comparator_);
mem_->Ref(); mem_->Ref();
force = false; // Do not force another compaction if have room force = false; // Do not force another compaction if have room
@ -1415,18 +1407,13 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
snprintf(buf, sizeof(buf), snprintf(buf, sizeof(buf),
" Compactions\n" " Compactions\n"
"Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n" "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n"
"--------------------------------------------------\n" "--------------------------------------------------\n");
);
value->append(buf); value->append(buf);
for (int level = 0; level < config::kNumLevels; level++) { for (int level = 0; level < config::kNumLevels; level++) {
int files = versions_->NumLevelFiles(level); int files = versions_->NumLevelFiles(level);
if (stats_[level].micros > 0 || files > 0) { if (stats_[level].micros > 0 || files > 0) {
snprintf( snprintf(buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", level,
buf, sizeof(buf), files, versions_->NumLevelBytes(level) / 1048576.0,
"%3d %8d %8.0f %9.0f %8.0f %9.0f\n",
level,
files,
versions_->NumLevelBytes(level) / 1048576.0,
stats_[level].micros / 1e6, stats_[level].micros / 1e6,
stats_[level].bytes_read / 1048576.0, stats_[level].bytes_read / 1048576.0,
stats_[level].bytes_written / 1048576.0); stats_[level].bytes_written / 1048576.0);
@ -1455,16 +1442,11 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
return false; return false;
} }
void DBImpl::GetApproximateSizes( void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) {
const Range* range, int n,
uint64_t* sizes) {
// TODO(opt): better implementation // TODO(opt): better implementation
Version* v;
{
MutexLock l(&mutex_); MutexLock l(&mutex_);
versions_->current()->Ref(); Version* v = versions_->current();
v = versions_->current(); v->Ref();
}
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
// Convert user_key into a corresponding internal key. // Convert user_key into a corresponding internal key.
@ -1475,11 +1457,8 @@ void DBImpl::GetApproximateSizes(
sizes[i] = (limit >= start ? limit - start : 0); sizes[i] = (limit >= start ? limit - start : 0);
} }
{
MutexLock l(&mutex_);
v->Unref(); v->Unref();
} }
}
// Default implementations of convenience methods that subclasses of DB // Default implementations of convenience methods that subclasses of DB
// can call if they wish // can call if they wish
@ -1495,10 +1474,9 @@ Status DB::Delete(const WriteOptions& opt, const Slice& key) {
return Write(opt, &batch); return Write(opt, &batch);
} }
DB::~DB() { } DB::~DB() = default;
Status DB::Open(const Options& options, const std::string& dbname, Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
DB** dbptr) {
*dbptr = nullptr; *dbptr = nullptr;
DBImpl* impl = new DBImpl(options, dbname); DBImpl* impl = new DBImpl(options, dbname);
@ -1541,8 +1519,7 @@ Status DB::Open(const Options& options, const std::string& dbname,
return s; return s;
} }
Snapshot::~Snapshot() { Snapshot::~Snapshot() = default;
}
Status DestroyDB(const std::string& dbname, const Options& options) { Status DestroyDB(const std::string& dbname, const Options& options) {
Env* env = options.env; Env* env = options.env;

View File

@ -5,8 +5,11 @@
#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_ #ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
#define STORAGE_LEVELDB_DB_DB_IMPL_H_ #define STORAGE_LEVELDB_DB_DB_IMPL_H_
#include <atomic>
#include <deque> #include <deque>
#include <set> #include <set>
#include <string>
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/log_writer.h" #include "db/log_writer.h"
#include "db/snapshot.h" #include "db/snapshot.h"
@ -26,21 +29,25 @@ class VersionSet;
class DBImpl : public DB { class DBImpl : public DB {
public: public:
DBImpl(const Options& options, const std::string& dbname); DBImpl(const Options& options, const std::string& dbname);
virtual ~DBImpl();
DBImpl(const DBImpl&) = delete;
DBImpl& operator=(const DBImpl&) = delete;
~DBImpl() override;
// Implementations of the DB interface // Implementations of the DB interface
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value); Status Put(const WriteOptions&, const Slice& key,
virtual Status Delete(const WriteOptions&, const Slice& key); const Slice& value) override;
virtual Status Write(const WriteOptions& options, WriteBatch* updates); Status Delete(const WriteOptions&, const Slice& key) override;
virtual Status Get(const ReadOptions& options, Status Write(const WriteOptions& options, WriteBatch* updates) override;
const Slice& key, Status Get(const ReadOptions& options, const Slice& key,
std::string* value); std::string* value) override;
virtual Iterator* NewIterator(const ReadOptions&); Iterator* NewIterator(const ReadOptions&) override;
virtual const Snapshot* GetSnapshot(); const Snapshot* GetSnapshot() override;
virtual void ReleaseSnapshot(const Snapshot* snapshot); void ReleaseSnapshot(const Snapshot* snapshot) override;
virtual bool GetProperty(const Slice& property, std::string* value); bool GetProperty(const Slice& property, std::string* value) override;
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes); void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) override;
virtual void CompactRange(const Slice* begin, const Slice* end); void CompactRange(const Slice* begin, const Slice* end) override;
// Extra methods (for testing) that are not in the public DB interface // Extra methods (for testing) that are not in the public DB interface
@ -69,6 +76,31 @@ class DBImpl : public DB {
struct CompactionState; struct CompactionState;
struct Writer; struct Writer;
// Information for a manual compaction
struct ManualCompaction {
int level;
bool done;
const InternalKey* begin; // null means beginning of key range
const InternalKey* end; // null means end of key range
InternalKey tmp_storage; // Used to keep track of compaction progress
};
// Per level compaction stats. stats_[level] stores the stats for
// compactions that produced data for the specified "level".
struct CompactionStats {
CompactionStats() : micros(0), bytes_read(0), bytes_written(0) {}
void Add(const CompactionStats& c) {
this->micros += c.micros;
this->bytes_read += c.bytes_read;
this->bytes_written += c.bytes_written;
}
int64_t micros;
int64_t bytes_read;
int64_t bytes_written;
};
Iterator* NewInternalIterator(const ReadOptions&, Iterator* NewInternalIterator(const ReadOptions&,
SequenceNumber* latest_snapshot, SequenceNumber* latest_snapshot,
uint32_t* seed); uint32_t* seed);
@ -119,6 +151,10 @@ class DBImpl : public DB {
Status InstallCompactionResults(CompactionState* compact) Status InstallCompactionResults(CompactionState* compact)
EXCLUSIVE_LOCKS_REQUIRED(mutex_); EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const Comparator* user_comparator() const {
return internal_comparator_.user_comparator();
}
// Constant after construction // Constant after construction
Env* const env_; Env* const env_;
const InternalKeyComparator internal_comparator_; const InternalKeyComparator internal_comparator_;
@ -136,11 +172,11 @@ class DBImpl : public DB {
// State below is protected by mutex_ // State below is protected by mutex_
port::Mutex mutex_; port::Mutex mutex_;
port::AtomicPointer shutting_down_; std::atomic<bool> shutting_down_;
port::CondVar background_work_finished_signal_ GUARDED_BY(mutex_); port::CondVar background_work_finished_signal_ GUARDED_BY(mutex_);
MemTable* mem_; MemTable* mem_;
MemTable* imm_ GUARDED_BY(mutex_); // Memtable being compacted MemTable* imm_ GUARDED_BY(mutex_); // Memtable being compacted
port::AtomicPointer has_imm_; // So bg thread can detect non-null imm_ std::atomic<bool> has_imm_; // So bg thread can detect non-null imm_
WritableFile* logfile_; WritableFile* logfile_;
uint64_t logfile_number_ GUARDED_BY(mutex_); uint64_t logfile_number_ GUARDED_BY(mutex_);
log::Writer* log_; log::Writer* log_;
@ -159,45 +195,14 @@ class DBImpl : public DB {
// Has a background compaction been scheduled or is running? // Has a background compaction been scheduled or is running?
bool background_compaction_scheduled_ GUARDED_BY(mutex_); bool background_compaction_scheduled_ GUARDED_BY(mutex_);
// Information for a manual compaction
struct ManualCompaction {
int level;
bool done;
const InternalKey* begin; // null means beginning of key range
const InternalKey* end; // null means end of key range
InternalKey tmp_storage; // Used to keep track of compaction progress
};
ManualCompaction* manual_compaction_ GUARDED_BY(mutex_); ManualCompaction* manual_compaction_ GUARDED_BY(mutex_);
VersionSet* const versions_; VersionSet* const versions_ GUARDED_BY(mutex_);
// Have we encountered a background error in paranoid mode? // Have we encountered a background error in paranoid mode?
Status bg_error_ GUARDED_BY(mutex_); Status bg_error_ GUARDED_BY(mutex_);
// Per level compaction stats. stats_[level] stores the stats for
// compactions that produced data for the specified "level".
struct CompactionStats {
int64_t micros;
int64_t bytes_read;
int64_t bytes_written;
CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { }
void Add(const CompactionStats& c) {
this->micros += c.micros;
this->bytes_read += c.bytes_read;
this->bytes_written += c.bytes_written;
}
};
CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_); CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_);
// No copying allowed
DBImpl(const DBImpl&);
void operator=(const DBImpl&);
const Comparator* user_comparator() const {
return internal_comparator_.user_comparator();
}
}; };
// Sanitize db options. The caller should delete result.info_log if // Sanitize db options. The caller should delete result.info_log if

View File

@ -4,9 +4,9 @@
#include "db/db_iter.h" #include "db/db_iter.h"
#include "db/filename.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/filename.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/iterator.h" #include "leveldb/iterator.h"
#include "port/port.h" #include "port/port.h"
@ -43,10 +43,7 @@ class DBIter: public Iterator {
// the exact entry that yields this->key(), this->value() // the exact entry that yields this->key(), this->value()
// (2) When moving backwards, the internal iterator is positioned // (2) When moving backwards, the internal iterator is positioned
// just before all entries whose user key == this->key(). // just before all entries whose user key == this->key().
enum Direction { enum Direction { kForward, kReverse };
kForward,
kReverse
};
DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s, DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
uint32_t seed) uint32_t seed)
@ -57,21 +54,22 @@ class DBIter: public Iterator {
direction_(kForward), direction_(kForward),
valid_(false), valid_(false),
rnd_(seed), rnd_(seed),
bytes_counter_(RandomPeriod()) { bytes_until_read_sampling_(RandomCompactionPeriod()) {}
}
virtual ~DBIter() { DBIter(const DBIter&) = delete;
delete iter_; DBIter& operator=(const DBIter&) = delete;
}
virtual bool Valid() const { return valid_; } ~DBIter() override { delete iter_; }
virtual Slice key() const { bool Valid() const override { return valid_; }
Slice key() const override {
assert(valid_); assert(valid_);
return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_; return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
} }
virtual Slice value() const { Slice value() const override {
assert(valid_); assert(valid_);
return (direction_ == kForward) ? iter_->value() : saved_value_; return (direction_ == kForward) ? iter_->value() : saved_value_;
} }
virtual Status status() const { Status status() const override {
if (status_.ok()) { if (status_.ok()) {
return iter_->status(); return iter_->status();
} else { } else {
@ -79,11 +77,11 @@ class DBIter: public Iterator {
} }
} }
virtual void Next(); void Next() override;
virtual void Prev(); void Prev() override;
virtual void Seek(const Slice& target); void Seek(const Slice& target) override;
virtual void SeekToFirst(); void SeekToFirst() override;
virtual void SeekToLast(); void SeekToLast() override;
private: private:
void FindNextUserEntry(bool skipping, std::string* skip); void FindNextUserEntry(bool skipping, std::string* skip);
@ -103,8 +101,8 @@ class DBIter: public Iterator {
} }
} }
// Pick next gap with average value of config::kReadBytesPeriod. // Picks the number of bytes that can be read until a compaction is scheduled.
ssize_t RandomPeriod() { size_t RandomCompactionPeriod() {
return rnd_.Uniform(2 * config::kReadBytesPeriod); return rnd_.Uniform(2 * config::kReadBytesPeriod);
} }
@ -112,29 +110,26 @@ class DBIter: public Iterator {
const Comparator* const user_comparator_; const Comparator* const user_comparator_;
Iterator* const iter_; Iterator* const iter_;
SequenceNumber const sequence_; SequenceNumber const sequence_;
Status status_; Status status_;
std::string saved_key_; // == current key when direction_==kReverse std::string saved_key_; // == current key when direction_==kReverse
std::string saved_value_; // == current raw value when direction_==kReverse std::string saved_value_; // == current raw value when direction_==kReverse
Direction direction_; Direction direction_;
bool valid_; bool valid_;
Random rnd_; Random rnd_;
ssize_t bytes_counter_; size_t bytes_until_read_sampling_;
// No copying allowed
DBIter(const DBIter&);
void operator=(const DBIter&);
}; };
inline bool DBIter::ParseKey(ParsedInternalKey* ikey) { inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
Slice k = iter_->key(); Slice k = iter_->key();
ssize_t n = k.size() + iter_->value().size();
bytes_counter_ -= n; size_t bytes_read = k.size() + iter_->value().size();
while (bytes_counter_ < 0) { while (bytes_until_read_sampling_ < bytes_read) {
bytes_counter_ += RandomPeriod(); bytes_until_read_sampling_ += RandomCompactionPeriod();
db_->RecordReadSample(k); db_->RecordReadSample(k);
} }
assert(bytes_until_read_sampling_ >= bytes_read);
bytes_until_read_sampling_ -= bytes_read;
if (!ParseInternalKey(k, ikey)) { if (!ParseInternalKey(k, ikey)) {
status_ = Status::Corruption("corrupted internal key in DBIter"); status_ = Status::Corruption("corrupted internal key in DBIter");
return false; return false;
@ -165,6 +160,15 @@ void DBIter::Next() {
} else { } else {
// Store in saved_key_ the current key so we skip it below. // Store in saved_key_ the current key so we skip it below.
SaveKey(ExtractUserKey(iter_->key()), &saved_key_); SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
// iter_ is pointing to current key. We can now safely move to the next to
// avoid checking current key.
iter_->Next();
if (!iter_->Valid()) {
valid_ = false;
saved_key_.clear();
return;
}
} }
FindNextUserEntry(true, &saved_key_); FindNextUserEntry(true, &saved_key_);
@ -218,8 +222,8 @@ void DBIter::Prev() {
ClearSavedValue(); ClearSavedValue();
return; return;
} }
if (user_comparator_->Compare(ExtractUserKey(iter_->key()), if (user_comparator_->Compare(ExtractUserKey(iter_->key()), saved_key_) <
saved_key_) < 0) { 0) {
break; break;
} }
} }
@ -275,8 +279,8 @@ void DBIter::Seek(const Slice& target) {
direction_ = kForward; direction_ = kForward;
ClearSavedValue(); ClearSavedValue();
saved_key_.clear(); saved_key_.clear();
AppendInternalKey( AppendInternalKey(&saved_key_,
&saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek)); ParsedInternalKey(target, sequence_, kValueTypeForSeek));
iter_->Seek(saved_key_); iter_->Seek(saved_key_);
if (iter_->Valid()) { if (iter_->Valid()) {
FindNextUserEntry(false, &saved_key_ /* temporary storage */); FindNextUserEntry(false, &saved_key_ /* temporary storage */);
@ -305,11 +309,8 @@ void DBIter::SeekToLast() {
} // anonymous namespace } // anonymous namespace
Iterator* NewDBIterator( Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
DBImpl* db, Iterator* internal_iter, SequenceNumber sequence,
const Comparator* user_key_comparator,
Iterator* internal_iter,
SequenceNumber sequence,
uint32_t seed) { uint32_t seed) {
return new DBIter(db, user_key_comparator, internal_iter, sequence, seed); return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
} }

View File

@ -6,8 +6,9 @@
#define STORAGE_LEVELDB_DB_DB_ITER_H_ #define STORAGE_LEVELDB_DB_DB_ITER_H_
#include <stdint.h> #include <stdint.h>
#include "leveldb/db.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "leveldb/db.h"
namespace leveldb { namespace leveldb {
@ -16,10 +17,8 @@ class DBImpl;
// Return a new iterator that converts internal keys (yielded by // Return a new iterator that converts internal keys (yielded by
// "*internal_iter") that were live at the specified "sequence" number // "*internal_iter") that were live at the specified "sequence" number
// into appropriate user keys. // into appropriate user keys.
Iterator* NewDBIterator(DBImpl* db, Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
const Comparator* user_key_comparator, Iterator* internal_iter, SequenceNumber sequence,
Iterator* internal_iter,
SequenceNumber sequence,
uint32_t seed); uint32_t seed);
} // namespace leveldb } // namespace leveldb

File diff suppressed because it is too large Load Diff

View File

@ -2,8 +2,12 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <stdio.h>
#include "db/dbformat.h" #include "db/dbformat.h"
#include <stdio.h>
#include <sstream>
#include "port/port.h" #include "port/port.h"
#include "util/coding.h" #include "util/coding.h"
@ -21,26 +25,20 @@ void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
} }
std::string ParsedInternalKey::DebugString() const { std::string ParsedInternalKey::DebugString() const {
char buf[50]; std::ostringstream ss;
snprintf(buf, sizeof(buf), "' @ %llu : %d", ss << '\'' << EscapeString(user_key.ToString()) << "' @ " << sequence << " : "
(unsigned long long) sequence, << static_cast<int>(type);
int(type)); return ss.str();
std::string result = "'";
result += EscapeString(user_key.ToString());
result += buf;
return result;
} }
std::string InternalKey::DebugString() const { std::string InternalKey::DebugString() const {
std::string result;
ParsedInternalKey parsed; ParsedInternalKey parsed;
if (ParseInternalKey(rep_, &parsed)) { if (ParseInternalKey(rep_, &parsed)) {
result = parsed.DebugString(); return parsed.DebugString();
} else {
result = "(bad)";
result.append(EscapeString(rep_));
} }
return result; std::ostringstream ss;
ss << "(bad)" << EscapeString(rep_);
return ss.str();
} }
const char* InternalKeyComparator::Name() const { const char* InternalKeyComparator::Name() const {
@ -65,8 +63,7 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
return r; return r;
} }
void InternalKeyComparator::FindShortestSeparator( void InternalKeyComparator::FindShortestSeparator(std::string* start,
std::string* start,
const Slice& limit) const { const Slice& limit) const {
// Attempt to shorten the user portion of the key // Attempt to shorten the user portion of the key
Slice user_start = ExtractUserKey(*start); Slice user_start = ExtractUserKey(*start);
@ -77,7 +74,8 @@ void InternalKeyComparator::FindShortestSeparator(
user_comparator_->Compare(user_start, tmp) < 0) { user_comparator_->Compare(user_start, tmp) < 0) {
// User key has become shorter physically, but larger logically. // User key has become shorter physically, but larger logically.
// Tack on the earliest possible number to the shortened user key. // Tack on the earliest possible number to the shortened user key.
PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); PutFixed64(&tmp,
PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
assert(this->Compare(*start, tmp) < 0); assert(this->Compare(*start, tmp) < 0);
assert(this->Compare(tmp, limit) < 0); assert(this->Compare(tmp, limit) < 0);
start->swap(tmp); start->swap(tmp);
@ -92,15 +90,14 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
user_comparator_->Compare(user_key, tmp) < 0) { user_comparator_->Compare(user_key, tmp) < 0) {
// User key has become shorter physically, but larger logically. // User key has become shorter physically, but larger logically.
// Tack on the earliest possible number to the shortened user key. // Tack on the earliest possible number to the shortened user key.
PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); PutFixed64(&tmp,
PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
assert(this->Compare(*key, tmp) < 0); assert(this->Compare(*key, tmp) < 0);
key->swap(tmp); key->swap(tmp);
} }
} }
const char* InternalFilterPolicy::Name() const { const char* InternalFilterPolicy::Name() const { return user_policy_->Name(); }
return user_policy_->Name();
}
void InternalFilterPolicy::CreateFilter(const Slice* keys, int n, void InternalFilterPolicy::CreateFilter(const Slice* keys, int n,
std::string* dst) const { std::string* dst) const {

View File

@ -5,7 +5,10 @@
#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_ #ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
#define STORAGE_LEVELDB_DB_DBFORMAT_H_ #define STORAGE_LEVELDB_DB_DBFORMAT_H_
#include <stdio.h> #include <cstddef>
#include <cstdint>
#include <string>
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
#include "leveldb/db.h" #include "leveldb/db.h"
#include "leveldb/filter_policy.h" #include "leveldb/filter_policy.h"
@ -48,10 +51,7 @@ class InternalKey;
// Value types encoded as the last component of internal keys. // Value types encoded as the last component of internal keys.
// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk // DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
// data structures. // data structures.
enum ValueType { enum ValueType { kTypeDeletion = 0x0, kTypeValue = 0x1 };
kTypeDeletion = 0x0,
kTypeValue = 0x1
};
// kValueTypeForSeek defines the ValueType that should be passed when // kValueTypeForSeek defines the ValueType that should be passed when
// constructing a ParsedInternalKey object for seeking to a particular // constructing a ParsedInternalKey object for seeking to a particular
// sequence number (since we sort sequence numbers in decreasing order // sequence number (since we sort sequence numbers in decreasing order
@ -64,8 +64,7 @@ typedef uint64_t SequenceNumber;
// We leave eight bits empty at the bottom so a type and sequence# // We leave eight bits empty at the bottom so a type and sequence#
// can be packed together into 64-bits. // can be packed together into 64-bits.
static const SequenceNumber kMaxSequenceNumber = static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1);
((0x1ull << 56) - 1);
struct ParsedInternalKey { struct ParsedInternalKey {
Slice user_key; Slice user_key;
@ -103,14 +102,14 @@ inline Slice ExtractUserKey(const Slice& internal_key) {
class InternalKeyComparator : public Comparator { class InternalKeyComparator : public Comparator {
private: private:
const Comparator* user_comparator_; const Comparator* user_comparator_;
public: public:
explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {} explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {}
virtual const char* Name() const; const char* Name() const override;
virtual int Compare(const Slice& a, const Slice& b) const; int Compare(const Slice& a, const Slice& b) const override;
virtual void FindShortestSeparator( void FindShortestSeparator(std::string* start,
std::string* start, const Slice& limit) const override;
const Slice& limit) const; void FindShortSuccessor(std::string* key) const override;
virtual void FindShortSuccessor(std::string* key) const;
const Comparator* user_comparator() const { return user_comparator_; } const Comparator* user_comparator() const { return user_comparator_; }
@ -121,11 +120,12 @@ class InternalKeyComparator : public Comparator {
class InternalFilterPolicy : public FilterPolicy { class InternalFilterPolicy : public FilterPolicy {
private: private:
const FilterPolicy* const user_policy_; const FilterPolicy* const user_policy_;
public: public:
explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {} explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {}
virtual const char* Name() const; const char* Name() const override;
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const; void CreateFilter(const Slice* keys, int n, std::string* dst) const override;
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const; bool KeyMayMatch(const Slice& key, const Slice& filter) const override;
}; };
// Modules in this directory should keep internal keys wrapped inside // Modules in this directory should keep internal keys wrapped inside
@ -134,13 +134,18 @@ class InternalFilterPolicy : public FilterPolicy {
class InternalKey { class InternalKey {
private: private:
std::string rep_; std::string rep_;
public: public:
InternalKey() {} // Leave rep_ as empty to indicate it is invalid InternalKey() {} // Leave rep_ as empty to indicate it is invalid
InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) { InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t)); AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
} }
void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); } bool DecodeFrom(const Slice& s) {
rep_.assign(s.data(), s.size());
return !rep_.empty();
}
Slice Encode() const { Slice Encode() const {
assert(!rep_.empty()); assert(!rep_.empty());
return rep_; return rep_;
@ -158,8 +163,8 @@ class InternalKey {
std::string DebugString() const; std::string DebugString() const;
}; };
inline int InternalKeyComparator::Compare( inline int InternalKeyComparator::Compare(const InternalKey& a,
const InternalKey& a, const InternalKey& b) const { const InternalKey& b) const {
return Compare(a.Encode(), b.Encode()); return Compare(a.Encode(), b.Encode());
} }
@ -168,11 +173,11 @@ inline bool ParseInternalKey(const Slice& internal_key,
const size_t n = internal_key.size(); const size_t n = internal_key.size();
if (n < 8) return false; if (n < 8) return false;
uint64_t num = DecodeFixed64(internal_key.data() + n - 8); uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
unsigned char c = num & 0xff; uint8_t c = num & 0xff;
result->sequence = num >> 8; result->sequence = num >> 8;
result->type = static_cast<ValueType>(c); result->type = static_cast<ValueType>(c);
result->user_key = Slice(internal_key.data(), n - 8); result->user_key = Slice(internal_key.data(), n - 8);
return (c <= static_cast<unsigned char>(kTypeValue)); return (c <= static_cast<uint8_t>(kTypeValue));
} }
// A helper class useful for DBImpl::Get() // A helper class useful for DBImpl::Get()
@ -182,6 +187,9 @@ class LookupKey {
// the specified sequence number. // the specified sequence number.
LookupKey(const Slice& user_key, SequenceNumber sequence); LookupKey(const Slice& user_key, SequenceNumber sequence);
LookupKey(const LookupKey&) = delete;
LookupKey& operator=(const LookupKey&) = delete;
~LookupKey(); ~LookupKey();
// Return a key suitable for lookup in a MemTable. // Return a key suitable for lookup in a MemTable.
@ -205,10 +213,6 @@ class LookupKey {
const char* kstart_; const char* kstart_;
const char* end_; const char* end_;
char space_[200]; // Avoid allocation for short keys char space_[200]; // Avoid allocation for short keys
// No copying allowed
LookupKey(const LookupKey&);
void operator=(const LookupKey&);
}; };
inline LookupKey::~LookupKey() { inline LookupKey::~LookupKey() {

View File

@ -3,13 +3,13 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/dbformat.h" #include "db/dbformat.h"
#include "gtest/gtest.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
namespace leveldb { namespace leveldb {
static std::string IKey(const std::string& user_key, static std::string IKey(const std::string& user_key, uint64_t seq,
uint64_t seq,
ValueType vt) { ValueType vt) {
std::string encoded; std::string encoded;
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
@ -28,9 +28,7 @@ static std::string ShortSuccessor(const std::string& s) {
return result; return result;
} }
static void TestKey(const std::string& key, static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
uint64_t seq,
ValueType vt) {
std::string encoded = IKey(key, seq, vt); std::string encoded = IKey(key, seq, vt);
Slice in(encoded); Slice in(encoded);
@ -44,16 +42,20 @@ static void TestKey(const std::string& key,
ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded));
} }
class FormatTest { };
TEST(FormatTest, InternalKey_EncodeDecode) { TEST(FormatTest, InternalKey_EncodeDecode) {
const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"}; const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
const uint64_t seq[] = { const uint64_t seq[] = {1,
1, 2, 3, 2,
(1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, 3,
(1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, (1ull << 8) - 1,
(1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 1ull << 8,
}; (1ull << 8) + 1,
(1ull << 16) - 1,
1ull << 16,
(1ull << 16) + 1,
(1ull << 32) - 1,
1ull << 32,
(1ull << 32) + 1};
for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
TestKey(keys[k], seq[s], kTypeValue); TestKey(keys[k], seq[s], kTypeValue);
@ -62,40 +64,44 @@ TEST(FormatTest, InternalKey_EncodeDecode) {
} }
} }
TEST(FormatTest, InternalKey_DecodeFromEmpty) {
InternalKey internal_key;
ASSERT_TRUE(!internal_key.DecodeFrom(""));
}
TEST(FormatTest, InternalKeyShortSeparator) { TEST(FormatTest, InternalKeyShortSeparator) {
// When user keys are same // When user keys are same
ASSERT_EQ(IKey("foo", 100, kTypeValue), ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue)));
IKey("foo", 99, kTypeValue))); ASSERT_EQ(
ASSERT_EQ(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue)));
IKey("foo", 101, kTypeValue))); ASSERT_EQ(
ASSERT_EQ(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue)));
IKey("foo", 100, kTypeValue))); ASSERT_EQ(
ASSERT_EQ(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion)));
IKey("foo", 100, kTypeDeletion)));
// When user keys are misordered // When user keys are misordered
ASSERT_EQ(IKey("foo", 100, kTypeValue), ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue)));
IKey("bar", 99, kTypeValue)));
// When user keys are different, but correctly ordered // When user keys are different, but correctly ordered
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), ASSERT_EQ(
Shorten(IKey("foo", 100, kTypeValue), IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
IKey("hello", 200, kTypeValue))); Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue)));
// When start user key is prefix of limit user key // When start user key is prefix of limit user key
ASSERT_EQ(IKey("foo", 100, kTypeValue), ASSERT_EQ(
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
IKey("foobar", 200, kTypeValue))); Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue)));
// When limit user key is prefix of start user key // When limit user key is prefix of start user key
ASSERT_EQ(IKey("foobar", 100, kTypeValue), ASSERT_EQ(
Shorten(IKey("foobar", 100, kTypeValue), IKey("foobar", 100, kTypeValue),
IKey("foo", 200, kTypeValue))); Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue)));
} }
TEST(FormatTest, InternalKeyShortestSuccessor) { TEST(FormatTest, InternalKeyShortestSuccessor) {
@ -105,8 +111,23 @@ TEST(FormatTest, InternalKeyShortestSuccessor) {
ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
} }
TEST(FormatTest, ParsedInternalKeyDebugString) {
ParsedInternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue);
ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString());
}
TEST(FormatTest, InternalKeyDebugString) {
InternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue);
ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString());
InternalKey invalid_key;
ASSERT_EQ("(bad)", invalid_key.DebugString());
}
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -38,8 +38,7 @@ bool GuessType(const std::string& fname, FileType* type) {
// Notified when log reader encounters corruption. // Notified when log reader encounters corruption.
class CorruptionReporter : public log::Reader::Reporter { class CorruptionReporter : public log::Reader::Reporter {
public: public:
WritableFile* dst_; void Corruption(size_t bytes, const Status& status) override {
virtual void Corruption(size_t bytes, const Status& status) {
std::string r = "corruption: "; std::string r = "corruption: ";
AppendNumberTo(&r, bytes); AppendNumberTo(&r, bytes);
r += " bytes; "; r += " bytes; ";
@ -47,6 +46,8 @@ class CorruptionReporter : public log::Reader::Reporter {
r.push_back('\n'); r.push_back('\n');
dst_->Append(r); dst_->Append(r);
} }
WritableFile* dst_;
}; };
// Print contents of a log file. (*func)() is called on every record. // Print contents of a log file. (*func)() is called on every record.
@ -73,8 +74,7 @@ Status PrintLogContents(Env* env, const std::string& fname,
// Called on every item found in a WriteBatch. // Called on every item found in a WriteBatch.
class WriteBatchItemPrinter : public WriteBatch::Handler { class WriteBatchItemPrinter : public WriteBatch::Handler {
public: public:
WritableFile* dst_; void Put(const Slice& key, const Slice& value) override {
virtual void Put(const Slice& key, const Slice& value) {
std::string r = " put '"; std::string r = " put '";
AppendEscapedStringTo(&r, key); AppendEscapedStringTo(&r, key);
r += "' '"; r += "' '";
@ -82,14 +82,15 @@ class WriteBatchItemPrinter : public WriteBatch::Handler {
r += "'\n"; r += "'\n";
dst_->Append(r); dst_->Append(r);
} }
virtual void Delete(const Slice& key) { void Delete(const Slice& key) override {
std::string r = " del '"; std::string r = " del '";
AppendEscapedStringTo(&r, key); AppendEscapedStringTo(&r, key);
r += "'\n"; r += "'\n";
dst_->Append(r); dst_->Append(r);
} }
};
WritableFile* dst_;
};
// Called on every log record (each one of which is a WriteBatch) // Called on every log record (each one of which is a WriteBatch)
// found in a kLogFile. // found in a kLogFile.
@ -216,9 +217,12 @@ Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
return Status::InvalidArgument(fname + ": unknown file type"); return Status::InvalidArgument(fname + ": unknown file type");
} }
switch (ftype) { switch (ftype) {
case kLogFile: return DumpLog(env, fname, dst); case kLogFile:
case kDescriptorFile: return DumpDescriptor(env, fname, dst); return DumpLog(env, fname, dst);
case kTableFile: return DumpTable(env, fname, dst); case kDescriptorFile:
return DumpDescriptor(env, fname, dst);
case kTableFile:
return DumpTable(env, fname, dst);
default: default:
break; break;
} }

View File

@ -9,12 +9,13 @@
#include <map> #include <map>
#include <set> #include <set>
#include "leveldb/db.h" #include "gtest/gtest.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/filename.h" #include "db/filename.h"
#include "db/log_format.h" #include "db/log_format.h"
#include "db/version_set.h" #include "db/version_set.h"
#include "leveldb/cache.h" #include "leveldb/cache.h"
#include "leveldb/db.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/table.h" #include "leveldb/table.h"
#include "leveldb/write_batch.h" #include "leveldb/write_batch.h"
@ -22,7 +23,6 @@
#include "port/thread_annotations.h" #include "port/thread_annotations.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
@ -56,8 +56,7 @@ Status Truncate(const std::string& filename, uint64_t length) {
SequentialFile* orig_file; SequentialFile* orig_file;
Status s = env->NewSequentialFile(filename, &orig_file); Status s = env->NewSequentialFile(filename, &orig_file);
if (!s.ok()) if (!s.ok()) return s;
return s;
char* scratch = new char[length]; char* scratch = new char[length];
leveldb::Slice result; leveldb::Slice result;
@ -85,9 +84,9 @@ Status Truncate(const std::string& filename, uint64_t length) {
struct FileState { struct FileState {
std::string filename_; std::string filename_;
ssize_t pos_; int64_t pos_;
ssize_t pos_at_last_sync_; int64_t pos_at_last_sync_;
ssize_t pos_at_last_flush_; int64_t pos_at_last_flush_;
FileState(const std::string& filename) FileState(const std::string& filename)
: filename_(filename), : filename_(filename),
@ -108,14 +107,13 @@ struct FileState {
// is written to or sync'ed. // is written to or sync'ed.
class TestWritableFile : public WritableFile { class TestWritableFile : public WritableFile {
public: public:
TestWritableFile(const FileState& state, TestWritableFile(const FileState& state, WritableFile* f,
WritableFile* f,
FaultInjectionTestEnv* env); FaultInjectionTestEnv* env);
virtual ~TestWritableFile(); ~TestWritableFile() override;
virtual Status Append(const Slice& data); Status Append(const Slice& data) override;
virtual Status Close(); Status Close() override;
virtual Status Flush(); Status Flush() override;
virtual Status Sync(); Status Sync() override;
private: private:
FileState state_; FileState state_;
@ -130,13 +128,13 @@ class FaultInjectionTestEnv : public EnvWrapper {
public: public:
FaultInjectionTestEnv() FaultInjectionTestEnv()
: EnvWrapper(Env::Default()), filesystem_active_(true) {} : EnvWrapper(Env::Default()), filesystem_active_(true) {}
virtual ~FaultInjectionTestEnv() { } ~FaultInjectionTestEnv() override = default;
virtual Status NewWritableFile(const std::string& fname, Status NewWritableFile(const std::string& fname,
WritableFile** result); WritableFile** result) override;
virtual Status NewAppendableFile(const std::string& fname, Status NewAppendableFile(const std::string& fname,
WritableFile** result); WritableFile** result) override;
virtual Status DeleteFile(const std::string& f); Status DeleteFile(const std::string& f) override;
virtual Status RenameFile(const std::string& s, const std::string& t); Status RenameFile(const std::string& s, const std::string& t) override;
void WritableFileClosed(const FileState& state); void WritableFileClosed(const FileState& state);
Status DropUnsyncedFileData(); Status DropUnsyncedFileData();
@ -165,13 +163,9 @@ class FaultInjectionTestEnv : public EnvWrapper {
bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes
}; };
TestWritableFile::TestWritableFile(const FileState& state, TestWritableFile::TestWritableFile(const FileState& state, WritableFile* f,
WritableFile* f,
FaultInjectionTestEnv* env) FaultInjectionTestEnv* env)
: state_(state), : state_(state), target_(f), writable_file_opened_(true), env_(env) {
target_(f),
writable_file_opened_(true),
env_(env) {
assert(f != nullptr); assert(f != nullptr);
} }
@ -274,10 +268,11 @@ Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname,
Status FaultInjectionTestEnv::DropUnsyncedFileData() { Status FaultInjectionTestEnv::DropUnsyncedFileData() {
Status s; Status s;
MutexLock l(&mutex_); MutexLock l(&mutex_);
for (std::map<std::string, FileState>::const_iterator it = for (const auto& kvp : db_file_state_) {
db_file_state_.begin(); if (!s.ok()) {
s.ok() && it != db_file_state_.end(); ++it) { break;
const FileState& state = it->second; }
const FileState& state = kvp.second;
if (!state.IsFullySynced()) { if (!state.IsFullySynced()) {
s = state.DropUnsyncedData(); s = state.DropUnsyncedData();
} }
@ -305,7 +300,7 @@ void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
Status FaultInjectionTestEnv::DeleteFile(const std::string& f) { Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
Status s = EnvWrapper::DeleteFile(f); Status s = EnvWrapper::DeleteFile(f);
ASSERT_OK(s); EXPECT_LEVELDB_OK(s);
if (s.ok()) { if (s.ok()) {
UntrackFile(f); UntrackFile(f);
} }
@ -346,12 +341,14 @@ Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(), std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(),
new_files_since_last_dir_sync_.end()); new_files_since_last_dir_sync_.end());
mutex_.Unlock(); mutex_.Unlock();
Status s; Status status;
std::set<std::string>::const_iterator it; for (const auto& new_file : new_files) {
for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) { Status delete_status = DeleteFile(new_file);
s = DeleteFile(*it); if (!delete_status.ok() && status.ok()) {
status = std::move(delete_status);
} }
return s; }
return status;
} }
void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
@ -360,11 +357,11 @@ void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
} }
Status FileState::DropUnsyncedData() const { Status FileState::DropUnsyncedData() const {
ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; int64_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
return Truncate(filename_, sync_pos); return Truncate(filename_, sync_pos);
} }
class FaultInjectionTest { class FaultInjectionTest : public testing::Test {
public: public:
enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR }; enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR };
enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES }; enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES };
@ -379,7 +376,7 @@ class FaultInjectionTest {
: env_(new FaultInjectionTestEnv), : env_(new FaultInjectionTestEnv),
tiny_cache_(NewLRUCache(100)), tiny_cache_(NewLRUCache(100)),
db_(nullptr) { db_(nullptr) {
dbname_ = test::TmpDir() + "/fault_test"; dbname_ = testing::TempDir() + "fault_test";
DestroyDB(dbname_, Options()); // Destroy any db from earlier run DestroyDB(dbname_, Options()); // Destroy any db from earlier run
options_.reuse_logs = true; options_.reuse_logs = true;
options_.env = env_; options_.env = env_;
@ -395,9 +392,7 @@ class FaultInjectionTest {
delete env_; delete env_;
} }
void ReuseLogs(bool reuse) { void ReuseLogs(bool reuse) { options_.reuse_logs = reuse; }
options_.reuse_logs = reuse;
}
void Build(int start_idx, int num_vals) { void Build(int start_idx, int num_vals) {
std::string key_space, value_space; std::string key_space, value_space;
@ -407,7 +402,7 @@ class FaultInjectionTest {
batch.Clear(); batch.Clear();
batch.Put(key, Value(i, &value_space)); batch.Put(key, Value(i, &value_space));
WriteOptions options; WriteOptions options;
ASSERT_OK(db_->Write(options, &batch)); ASSERT_LEVELDB_OK(db_->Write(options, &batch));
} }
} }
@ -429,7 +424,7 @@ class FaultInjectionTest {
s = ReadValue(i, &val); s = ReadValue(i, &val);
if (expected == VAL_EXPECT_NO_ERROR) { if (expected == VAL_EXPECT_NO_ERROR) {
if (s.ok()) { if (s.ok()) {
ASSERT_EQ(value_space, val); EXPECT_EQ(value_space, val);
} }
} else if (s.ok()) { } else if (s.ok()) {
fprintf(stderr, "Expected an error at %d, but was OK\n", i); fprintf(stderr, "Expected an error at %d, but was OK\n", i);
@ -469,9 +464,8 @@ class FaultInjectionTest {
void DeleteAllData() { void DeleteAllData() {
Iterator* iter = db_->NewIterator(ReadOptions()); Iterator* iter = db_->NewIterator(ReadOptions());
WriteOptions options;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ASSERT_OK(db_->Delete(WriteOptions(), iter->key())); ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), iter->key()));
} }
delete iter; delete iter;
@ -480,10 +474,10 @@ class FaultInjectionTest {
void ResetDBState(ResetMethod reset_method) { void ResetDBState(ResetMethod reset_method) {
switch (reset_method) { switch (reset_method) {
case RESET_DROP_UNSYNCED_DATA: case RESET_DROP_UNSYNCED_DATA:
ASSERT_OK(env_->DropUnsyncedFileData()); ASSERT_LEVELDB_OK(env_->DropUnsyncedFileData());
break; break;
case RESET_DELETE_UNSYNCED_FILES: case RESET_DELETE_UNSYNCED_FILES:
ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); ASSERT_LEVELDB_OK(env_->DeleteFilesCreatedAfterLastDirSync());
break; break;
default: default:
assert(false); assert(false);
@ -498,35 +492,34 @@ class FaultInjectionTest {
} }
void PartialCompactTestReopenWithFault(ResetMethod reset_method, void PartialCompactTestReopenWithFault(ResetMethod reset_method,
int num_pre_sync, int num_pre_sync, int num_post_sync) {
int num_post_sync) {
env_->SetFilesystemActive(false); env_->SetFilesystemActive(false);
CloseDB(); CloseDB();
ResetDBState(reset_method); ResetDBState(reset_method);
ASSERT_OK(OpenDB()); ASSERT_LEVELDB_OK(OpenDB());
ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR)); ASSERT_LEVELDB_OK(
ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR)); Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
ASSERT_LEVELDB_OK(Verify(num_pre_sync, num_post_sync,
FaultInjectionTest::VAL_EXPECT_ERROR));
} }
void NoWriteTestPreFault() { void NoWriteTestPreFault() {}
}
void NoWriteTestReopenWithFault(ResetMethod reset_method) { void NoWriteTestReopenWithFault(ResetMethod reset_method) {
CloseDB(); CloseDB();
ResetDBState(reset_method); ResetDBState(reset_method);
ASSERT_OK(OpenDB()); ASSERT_LEVELDB_OK(OpenDB());
} }
void DoTest() { void DoTest() {
Random rnd(0); Random rnd(0);
ASSERT_OK(OpenDB()); ASSERT_LEVELDB_OK(OpenDB());
for (size_t idx = 0; idx < kNumIterations; idx++) { for (size_t idx = 0; idx < kNumIterations; idx++) {
int num_pre_sync = rnd.Uniform(kMaxNumValues); int num_pre_sync = rnd.Uniform(kMaxNumValues);
int num_post_sync = rnd.Uniform(kMaxNumValues); int num_post_sync = rnd.Uniform(kMaxNumValues);
PartialCompactTestPreFault(num_pre_sync, num_post_sync); PartialCompactTestPreFault(num_pre_sync, num_post_sync);
PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, num_pre_sync,
num_pre_sync,
num_post_sync); num_post_sync);
NoWriteTestPreFault(); NoWriteTestPreFault();
@ -536,8 +529,7 @@ class FaultInjectionTest {
// No new files created so we expect all values since no files will be // No new files created so we expect all values since no files will be
// dropped. // dropped.
PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES, PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
num_pre_sync + num_post_sync, num_pre_sync + num_post_sync, 0);
0);
NoWriteTestPreFault(); NoWriteTestPreFault();
NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES); NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
@ -545,12 +537,12 @@ class FaultInjectionTest {
} }
}; };
TEST(FaultInjectionTest, FaultTestNoLogReuse) { TEST_F(FaultInjectionTest, FaultTestNoLogReuse) {
ReuseLogs(false); ReuseLogs(false);
DoTest(); DoTest();
} }
TEST(FaultInjectionTest, FaultTestWithLogReuse) { TEST_F(FaultInjectionTest, FaultTestWithLogReuse) {
ReuseLogs(true); ReuseLogs(true);
DoTest(); DoTest();
} }
@ -558,5 +550,6 @@ TEST(FaultInjectionTest, FaultTestWithLogReuse) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -2,9 +2,11 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/filename.h"
#include <ctype.h> #include <ctype.h>
#include <stdio.h> #include <stdio.h>
#include "db/filename.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "util/logging.h" #include "util/logging.h"
@ -19,8 +21,7 @@ static std::string MakeFileName(const std::string& dbname, uint64_t number,
const char* suffix) { const char* suffix) {
char buf[100]; char buf[100];
snprintf(buf, sizeof(buf), "/%06llu.%s", snprintf(buf, sizeof(buf), "/%06llu.%s",
static_cast<unsigned long long>(number), static_cast<unsigned long long>(number), suffix);
suffix);
return dbname + buf; return dbname + buf;
} }
@ -51,9 +52,7 @@ std::string CurrentFileName(const std::string& dbname) {
return dbname + "/CURRENT"; return dbname + "/CURRENT";
} }
std::string LockFileName(const std::string& dbname) { std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; }
return dbname + "/LOCK";
}
std::string TempFileName(const std::string& dbname, uint64_t number) { std::string TempFileName(const std::string& dbname, uint64_t number) {
assert(number > 0); assert(number > 0);
@ -69,7 +68,6 @@ std::string OldInfoLogFileName(const std::string& dbname) {
return dbname + "/LOG.old"; return dbname + "/LOG.old";
} }
// Owned filenames have the form: // Owned filenames have the form:
// dbname/CURRENT // dbname/CURRENT
// dbname/LOCK // dbname/LOCK
@ -77,8 +75,7 @@ std::string OldInfoLogFileName(const std::string& dbname) {
// dbname/LOG.old // dbname/LOG.old
// dbname/MANIFEST-[0-9]+ // dbname/MANIFEST-[0-9]+
// dbname/[0-9]+.(log|sst|ldb) // dbname/[0-9]+.(log|sst|ldb)
bool ParseFileName(const std::string& filename, bool ParseFileName(const std::string& filename, uint64_t* number,
uint64_t* number,
FileType* type) { FileType* type) {
Slice rest(filename); Slice rest(filename);
if (rest == "CURRENT") { if (rest == "CURRENT") {

View File

@ -8,7 +8,9 @@
#define STORAGE_LEVELDB_DB_FILENAME_H_ #define STORAGE_LEVELDB_DB_FILENAME_H_
#include <stdint.h> #include <stdint.h>
#include <string> #include <string>
#include "leveldb/slice.h" #include "leveldb/slice.h"
#include "leveldb/status.h" #include "leveldb/status.h"
#include "port/port.h" #include "port/port.h"
@ -69,8 +71,7 @@ std::string OldInfoLogFileName(const std::string& dbname);
// If filename is a leveldb file, store the type of the file in *type. // If filename is a leveldb file, store the type of the file in *type.
// The number encoded in the filename is stored in *number. If the // The number encoded in the filename is stored in *number. If the
// filename was successfully parsed, returns true. Else return false. // filename was successfully parsed, returns true. Else return false.
bool ParseFileName(const std::string& filename, bool ParseFileName(const std::string& filename, uint64_t* number,
uint64_t* number,
FileType* type); FileType* type);
// Make the CURRENT file point to the descriptor file with the // Make the CURRENT file point to the descriptor file with the

View File

@ -4,15 +4,13 @@
#include "db/filename.h" #include "db/filename.h"
#include "gtest/gtest.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "port/port.h" #include "port/port.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
namespace leveldb { namespace leveldb {
class FileNameTest { };
TEST(FileNameTest, Parse) { TEST(FileNameTest, Parse) {
Slice db; Slice db;
FileType type; FileType type;
@ -44,8 +42,7 @@ TEST(FileNameTest, Parse) {
} }
// Errors // Errors
static const char* errors[] = { static const char* errors[] = {"",
"",
"foo", "foo",
"foo-dx-100.log", "foo-dx-100.log",
".log", ".log",
@ -66,8 +63,7 @@ TEST(FileNameTest, Parse) {
"184467440737095516150.log", "184467440737095516150.log",
"100", "100",
"100.", "100.",
"100.lop" "100.lop"};
};
for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
std::string f = errors[i]; std::string f = errors[i];
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
@ -131,5 +127,6 @@ TEST(FileNameTest, Construction) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -3,6 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <stdio.h> #include <stdio.h>
#include "leveldb/dumpfile.h" #include "leveldb/dumpfile.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/status.h" #include "leveldb/status.h"
@ -12,13 +13,13 @@ namespace {
class StdoutPrinter : public WritableFile { class StdoutPrinter : public WritableFile {
public: public:
virtual Status Append(const Slice& data) { Status Append(const Slice& data) override {
fwrite(data.data(), 1, data.size(), stdout); fwrite(data.data(), 1, data.size(), stdout);
return Status::OK(); return Status::OK();
} }
virtual Status Close() { return Status::OK(); } Status Close() override { return Status::OK(); }
virtual Status Flush() { return Status::OK(); } Status Flush() override { return Status::OK(); }
virtual Status Sync() { return Status::OK(); } Status Sync() override { return Status::OK(); }
}; };
bool HandleDumpCommand(Env* env, char** files, int num) { bool HandleDumpCommand(Env* env, char** files, int num) {
@ -38,11 +39,9 @@ bool HandleDumpCommand(Env* env, char** files, int num) {
} // namespace leveldb } // namespace leveldb
static void Usage() { static void Usage() {
fprintf( fprintf(stderr,
stderr,
"Usage: leveldbutil command...\n" "Usage: leveldbutil command...\n"
" dump files... -- dump contents of specified files\n" " dump files... -- dump contents of specified files\n");
);
} }
int main(int argc, char** argv) { int main(int argc, char** argv) {

View File

@ -5,6 +5,7 @@
#include "db/log_reader.h" #include "db/log_reader.h"
#include <stdio.h> #include <stdio.h>
#include "leveldb/env.h" #include "leveldb/env.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/crc32c.h" #include "util/crc32c.h"
@ -12,8 +13,7 @@
namespace leveldb { namespace leveldb {
namespace log { namespace log {
Reader::Reporter::~Reporter() { Reader::Reporter::~Reporter() = default;
}
Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
uint64_t initial_offset) uint64_t initial_offset)
@ -26,12 +26,9 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
last_record_offset_(0), last_record_offset_(0),
end_of_buffer_offset_(0), end_of_buffer_offset_(0),
initial_offset_(initial_offset), initial_offset_(initial_offset),
resyncing_(initial_offset > 0) { resyncing_(initial_offset > 0) {}
}
Reader::~Reader() { Reader::~Reader() { delete[] backing_store_; }
delete[] backing_store_;
}
bool Reader::SkipToInitialBlock() { bool Reader::SkipToInitialBlock() {
const size_t offset_in_block = initial_offset_ % kBlockSize; const size_t offset_in_block = initial_offset_ % kBlockSize;
@ -176,9 +173,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
return false; return false;
} }
uint64_t Reader::LastRecordOffset() { uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
return last_record_offset_;
}
void Reader::ReportCorruption(uint64_t bytes, const char* reason) { void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
ReportDrop(bytes, Status::Corruption(reason)); ReportDrop(bytes, Status::Corruption(reason));

View File

@ -43,6 +43,9 @@ class Reader {
Reader(SequentialFile* file, Reporter* reporter, bool checksum, Reader(SequentialFile* file, Reporter* reporter, bool checksum,
uint64_t initial_offset); uint64_t initial_offset);
Reader(const Reader&) = delete;
Reader& operator=(const Reader&) = delete;
~Reader(); ~Reader();
// Read the next record into *record. Returns true if read // Read the next record into *record. Returns true if read
@ -58,26 +61,6 @@ class Reader {
uint64_t LastRecordOffset(); uint64_t LastRecordOffset();
private: private:
SequentialFile* const file_;
Reporter* const reporter_;
bool const checksum_;
char* const backing_store_;
Slice buffer_;
bool eof_; // Last Read() indicated EOF by returning < kBlockSize
// Offset of the last record returned by ReadRecord.
uint64_t last_record_offset_;
// Offset of the first location past the end of buffer_.
uint64_t end_of_buffer_offset_;
// Offset at which to start looking for the first record to return
uint64_t const initial_offset_;
// True if we are resynchronizing after a seek (initial_offset_ > 0). In
// particular, a run of kMiddleType and kLastType records can be silently
// skipped in this mode
bool resyncing_;
// Extend record types with the following special values // Extend record types with the following special values
enum { enum {
kEof = kMaxRecordType + 1, kEof = kMaxRecordType + 1,
@ -102,9 +85,25 @@ class Reader {
void ReportCorruption(uint64_t bytes, const char* reason); void ReportCorruption(uint64_t bytes, const char* reason);
void ReportDrop(uint64_t bytes, const Status& reason); void ReportDrop(uint64_t bytes, const Status& reason);
// No copying allowed SequentialFile* const file_;
Reader(const Reader&); Reporter* const reporter_;
void operator=(const Reader&); bool const checksum_;
char* const backing_store_;
Slice buffer_;
bool eof_; // Last Read() indicated EOF by returning < kBlockSize
// Offset of the last record returned by ReadRecord.
uint64_t last_record_offset_;
// Offset of the first location past the end of buffer_.
uint64_t end_of_buffer_offset_;
// Offset at which to start looking for the first record to return
uint64_t const initial_offset_;
// True if we are resynchronizing after a seek (initial_offset_ > 0). In
// particular, a run of kMiddleType and kLastType records can be silently
// skipped in this mode
bool resyncing_;
}; };
} // namespace log } // namespace log

View File

@ -2,13 +2,13 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "gtest/gtest.h"
#include "db/log_reader.h" #include "db/log_reader.h"
#include "db/log_writer.h" #include "db/log_writer.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/crc32c.h" #include "util/crc32c.h"
#include "util/random.h" #include "util/random.h"
#include "util/testharness.h"
namespace leveldb { namespace leveldb {
namespace log { namespace log {
@ -36,88 +36,13 @@ static std::string RandomSkewedString(int i, Random* rnd) {
return BigString(NumberString(i), rnd->Skewed(17)); return BigString(NumberString(i), rnd->Skewed(17));
} }
class LogTest { class LogTest : public testing::Test {
private:
class StringDest : public WritableFile {
public: public:
std::string contents_; LogTest()
: reading_(false),
virtual Status Close() { return Status::OK(); }
virtual Status Flush() { return Status::OK(); }
virtual Status Sync() { return Status::OK(); }
virtual Status Append(const Slice& slice) {
contents_.append(slice.data(), slice.size());
return Status::OK();
}
};
class StringSource : public SequentialFile {
public:
Slice contents_;
bool force_error_;
bool returned_partial_;
StringSource() : force_error_(false), returned_partial_(false) { }
virtual Status Read(size_t n, Slice* result, char* scratch) {
ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error";
if (force_error_) {
force_error_ = false;
returned_partial_ = true;
return Status::Corruption("read error");
}
if (contents_.size() < n) {
n = contents_.size();
returned_partial_ = true;
}
*result = Slice(contents_.data(), n);
contents_.remove_prefix(n);
return Status::OK();
}
virtual Status Skip(uint64_t n) {
if (n > contents_.size()) {
contents_.clear();
return Status::NotFound("in-memory file skipped past end");
}
contents_.remove_prefix(n);
return Status::OK();
}
};
class ReportCollector : public Reader::Reporter {
public:
size_t dropped_bytes_;
std::string message_;
ReportCollector() : dropped_bytes_(0) { }
virtual void Corruption(size_t bytes, const Status& status) {
dropped_bytes_ += bytes;
message_.append(status.ToString());
}
};
StringDest dest_;
StringSource source_;
ReportCollector report_;
bool reading_;
Writer* writer_;
Reader* reader_;
// Record metadata for testing initial offset functionality
static size_t initial_offset_record_sizes_[];
static uint64_t initial_offset_last_record_offsets_[];
static int num_initial_offset_records_;
public:
LogTest() : reading_(false),
writer_(new Writer(&dest_)), writer_(new Writer(&dest_)),
reader_(new Reader(&source_, &report_, true /*checksum*/, reader_(new Reader(&source_, &report_, true /*checksum*/,
0/*initial_offset*/)) { 0 /*initial_offset*/)) {}
}
~LogTest() { ~LogTest() {
delete writer_; delete writer_;
@ -134,9 +59,7 @@ class LogTest {
writer_->AddRecord(Slice(msg)); writer_->AddRecord(Slice(msg));
} }
size_t WrittenBytes() const { size_t WrittenBytes() const { return dest_.contents_.size(); }
return dest_.contents_.size();
}
std::string Read() { std::string Read() {
if (!reading_) { if (!reading_) {
@ -171,17 +94,11 @@ class LogTest {
EncodeFixed32(&dest_.contents_[header_offset], crc); EncodeFixed32(&dest_.contents_[header_offset], crc);
} }
void ForceError() { void ForceError() { source_.force_error_ = true; }
source_.force_error_ = true;
}
size_t DroppedBytes() const { size_t DroppedBytes() const { return report_.dropped_bytes_; }
return report_.dropped_bytes_;
}
std::string ReportMessage() const { std::string ReportMessage() const { return report_.message_; }
return report_.message_;
}
// Returns OK iff recorded error message contains "msg" // Returns OK iff recorded error message contains "msg"
std::string MatchError(const std::string& msg) const { std::string MatchError(const std::string& msg) const {
@ -222,8 +139,8 @@ class LogTest {
WriteInitialOffsetLog(); WriteInitialOffsetLog();
reading_ = true; reading_ = true;
source_.contents_ = Slice(dest_.contents_); source_.contents_ = Slice(dest_.contents_);
Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, Reader* offset_reader =
initial_offset); new Reader(&source_, &report_, true /*checksum*/, initial_offset);
// Read all records from expected_record_offset through the last one. // Read all records from expected_record_offset through the last one.
ASSERT_LT(expected_record_offset, num_initial_offset_records_); ASSERT_LT(expected_record_offset, num_initial_offset_records_);
@ -240,10 +157,86 @@ class LogTest {
} }
delete offset_reader; delete offset_reader;
} }
private:
class StringDest : public WritableFile {
public:
Status Close() override { return Status::OK(); }
Status Flush() override { return Status::OK(); }
Status Sync() override { return Status::OK(); }
Status Append(const Slice& slice) override {
contents_.append(slice.data(), slice.size());
return Status::OK();
}
std::string contents_;
}; };
size_t LogTest::initial_offset_record_sizes_[] = class StringSource : public SequentialFile {
{10000, // Two sizable records in first block public:
StringSource() : force_error_(false), returned_partial_(false) {}
Status Read(size_t n, Slice* result, char* scratch) override {
EXPECT_TRUE(!returned_partial_) << "must not Read() after eof/error";
if (force_error_) {
force_error_ = false;
returned_partial_ = true;
return Status::Corruption("read error");
}
if (contents_.size() < n) {
n = contents_.size();
returned_partial_ = true;
}
*result = Slice(contents_.data(), n);
contents_.remove_prefix(n);
return Status::OK();
}
Status Skip(uint64_t n) override {
if (n > contents_.size()) {
contents_.clear();
return Status::NotFound("in-memory file skipped past end");
}
contents_.remove_prefix(n);
return Status::OK();
}
Slice contents_;
bool force_error_;
bool returned_partial_;
};
class ReportCollector : public Reader::Reporter {
public:
ReportCollector() : dropped_bytes_(0) {}
void Corruption(size_t bytes, const Status& status) override {
dropped_bytes_ += bytes;
message_.append(status.ToString());
}
size_t dropped_bytes_;
std::string message_;
};
// Record metadata for testing initial offset functionality
static size_t initial_offset_record_sizes_[];
static uint64_t initial_offset_last_record_offsets_[];
static int num_initial_offset_records_;
StringDest dest_;
StringSource source_;
ReportCollector report_;
bool reading_;
Writer* writer_;
Reader* reader_;
};
size_t LogTest::initial_offset_record_sizes_[] = {
10000, // Two sizable records in first block
10000, 10000,
2 * log::kBlockSize - 1000, // Span three blocks 2 * log::kBlockSize - 1000, // Span three blocks
1, 1,
@ -251,15 +244,13 @@ size_t LogTest::initial_offset_record_sizes_[] =
log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
}; };
uint64_t LogTest::initial_offset_last_record_offsets_[] = uint64_t LogTest::initial_offset_last_record_offsets_[] = {
{0, 0,
kHeaderSize + 10000, kHeaderSize + 10000,
2 * (kHeaderSize + 10000), 2 * (kHeaderSize + 10000),
2 * (kHeaderSize + 10000) + 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize, 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize +
2 * (kHeaderSize + 10000) + kHeaderSize + 1,
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize
+ kHeaderSize + 1,
3 * log::kBlockSize, 3 * log::kBlockSize,
}; };
@ -267,11 +258,9 @@ uint64_t LogTest::initial_offset_last_record_offsets_[] =
int LogTest::num_initial_offset_records_ = int LogTest::num_initial_offset_records_ =
sizeof(LogTest::initial_offset_last_record_offsets_) / sizeof(uint64_t); sizeof(LogTest::initial_offset_last_record_offsets_) / sizeof(uint64_t);
TEST(LogTest, Empty) { TEST_F(LogTest, Empty) { ASSERT_EQ("EOF", Read()); }
ASSERT_EQ("EOF", Read());
}
TEST(LogTest, ReadWrite) { TEST_F(LogTest, ReadWrite) {
Write("foo"); Write("foo");
Write("bar"); Write("bar");
Write(""); Write("");
@ -284,7 +273,7 @@ TEST(LogTest, ReadWrite) {
ASSERT_EQ("EOF", Read()); // Make sure reads at eof work ASSERT_EQ("EOF", Read()); // Make sure reads at eof work
} }
TEST(LogTest, ManyBlocks) { TEST_F(LogTest, ManyBlocks) {
for (int i = 0; i < 100000; i++) { for (int i = 0; i < 100000; i++) {
Write(NumberString(i)); Write(NumberString(i));
} }
@ -294,7 +283,7 @@ TEST(LogTest, ManyBlocks) {
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
} }
TEST(LogTest, Fragmentation) { TEST_F(LogTest, Fragmentation) {
Write("small"); Write("small");
Write(BigString("medium", 50000)); Write(BigString("medium", 50000));
Write(BigString("large", 100000)); Write(BigString("large", 100000));
@ -304,7 +293,7 @@ TEST(LogTest, Fragmentation) {
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
} }
TEST(LogTest, MarginalTrailer) { TEST_F(LogTest, MarginalTrailer) {
// Make a trailer that is exactly the same length as an empty record. // Make a trailer that is exactly the same length as an empty record.
const int n = kBlockSize - 2 * kHeaderSize; const int n = kBlockSize - 2 * kHeaderSize;
Write(BigString("foo", n)); Write(BigString("foo", n));
@ -317,7 +306,7 @@ TEST(LogTest, MarginalTrailer) {
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
} }
TEST(LogTest, MarginalTrailer2) { TEST_F(LogTest, MarginalTrailer2) {
// Make a trailer that is exactly the same length as an empty record. // Make a trailer that is exactly the same length as an empty record.
const int n = kBlockSize - 2 * kHeaderSize; const int n = kBlockSize - 2 * kHeaderSize;
Write(BigString("foo", n)); Write(BigString("foo", n));
@ -330,7 +319,7 @@ TEST(LogTest, MarginalTrailer2) {
ASSERT_EQ("", ReportMessage()); ASSERT_EQ("", ReportMessage());
} }
TEST(LogTest, ShortTrailer) { TEST_F(LogTest, ShortTrailer) {
const int n = kBlockSize - 2 * kHeaderSize + 4; const int n = kBlockSize - 2 * kHeaderSize + 4;
Write(BigString("foo", n)); Write(BigString("foo", n));
ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
@ -342,7 +331,7 @@ TEST(LogTest, ShortTrailer) {
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
} }
TEST(LogTest, AlignedEof) { TEST_F(LogTest, AlignedEof) {
const int n = kBlockSize - 2 * kHeaderSize + 4; const int n = kBlockSize - 2 * kHeaderSize + 4;
Write(BigString("foo", n)); Write(BigString("foo", n));
ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
@ -350,7 +339,7 @@ TEST(LogTest, AlignedEof) {
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
} }
TEST(LogTest, OpenForAppend) { TEST_F(LogTest, OpenForAppend) {
Write("hello"); Write("hello");
ReopenForAppend(); ReopenForAppend();
Write("world"); Write("world");
@ -359,7 +348,7 @@ TEST(LogTest, OpenForAppend) {
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
} }
TEST(LogTest, RandomRead) { TEST_F(LogTest, RandomRead) {
const int N = 500; const int N = 500;
Random write_rnd(301); Random write_rnd(301);
for (int i = 0; i < N; i++) { for (int i = 0; i < N; i++) {
@ -374,7 +363,7 @@ TEST(LogTest, RandomRead) {
// Tests of all the error paths in log_reader.cc follow: // Tests of all the error paths in log_reader.cc follow:
TEST(LogTest, ReadError) { TEST_F(LogTest, ReadError) {
Write("foo"); Write("foo");
ForceError(); ForceError();
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
@ -382,7 +371,7 @@ TEST(LogTest, ReadError) {
ASSERT_EQ("OK", MatchError("read error")); ASSERT_EQ("OK", MatchError("read error"));
} }
TEST(LogTest, BadRecordType) { TEST_F(LogTest, BadRecordType) {
Write("foo"); Write("foo");
// Type is stored in header[6] // Type is stored in header[6]
IncrementByte(6, 100); IncrementByte(6, 100);
@ -392,7 +381,7 @@ TEST(LogTest, BadRecordType) {
ASSERT_EQ("OK", MatchError("unknown record type")); ASSERT_EQ("OK", MatchError("unknown record type"));
} }
TEST(LogTest, TruncatedTrailingRecordIsIgnored) { TEST_F(LogTest, TruncatedTrailingRecordIsIgnored) {
Write("foo"); Write("foo");
ShrinkSize(4); // Drop all payload as well as a header byte ShrinkSize(4); // Drop all payload as well as a header byte
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
@ -401,7 +390,7 @@ TEST(LogTest, TruncatedTrailingRecordIsIgnored) {
ASSERT_EQ("", ReportMessage()); ASSERT_EQ("", ReportMessage());
} }
TEST(LogTest, BadLength) { TEST_F(LogTest, BadLength) {
const int kPayloadSize = kBlockSize - kHeaderSize; const int kPayloadSize = kBlockSize - kHeaderSize;
Write(BigString("bar", kPayloadSize)); Write(BigString("bar", kPayloadSize));
Write("foo"); Write("foo");
@ -412,7 +401,7 @@ TEST(LogTest, BadLength) {
ASSERT_EQ("OK", MatchError("bad record length")); ASSERT_EQ("OK", MatchError("bad record length"));
} }
TEST(LogTest, BadLengthAtEndIsIgnored) { TEST_F(LogTest, BadLengthAtEndIsIgnored) {
Write("foo"); Write("foo");
ShrinkSize(1); ShrinkSize(1);
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
@ -420,7 +409,7 @@ TEST(LogTest, BadLengthAtEndIsIgnored) {
ASSERT_EQ("", ReportMessage()); ASSERT_EQ("", ReportMessage());
} }
TEST(LogTest, ChecksumMismatch) { TEST_F(LogTest, ChecksumMismatch) {
Write("foo"); Write("foo");
IncrementByte(0, 10); IncrementByte(0, 10);
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
@ -428,7 +417,7 @@ TEST(LogTest, ChecksumMismatch) {
ASSERT_EQ("OK", MatchError("checksum mismatch")); ASSERT_EQ("OK", MatchError("checksum mismatch"));
} }
TEST(LogTest, UnexpectedMiddleType) { TEST_F(LogTest, UnexpectedMiddleType) {
Write("foo"); Write("foo");
SetByte(6, kMiddleType); SetByte(6, kMiddleType);
FixChecksum(0, 3); FixChecksum(0, 3);
@ -437,7 +426,7 @@ TEST(LogTest, UnexpectedMiddleType) {
ASSERT_EQ("OK", MatchError("missing start")); ASSERT_EQ("OK", MatchError("missing start"));
} }
TEST(LogTest, UnexpectedLastType) { TEST_F(LogTest, UnexpectedLastType) {
Write("foo"); Write("foo");
SetByte(6, kLastType); SetByte(6, kLastType);
FixChecksum(0, 3); FixChecksum(0, 3);
@ -446,7 +435,7 @@ TEST(LogTest, UnexpectedLastType) {
ASSERT_EQ("OK", MatchError("missing start")); ASSERT_EQ("OK", MatchError("missing start"));
} }
TEST(LogTest, UnexpectedFullType) { TEST_F(LogTest, UnexpectedFullType) {
Write("foo"); Write("foo");
Write("bar"); Write("bar");
SetByte(6, kFirstType); SetByte(6, kFirstType);
@ -457,7 +446,7 @@ TEST(LogTest, UnexpectedFullType) {
ASSERT_EQ("OK", MatchError("partial record without end")); ASSERT_EQ("OK", MatchError("partial record without end"));
} }
TEST(LogTest, UnexpectedFirstType) { TEST_F(LogTest, UnexpectedFirstType) {
Write("foo"); Write("foo");
Write(BigString("bar", 100000)); Write(BigString("bar", 100000));
SetByte(6, kFirstType); SetByte(6, kFirstType);
@ -468,7 +457,7 @@ TEST(LogTest, UnexpectedFirstType) {
ASSERT_EQ("OK", MatchError("partial record without end")); ASSERT_EQ("OK", MatchError("partial record without end"));
} }
TEST(LogTest, MissingLastIsIgnored) { TEST_F(LogTest, MissingLastIsIgnored) {
Write(BigString("bar", kBlockSize)); Write(BigString("bar", kBlockSize));
// Remove the LAST block, including header. // Remove the LAST block, including header.
ShrinkSize(14); ShrinkSize(14);
@ -477,7 +466,7 @@ TEST(LogTest, MissingLastIsIgnored) {
ASSERT_EQ(0, DroppedBytes()); ASSERT_EQ(0, DroppedBytes());
} }
TEST(LogTest, PartialLastIsIgnored) { TEST_F(LogTest, PartialLastIsIgnored) {
Write(BigString("bar", kBlockSize)); Write(BigString("bar", kBlockSize));
// Cause a bad record length in the LAST block. // Cause a bad record length in the LAST block.
ShrinkSize(1); ShrinkSize(1);
@ -486,7 +475,7 @@ TEST(LogTest, PartialLastIsIgnored) {
ASSERT_EQ(0, DroppedBytes()); ASSERT_EQ(0, DroppedBytes());
} }
TEST(LogTest, SkipIntoMultiRecord) { TEST_F(LogTest, SkipIntoMultiRecord) {
// Consider a fragmented record: // Consider a fragmented record:
// first(R1), middle(R1), last(R1), first(R2) // first(R1), middle(R1), last(R1), first(R2)
// If initial_offset points to a record after first(R1) but before first(R2) // If initial_offset points to a record after first(R1) but before first(R2)
@ -502,7 +491,7 @@ TEST(LogTest, SkipIntoMultiRecord) {
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
} }
TEST(LogTest, ErrorJoinsRecords) { TEST_F(LogTest, ErrorJoinsRecords) {
// Consider two fragmented records: // Consider two fragmented records:
// first(R1) last(R1) first(R2) last(R2) // first(R1) last(R1) first(R2) last(R2)
// where the middle two fragments disappear. We do not want // where the middle two fragments disappear. We do not want
@ -525,67 +514,50 @@ TEST(LogTest, ErrorJoinsRecords) {
ASSERT_GE(dropped, 2 * kBlockSize); ASSERT_GE(dropped, 2 * kBlockSize);
} }
TEST(LogTest, ReadStart) { TEST_F(LogTest, ReadStart) { CheckInitialOffsetRecord(0, 0); }
CheckInitialOffsetRecord(0, 0);
}
TEST(LogTest, ReadSecondOneOff) { TEST_F(LogTest, ReadSecondOneOff) { CheckInitialOffsetRecord(1, 1); }
CheckInitialOffsetRecord(1, 1);
}
TEST(LogTest, ReadSecondTenThousand) { TEST_F(LogTest, ReadSecondTenThousand) { CheckInitialOffsetRecord(10000, 1); }
CheckInitialOffsetRecord(10000, 1);
}
TEST(LogTest, ReadSecondStart) { TEST_F(LogTest, ReadSecondStart) { CheckInitialOffsetRecord(10007, 1); }
CheckInitialOffsetRecord(10007, 1);
}
TEST(LogTest, ReadThirdOneOff) { TEST_F(LogTest, ReadThirdOneOff) { CheckInitialOffsetRecord(10008, 2); }
CheckInitialOffsetRecord(10008, 2);
}
TEST(LogTest, ReadThirdStart) { TEST_F(LogTest, ReadThirdStart) { CheckInitialOffsetRecord(20014, 2); }
CheckInitialOffsetRecord(20014, 2);
}
TEST(LogTest, ReadFourthOneOff) { TEST_F(LogTest, ReadFourthOneOff) { CheckInitialOffsetRecord(20015, 3); }
CheckInitialOffsetRecord(20015, 3);
}
TEST(LogTest, ReadFourthFirstBlockTrailer) { TEST_F(LogTest, ReadFourthFirstBlockTrailer) {
CheckInitialOffsetRecord(log::kBlockSize - 4, 3); CheckInitialOffsetRecord(log::kBlockSize - 4, 3);
} }
TEST(LogTest, ReadFourthMiddleBlock) { TEST_F(LogTest, ReadFourthMiddleBlock) {
CheckInitialOffsetRecord(log::kBlockSize + 1, 3); CheckInitialOffsetRecord(log::kBlockSize + 1, 3);
} }
TEST(LogTest, ReadFourthLastBlock) { TEST_F(LogTest, ReadFourthLastBlock) {
CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3); CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3);
} }
TEST(LogTest, ReadFourthStart) { TEST_F(LogTest, ReadFourthStart) {
CheckInitialOffsetRecord( CheckInitialOffsetRecord(
2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, 2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
3); 3);
} }
TEST(LogTest, ReadInitialOffsetIntoBlockPadding) { TEST_F(LogTest, ReadInitialOffsetIntoBlockPadding) {
CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5); CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5);
} }
TEST(LogTest, ReadEnd) { TEST_F(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); }
CheckOffsetPastEndReturnsNoRecords(0);
}
TEST(LogTest, ReadPastEnd) { TEST_F(LogTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); }
CheckOffsetPastEndReturnsNoRecords(5);
}
} // namespace log } // namespace log
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -5,6 +5,7 @@
#include "db/log_writer.h" #include "db/log_writer.h"
#include <stdint.h> #include <stdint.h>
#include "leveldb/env.h" #include "leveldb/env.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/crc32c.h" #include "util/crc32c.h"
@ -19,9 +20,7 @@ static void InitTypeCrc(uint32_t* type_crc) {
} }
} }
Writer::Writer(WritableFile* dest) Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) {
: dest_(dest),
block_offset_(0) {
InitTypeCrc(type_crc_); InitTypeCrc(type_crc_);
} }
@ -30,8 +29,7 @@ Writer::Writer(WritableFile* dest, uint64_t dest_length)
InitTypeCrc(type_crc_); InitTypeCrc(type_crc_);
} }
Writer::~Writer() { Writer::~Writer() = default;
}
Status Writer::AddRecord(const Slice& slice) { Status Writer::AddRecord(const Slice& slice) {
const char* ptr = slice.data(); const char* ptr = slice.data();
@ -49,7 +47,7 @@ Status Writer::AddRecord(const Slice& slice) {
// Switch to a new block // Switch to a new block
if (leftover > 0) { if (leftover > 0) {
// Fill the trailer (literal below relies on kHeaderSize being 7) // Fill the trailer (literal below relies on kHeaderSize being 7)
assert(kHeaderSize == 7); static_assert(kHeaderSize == 7, "");
dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover)); dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover));
} }
block_offset_ = 0; block_offset_ = 0;
@ -81,30 +79,31 @@ Status Writer::AddRecord(const Slice& slice) {
return s; return s;
} }
Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) { Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr,
assert(n <= 0xffff); // Must fit in two bytes size_t length) {
assert(block_offset_ + kHeaderSize + n <= kBlockSize); assert(length <= 0xffff); // Must fit in two bytes
assert(block_offset_ + kHeaderSize + length <= kBlockSize);
// Format the header // Format the header
char buf[kHeaderSize]; char buf[kHeaderSize];
buf[4] = static_cast<char>(n & 0xff); buf[4] = static_cast<char>(length & 0xff);
buf[5] = static_cast<char>(n >> 8); buf[5] = static_cast<char>(length >> 8);
buf[6] = static_cast<char>(t); buf[6] = static_cast<char>(t);
// Compute the crc of the record type and the payload. // Compute the crc of the record type and the payload.
uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n); uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length);
crc = crc32c::Mask(crc); // Adjust for storage crc = crc32c::Mask(crc); // Adjust for storage
EncodeFixed32(buf, crc); EncodeFixed32(buf, crc);
// Write the header and the payload // Write the header and the payload
Status s = dest_->Append(Slice(buf, kHeaderSize)); Status s = dest_->Append(Slice(buf, kHeaderSize));
if (s.ok()) { if (s.ok()) {
s = dest_->Append(Slice(ptr, n)); s = dest_->Append(Slice(ptr, length));
if (s.ok()) { if (s.ok()) {
s = dest_->Flush(); s = dest_->Flush();
} }
} }
block_offset_ += kHeaderSize + n; block_offset_ += kHeaderSize + length;
return s; return s;
} }

View File

@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_DB_LOG_WRITER_H_ #define STORAGE_LEVELDB_DB_LOG_WRITER_H_
#include <stdint.h> #include <stdint.h>
#include "db/log_format.h" #include "db/log_format.h"
#include "leveldb/slice.h" #include "leveldb/slice.h"
#include "leveldb/status.h" #include "leveldb/status.h"
@ -28,11 +29,16 @@ class Writer {
// "*dest" must remain live while this Writer is in use. // "*dest" must remain live while this Writer is in use.
Writer(WritableFile* dest, uint64_t dest_length); Writer(WritableFile* dest, uint64_t dest_length);
Writer(const Writer&) = delete;
Writer& operator=(const Writer&) = delete;
~Writer(); ~Writer();
Status AddRecord(const Slice& slice); Status AddRecord(const Slice& slice);
private: private:
Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
WritableFile* dest_; WritableFile* dest_;
int block_offset_; // Current offset in block int block_offset_; // Current offset in block
@ -40,12 +46,6 @@ class Writer {
// pre-computed to reduce the overhead of computing the crc of the // pre-computed to reduce the overhead of computing the crc of the
// record type stored in the header. // record type stored in the header.
uint32_t type_crc_[kMaxRecordType + 1]; uint32_t type_crc_[kMaxRecordType + 1];
Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
// No copying allowed
Writer(const Writer&);
void operator=(const Writer&);
}; };
} // namespace log } // namespace log

View File

@ -18,20 +18,15 @@ static Slice GetLengthPrefixedSlice(const char* data) {
return Slice(p, len); return Slice(p, len);
} }
MemTable::MemTable(const InternalKeyComparator& cmp) MemTable::MemTable(const InternalKeyComparator& comparator)
: comparator_(cmp), : comparator_(comparator), refs_(0), table_(comparator_, &arena_) {}
refs_(0),
table_(comparator_, &arena_) {
}
MemTable::~MemTable() { MemTable::~MemTable() { assert(refs_ == 0); }
assert(refs_ == 0);
}
size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); } size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr) int MemTable::KeyComparator::operator()(const char* aptr,
const { const char* bptr) const {
// Internal keys are encoded as length-prefixed strings. // Internal keys are encoded as length-prefixed strings.
Slice a = GetLengthPrefixedSlice(aptr); Slice a = GetLengthPrefixedSlice(aptr);
Slice b = GetLengthPrefixedSlice(bptr); Slice b = GetLengthPrefixedSlice(bptr);
@ -52,35 +47,33 @@ class MemTableIterator: public Iterator {
public: public:
explicit MemTableIterator(MemTable::Table* table) : iter_(table) {} explicit MemTableIterator(MemTable::Table* table) : iter_(table) {}
virtual bool Valid() const { return iter_.Valid(); } MemTableIterator(const MemTableIterator&) = delete;
virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); } MemTableIterator& operator=(const MemTableIterator&) = delete;
virtual void SeekToFirst() { iter_.SeekToFirst(); }
virtual void SeekToLast() { iter_.SeekToLast(); } ~MemTableIterator() override = default;
virtual void Next() { iter_.Next(); }
virtual void Prev() { iter_.Prev(); } bool Valid() const override { return iter_.Valid(); }
virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); } void Seek(const Slice& k) override { iter_.Seek(EncodeKey(&tmp_, k)); }
virtual Slice value() const { void SeekToFirst() override { iter_.SeekToFirst(); }
void SeekToLast() override { iter_.SeekToLast(); }
void Next() override { iter_.Next(); }
void Prev() override { iter_.Prev(); }
Slice key() const override { return GetLengthPrefixedSlice(iter_.key()); }
Slice value() const override {
Slice key_slice = GetLengthPrefixedSlice(iter_.key()); Slice key_slice = GetLengthPrefixedSlice(iter_.key());
return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
} }
virtual Status status() const { return Status::OK(); } Status status() const override { return Status::OK(); }
private: private:
MemTable::Table::Iterator iter_; MemTable::Table::Iterator iter_;
std::string tmp_; // For passing to EncodeKey std::string tmp_; // For passing to EncodeKey
// No copying allowed
MemTableIterator(const MemTableIterator&);
void operator=(const MemTableIterator&);
}; };
Iterator* MemTable::NewIterator() { Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); }
return new MemTableIterator(&table_);
}
void MemTable::Add(SequenceNumber s, ValueType type, void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key,
const Slice& key,
const Slice& value) { const Slice& value) {
// Format of an entry is concatenation of: // Format of an entry is concatenation of:
// key_size : varint32 of internal_key.size() // key_size : varint32 of internal_key.size()
@ -90,9 +83,9 @@ void MemTable::Add(SequenceNumber s, ValueType type,
size_t key_size = key.size(); size_t key_size = key.size();
size_t val_size = value.size(); size_t val_size = value.size();
size_t internal_key_size = key_size + 8; size_t internal_key_size = key_size + 8;
const size_t encoded_len = const size_t encoded_len = VarintLength(internal_key_size) +
VarintLength(internal_key_size) + internal_key_size + internal_key_size + VarintLength(val_size) +
VarintLength(val_size) + val_size; val_size;
char* buf = arena_.Allocate(encoded_len); char* buf = arena_.Allocate(encoded_len);
char* p = EncodeVarint32(buf, internal_key_size); char* p = EncodeVarint32(buf, internal_key_size);
memcpy(p, key.data(), key_size); memcpy(p, key.data(), key_size);
@ -123,8 +116,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
uint32_t key_length; uint32_t key_length;
const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
if (comparator_.comparator.user_comparator()->Compare( if (comparator_.comparator.user_comparator()->Compare(
Slice(key_ptr, key_length - 8), Slice(key_ptr, key_length - 8), key.user_key()) == 0) {
key.user_key()) == 0) {
// Correct user key // Correct user key
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
switch (static_cast<ValueType>(tag & 0xff)) { switch (static_cast<ValueType>(tag & 0xff)) {

View File

@ -6,9 +6,10 @@
#define STORAGE_LEVELDB_DB_MEMTABLE_H_ #define STORAGE_LEVELDB_DB_MEMTABLE_H_
#include <string> #include <string>
#include "leveldb/db.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/skiplist.h" #include "db/skiplist.h"
#include "leveldb/db.h"
#include "util/arena.h" #include "util/arena.h"
namespace leveldb { namespace leveldb {
@ -22,6 +23,9 @@ class MemTable {
// is zero and the caller must call Ref() at least once. // is zero and the caller must call Ref() at least once.
explicit MemTable(const InternalKeyComparator& comparator); explicit MemTable(const InternalKeyComparator& comparator);
MemTable(const MemTable&) = delete;
MemTable& operator=(const MemTable&) = delete;
// Increase reference count. // Increase reference count.
void Ref() { ++refs_; } void Ref() { ++refs_; }
@ -49,8 +53,7 @@ class MemTable {
// Add an entry into memtable that maps key to value at the // Add an entry into memtable that maps key to value at the
// specified sequence number and with the specified type. // specified sequence number and with the specified type.
// Typically value will be empty if type==kTypeDeletion. // Typically value will be empty if type==kTypeDeletion.
void Add(SequenceNumber seq, ValueType type, void Add(SequenceNumber seq, ValueType type, const Slice& key,
const Slice& key,
const Slice& value); const Slice& value);
// If memtable contains a value for key, store it in *value and return true. // If memtable contains a value for key, store it in *value and return true.
@ -60,26 +63,23 @@ class MemTable {
bool Get(const LookupKey& key, std::string* value, Status* s); bool Get(const LookupKey& key, std::string* value, Status* s);
private: private:
~MemTable(); // Private since only Unref() should be used to delete it friend class MemTableIterator;
friend class MemTableBackwardIterator;
struct KeyComparator { struct KeyComparator {
const InternalKeyComparator comparator; const InternalKeyComparator comparator;
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {} explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
int operator()(const char* a, const char* b) const; int operator()(const char* a, const char* b) const;
}; };
friend class MemTableIterator;
friend class MemTableBackwardIterator;
typedef SkipList<const char*, KeyComparator> Table; typedef SkipList<const char*, KeyComparator> Table;
~MemTable(); // Private since only Unref() should be used to delete it
KeyComparator comparator_; KeyComparator comparator_;
int refs_; int refs_;
Arena arena_; Arena arena_;
Table table_; Table table_;
// No copying allowed
MemTable(const MemTable&);
void operator=(const MemTable&);
}; };
} // namespace leveldb } // namespace leveldb

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "gtest/gtest.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/filename.h" #include "db/filename.h"
#include "db/version_set.h" #include "db/version_set.h"
@ -10,15 +11,14 @@
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/write_batch.h" #include "leveldb/write_batch.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
class RecoveryTest { class RecoveryTest : public testing::Test {
public: public:
RecoveryTest() : env_(Env::Default()), db_(nullptr) { RecoveryTest() : env_(Env::Default()), db_(nullptr) {
dbname_ = test::TmpDir() + "/recovery_test"; dbname_ = testing::TempDir() + "/recovery_test";
DestroyDB(dbname_, Options()); DestroyDB(dbname_, Options());
Open(); Open();
} }
@ -63,7 +63,7 @@ class RecoveryTest {
} }
void Open(Options* options = nullptr) { void Open(Options* options = nullptr) {
ASSERT_OK(OpenWithStatus(options)); ASSERT_LEVELDB_OK(OpenWithStatus(options));
ASSERT_EQ(1, NumLogs()); ASSERT_EQ(1, NumLogs());
} }
@ -84,7 +84,8 @@ class RecoveryTest {
std::string ManifestFileName() { std::string ManifestFileName() {
std::string current; std::string current;
ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), &current)); EXPECT_LEVELDB_OK(
ReadFileToString(env_, CurrentFileName(dbname_), &current));
size_t len = current.size(); size_t len = current.size();
if (len > 0 && current[len - 1] == '\n') { if (len > 0 && current[len - 1] == '\n') {
current.resize(len - 1); current.resize(len - 1);
@ -92,29 +93,28 @@ class RecoveryTest {
return dbname_ + "/" + current; return dbname_ + "/" + current;
} }
std::string LogName(uint64_t number) { std::string LogName(uint64_t number) { return LogFileName(dbname_, number); }
return LogFileName(dbname_, number);
}
size_t DeleteLogFiles() { size_t DeleteLogFiles() {
// Linux allows unlinking open files, but Windows does not.
// Closing the db allows for file deletion.
Close();
std::vector<uint64_t> logs = GetFiles(kLogFile); std::vector<uint64_t> logs = GetFiles(kLogFile);
for (size_t i = 0; i < logs.size(); i++) { for (size_t i = 0; i < logs.size(); i++) {
ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); EXPECT_LEVELDB_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
} }
return logs.size(); return logs.size();
} }
void DeleteManifestFile() { void DeleteManifestFile() {
ASSERT_OK(env_->DeleteFile(ManifestFileName())); ASSERT_LEVELDB_OK(env_->DeleteFile(ManifestFileName()));
} }
uint64_t FirstLogFile() { uint64_t FirstLogFile() { return GetFiles(kLogFile)[0]; }
return GetFiles(kLogFile)[0];
}
std::vector<uint64_t> GetFiles(FileType t) { std::vector<uint64_t> GetFiles(FileType t) {
std::vector<std::string> filenames; std::vector<std::string> filenames;
ASSERT_OK(env_->GetChildren(dbname_, &filenames)); EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames));
std::vector<uint64_t> result; std::vector<uint64_t> result;
for (size_t i = 0; i < filenames.size(); i++) { for (size_t i = 0; i < filenames.size(); i++) {
uint64_t number; uint64_t number;
@ -126,35 +126,29 @@ class RecoveryTest {
return result; return result;
} }
int NumLogs() { int NumLogs() { return GetFiles(kLogFile).size(); }
return GetFiles(kLogFile).size();
}
int NumTables() { int NumTables() { return GetFiles(kTableFile).size(); }
return GetFiles(kTableFile).size();
}
uint64_t FileSize(const std::string& fname) { uint64_t FileSize(const std::string& fname) {
uint64_t result; uint64_t result;
ASSERT_OK(env_->GetFileSize(fname, &result)) << fname; EXPECT_LEVELDB_OK(env_->GetFileSize(fname, &result)) << fname;
return result; return result;
} }
void CompactMemTable() { void CompactMemTable() { dbfull()->TEST_CompactMemTable(); }
dbfull()->TEST_CompactMemTable();
}
// Directly construct a log file that sets key to val. // Directly construct a log file that sets key to val.
void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
std::string fname = LogFileName(dbname_, lognum); std::string fname = LogFileName(dbname_, lognum);
WritableFile* file; WritableFile* file;
ASSERT_OK(env_->NewWritableFile(fname, &file)); ASSERT_LEVELDB_OK(env_->NewWritableFile(fname, &file));
log::Writer writer(file); log::Writer writer(file);
WriteBatch batch; WriteBatch batch;
batch.Put(key, val); batch.Put(key, val);
WriteBatchInternal::SetSequence(&batch, seq); WriteBatchInternal::SetSequence(&batch, seq);
ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); ASSERT_LEVELDB_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
ASSERT_OK(file->Flush()); ASSERT_LEVELDB_OK(file->Flush());
delete file; delete file;
} }
@ -164,12 +158,12 @@ class RecoveryTest {
DB* db_; DB* db_;
}; };
TEST(RecoveryTest, ManifestReused) { TEST_F(RecoveryTest, ManifestReused) {
if (!CanAppend()) { if (!CanAppend()) {
fprintf(stderr, "skipping test because env does not support appending\n"); fprintf(stderr, "skipping test because env does not support appending\n");
return; return;
} }
ASSERT_OK(Put("foo", "bar")); ASSERT_LEVELDB_OK(Put("foo", "bar"));
Close(); Close();
std::string old_manifest = ManifestFileName(); std::string old_manifest = ManifestFileName();
Open(); Open();
@ -180,12 +174,12 @@ TEST(RecoveryTest, ManifestReused) {
ASSERT_EQ("bar", Get("foo")); ASSERT_EQ("bar", Get("foo"));
} }
TEST(RecoveryTest, LargeManifestCompacted) { TEST_F(RecoveryTest, LargeManifestCompacted) {
if (!CanAppend()) { if (!CanAppend()) {
fprintf(stderr, "skipping test because env does not support appending\n"); fprintf(stderr, "skipping test because env does not support appending\n");
return; return;
} }
ASSERT_OK(Put("foo", "bar")); ASSERT_LEVELDB_OK(Put("foo", "bar"));
Close(); Close();
std::string old_manifest = ManifestFileName(); std::string old_manifest = ManifestFileName();
@ -193,10 +187,10 @@ TEST(RecoveryTest, LargeManifestCompacted) {
{ {
uint64_t len = FileSize(old_manifest); uint64_t len = FileSize(old_manifest);
WritableFile* file; WritableFile* file;
ASSERT_OK(env()->NewAppendableFile(old_manifest, &file)); ASSERT_LEVELDB_OK(env()->NewAppendableFile(old_manifest, &file));
std::string zeroes(3 * 1048576 - static_cast<size_t>(len), 0); std::string zeroes(3 * 1048576 - static_cast<size_t>(len), 0);
ASSERT_OK(file->Append(zeroes)); ASSERT_LEVELDB_OK(file->Append(zeroes));
ASSERT_OK(file->Flush()); ASSERT_LEVELDB_OK(file->Flush());
delete file; delete file;
} }
@ -211,8 +205,8 @@ TEST(RecoveryTest, LargeManifestCompacted) {
ASSERT_EQ("bar", Get("foo")); ASSERT_EQ("bar", Get("foo"));
} }
TEST(RecoveryTest, NoLogFiles) { TEST_F(RecoveryTest, NoLogFiles) {
ASSERT_OK(Put("foo", "bar")); ASSERT_LEVELDB_OK(Put("foo", "bar"));
ASSERT_EQ(1, DeleteLogFiles()); ASSERT_EQ(1, DeleteLogFiles());
Open(); Open();
ASSERT_EQ("NOT_FOUND", Get("foo")); ASSERT_EQ("NOT_FOUND", Get("foo"));
@ -220,13 +214,13 @@ TEST(RecoveryTest, NoLogFiles) {
ASSERT_EQ("NOT_FOUND", Get("foo")); ASSERT_EQ("NOT_FOUND", Get("foo"));
} }
TEST(RecoveryTest, LogFileReuse) { TEST_F(RecoveryTest, LogFileReuse) {
if (!CanAppend()) { if (!CanAppend()) {
fprintf(stderr, "skipping test because env does not support appending\n"); fprintf(stderr, "skipping test because env does not support appending\n");
return; return;
} }
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
ASSERT_OK(Put("foo", "bar")); ASSERT_LEVELDB_OK(Put("foo", "bar"));
if (i == 0) { if (i == 0) {
// Compact to ensure current log is empty // Compact to ensure current log is empty
CompactMemTable(); CompactMemTable();
@ -250,13 +244,13 @@ TEST(RecoveryTest, LogFileReuse) {
} }
} }
TEST(RecoveryTest, MultipleMemTables) { TEST_F(RecoveryTest, MultipleMemTables) {
// Make a large log. // Make a large log.
const int kNum = 1000; const int kNum = 1000;
for (int i = 0; i < kNum; i++) { for (int i = 0; i < kNum; i++) {
char buf[100]; char buf[100];
snprintf(buf, sizeof(buf), "%050d", i); snprintf(buf, sizeof(buf), "%050d", i);
ASSERT_OK(Put(buf, buf)); ASSERT_LEVELDB_OK(Put(buf, buf));
} }
ASSERT_EQ(0, NumTables()); ASSERT_EQ(0, NumTables());
Close(); Close();
@ -279,8 +273,8 @@ TEST(RecoveryTest, MultipleMemTables) {
} }
} }
TEST(RecoveryTest, MultipleLogFiles) { TEST_F(RecoveryTest, MultipleLogFiles) {
ASSERT_OK(Put("foo", "bar")); ASSERT_LEVELDB_OK(Put("foo", "bar"));
Close(); Close();
ASSERT_EQ(1, NumLogs()); ASSERT_EQ(1, NumLogs());
@ -325,8 +319,8 @@ TEST(RecoveryTest, MultipleLogFiles) {
ASSERT_EQ("there", Get("hi")); ASSERT_EQ("there", Get("hi"));
} }
TEST(RecoveryTest, ManifestMissing) { TEST_F(RecoveryTest, ManifestMissing) {
ASSERT_OK(Put("foo", "bar")); ASSERT_LEVELDB_OK(Put("foo", "bar"));
Close(); Close();
DeleteManifestFile(); DeleteManifestFile();
@ -337,5 +331,6 @@ TEST(RecoveryTest, ManifestMissing) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -84,9 +84,7 @@ class Repairer {
"recovered %d files; %llu bytes. " "recovered %d files; %llu bytes. "
"Some data may have been lost. " "Some data may have been lost. "
"****", "****",
dbname_.c_str(), dbname_.c_str(), static_cast<int>(tables_.size()), bytes);
static_cast<int>(tables_.size()),
bytes);
} }
return status; return status;
} }
@ -97,22 +95,6 @@ class Repairer {
SequenceNumber max_sequence; SequenceNumber max_sequence;
}; };
std::string const dbname_;
Env* const env_;
InternalKeyComparator const icmp_;
InternalFilterPolicy const ipolicy_;
Options const options_;
bool owns_info_log_;
bool owns_cache_;
TableCache* table_cache_;
VersionEdit edit_;
std::vector<std::string> manifests_;
std::vector<uint64_t> table_numbers_;
std::vector<uint64_t> logs_;
std::vector<TableInfo> tables_;
uint64_t next_file_number_;
Status FindFiles() { Status FindFiles() {
std::vector<std::string> filenames; std::vector<std::string> filenames;
Status status = env_->GetChildren(dbname_, &filenames); Status status = env_->GetChildren(dbname_, &filenames);
@ -152,8 +134,7 @@ class Repairer {
Status status = ConvertLogToTable(logs_[i]); Status status = ConvertLogToTable(logs_[i]);
if (!status.ok()) { if (!status.ok()) {
Log(options_.info_log, "Log #%llu: ignoring conversion error: %s", Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
(unsigned long long) logs_[i], (unsigned long long)logs_[i], status.ToString().c_str());
status.ToString().c_str());
} }
ArchiveFile(logname); ArchiveFile(logname);
} }
@ -164,11 +145,10 @@ class Repairer {
Env* env; Env* env;
Logger* info_log; Logger* info_log;
uint64_t lognum; uint64_t lognum;
virtual void Corruption(size_t bytes, const Status& s) { void Corruption(size_t bytes, const Status& s) override {
// We print error messages for corruption, but continue repairing. // We print error messages for corruption, but continue repairing.
Log(info_log, "Log #%llu: dropping %d bytes; %s", Log(info_log, "Log #%llu: dropping %d bytes; %s",
(unsigned long long) lognum, (unsigned long long)lognum, static_cast<int>(bytes),
static_cast<int>(bytes),
s.ToString().c_str()); s.ToString().c_str());
} }
}; };
@ -202,8 +182,8 @@ class Repairer {
int counter = 0; int counter = 0;
while (reader.ReadRecord(&record, &scratch)) { while (reader.ReadRecord(&record, &scratch)) {
if (record.size() < 12) { if (record.size() < 12) {
reporter.Corruption( reporter.Corruption(record.size(),
record.size(), Status::Corruption("log record too small")); Status::Corruption("log record too small"));
continue; continue;
} }
WriteBatchInternal::SetContents(&batch, record); WriteBatchInternal::SetContents(&batch, record);
@ -212,8 +192,7 @@ class Repairer {
counter += WriteBatchInternal::Count(&batch); counter += WriteBatchInternal::Count(&batch);
} else { } else {
Log(options_.info_log, "Log #%llu: ignoring %s", Log(options_.info_log, "Log #%llu: ignoring %s",
(unsigned long long) log, (unsigned long long)log, status.ToString().c_str());
status.ToString().c_str());
status = Status::OK(); // Keep going with rest of file status = Status::OK(); // Keep going with rest of file
} }
} }
@ -234,9 +213,7 @@ class Repairer {
} }
} }
Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s", Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
(unsigned long long) log, (unsigned long long)log, counter, (unsigned long long)meta.number,
counter,
(unsigned long long) meta.number,
status.ToString().c_str()); status.ToString().c_str());
return status; return status;
} }
@ -272,8 +249,7 @@ class Repairer {
ArchiveFile(TableFileName(dbname_, number)); ArchiveFile(TableFileName(dbname_, number));
ArchiveFile(SSTTableFileName(dbname_, number)); ArchiveFile(SSTTableFileName(dbname_, number));
Log(options_.info_log, "Table #%llu: dropped: %s", Log(options_.info_log, "Table #%llu: dropped: %s",
(unsigned long long) t.meta.number, (unsigned long long)t.meta.number, status.ToString().c_str());
status.ToString().c_str());
return; return;
} }
@ -287,8 +263,7 @@ class Repairer {
Slice key = iter->key(); Slice key = iter->key();
if (!ParseInternalKey(key, &parsed)) { if (!ParseInternalKey(key, &parsed)) {
Log(options_.info_log, "Table #%llu: unparsable key %s", Log(options_.info_log, "Table #%llu: unparsable key %s",
(unsigned long long) t.meta.number, (unsigned long long)t.meta.number, EscapeString(key).c_str());
EscapeString(key).c_str());
continue; continue;
} }
@ -307,9 +282,7 @@ class Repairer {
} }
delete iter; delete iter;
Log(options_.info_log, "Table #%llu: %d entries %s", Log(options_.info_log, "Table #%llu: %d entries %s",
(unsigned long long) t.meta.number, (unsigned long long)t.meta.number, counter, status.ToString().c_str());
counter,
status.ToString().c_str());
if (status.ok()) { if (status.ok()) {
tables_.push_back(t); tables_.push_back(t);
@ -395,8 +368,8 @@ class Repairer {
for (size_t i = 0; i < tables_.size(); i++) { for (size_t i = 0; i < tables_.size(); i++) {
// TODO(opt): separate out into multiple levels // TODO(opt): separate out into multiple levels
const TableInfo& t = tables_[i]; const TableInfo& t = tables_[i];
edit_.AddFile(0, t.meta.number, t.meta.file_size, edit_.AddFile(0, t.meta.number, t.meta.file_size, t.meta.smallest,
t.meta.smallest, t.meta.largest); t.meta.largest);
} }
// fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str()); // fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
@ -447,9 +420,25 @@ class Repairer {
new_file.append("/"); new_file.append("/");
new_file.append((slash == nullptr) ? fname.c_str() : slash + 1); new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
Status s = env_->RenameFile(fname, new_file); Status s = env_->RenameFile(fname, new_file);
Log(options_.info_log, "Archiving %s: %s\n", Log(options_.info_log, "Archiving %s: %s\n", fname.c_str(),
fname.c_str(), s.ToString().c_str()); s.ToString().c_str());
} }
const std::string dbname_;
Env* const env_;
InternalKeyComparator const icmp_;
InternalFilterPolicy const ipolicy_;
const Options options_;
bool owns_info_log_;
bool owns_cache_;
TableCache* table_cache_;
VersionEdit edit_;
std::vector<std::string> manifests_;
std::vector<uint64_t> table_numbers_;
std::vector<uint64_t> logs_;
std::vector<TableInfo> tables_;
uint64_t next_file_number_;
}; };
} // namespace } // namespace

View File

@ -27,9 +27,10 @@
// //
// ... prev vs. next pointer ordering ... // ... prev vs. next pointer ordering ...
#include <assert.h> #include <atomic>
#include <stdlib.h> #include <cassert>
#include "port/port.h" #include <cstdlib>
#include "util/arena.h" #include "util/arena.h"
#include "util/random.h" #include "util/random.h"
@ -48,6 +49,9 @@ class SkipList {
// must remain allocated for the lifetime of the skiplist object. // must remain allocated for the lifetime of the skiplist object.
explicit SkipList(Comparator cmp, Arena* arena); explicit SkipList(Comparator cmp, Arena* arena);
SkipList(const SkipList&) = delete;
SkipList& operator=(const SkipList&) = delete;
// Insert key into the list. // Insert key into the list.
// REQUIRES: nothing that compares equal to key is currently in the list. // REQUIRES: nothing that compares equal to key is currently in the list.
void Insert(const Key& key); void Insert(const Key& key);
@ -97,24 +101,10 @@ class SkipList {
private: private:
enum { kMaxHeight = 12 }; enum { kMaxHeight = 12 };
// Immutable after construction
Comparator const compare_;
Arena* const arena_; // Arena used for allocations of nodes
Node* const head_;
// Modified only by Insert(). Read racily by readers, but stale
// values are ok.
port::AtomicPointer max_height_; // Height of the entire list
inline int GetMaxHeight() const { inline int GetMaxHeight() const {
return static_cast<int>( return max_height_.load(std::memory_order_relaxed);
reinterpret_cast<intptr_t>(max_height_.NoBarrier_Load()));
} }
// Read/written only by Insert().
Random rnd_;
Node* NewNode(const Key& key, int height); Node* NewNode(const Key& key, int height);
int RandomHeight(); int RandomHeight();
bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
@ -137,9 +127,18 @@ class SkipList {
// Return head_ if list is empty. // Return head_ if list is empty.
Node* FindLast() const; Node* FindLast() const;
// No copying allowed // Immutable after construction
SkipList(const SkipList&); Comparator const compare_;
void operator=(const SkipList&); Arena* const arena_; // Arena used for allocations of nodes
Node* const head_;
// Modified only by Insert(). Read racily by readers, but stale
// values are ok.
std::atomic<int> max_height_; // Height of the entire list
// Read/written only by Insert().
Random rnd_;
}; };
// Implementation details follow // Implementation details follow
@ -155,36 +154,36 @@ struct SkipList<Key,Comparator>::Node {
assert(n >= 0); assert(n >= 0);
// Use an 'acquire load' so that we observe a fully initialized // Use an 'acquire load' so that we observe a fully initialized
// version of the returned Node. // version of the returned Node.
return reinterpret_cast<Node*>(next_[n].Acquire_Load()); return next_[n].load(std::memory_order_acquire);
} }
void SetNext(int n, Node* x) { void SetNext(int n, Node* x) {
assert(n >= 0); assert(n >= 0);
// Use a 'release store' so that anybody who reads through this // Use a 'release store' so that anybody who reads through this
// pointer observes a fully initialized version of the inserted node. // pointer observes a fully initialized version of the inserted node.
next_[n].Release_Store(x); next_[n].store(x, std::memory_order_release);
} }
// No-barrier variants that can be safely used in a few locations. // No-barrier variants that can be safely used in a few locations.
Node* NoBarrier_Next(int n) { Node* NoBarrier_Next(int n) {
assert(n >= 0); assert(n >= 0);
return reinterpret_cast<Node*>(next_[n].NoBarrier_Load()); return next_[n].load(std::memory_order_relaxed);
} }
void NoBarrier_SetNext(int n, Node* x) { void NoBarrier_SetNext(int n, Node* x) {
assert(n >= 0); assert(n >= 0);
next_[n].NoBarrier_Store(x); next_[n].store(x, std::memory_order_relaxed);
} }
private: private:
// Array of length equal to the node height. next_[0] is lowest level link. // Array of length equal to the node height. next_[0] is lowest level link.
port::AtomicPointer next_[1]; std::atomic<Node*> next_[1];
}; };
template <typename Key, class Comparator> template <typename Key, class Comparator>
typename SkipList<Key,Comparator>::Node* typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::NewNode(
SkipList<Key,Comparator>::NewNode(const Key& key, int height) { const Key& key, int height) {
char* mem = arena_->AllocateAligned( char* const node_memory = arena_->AllocateAligned(
sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1)); sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
return new (mem) Node(key); return new (node_memory) Node(key);
} }
template <typename Key, class Comparator> template <typename Key, class Comparator>
@ -259,8 +258,9 @@ bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
} }
template <typename Key, class Comparator> template <typename Key, class Comparator>
typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOrEqual(const Key& key, Node** prev) typename SkipList<Key, Comparator>::Node*
const { SkipList<Key, Comparator>::FindGreaterOrEqual(const Key& key,
Node** prev) const {
Node* x = head_; Node* x = head_;
int level = GetMaxHeight() - 1; int level = GetMaxHeight() - 1;
while (true) { while (true) {
@ -326,7 +326,7 @@ SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
: compare_(cmp), : compare_(cmp),
arena_(arena), arena_(arena),
head_(NewNode(0 /* any key will do */, kMaxHeight)), head_(NewNode(0 /* any key will do */, kMaxHeight)),
max_height_(reinterpret_cast<void*>(1)), max_height_(1),
rnd_(0xdeadbeef) { rnd_(0xdeadbeef) {
for (int i = 0; i < kMaxHeight; i++) { for (int i = 0; i < kMaxHeight; i++) {
head_->SetNext(i, nullptr); head_->SetNext(i, nullptr);
@ -348,8 +348,6 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
for (int i = GetMaxHeight(); i < height; i++) { for (int i = GetMaxHeight(); i < height; i++) {
prev[i] = head_; prev[i] = head_;
} }
//fprintf(stderr, "Change height from %d to %d\n", max_height_, height);
// It is ok to mutate max_height_ without any synchronization // It is ok to mutate max_height_ without any synchronization
// with concurrent readers. A concurrent reader that observes // with concurrent readers. A concurrent reader that observes
// the new value of max_height_ will see either the old value of // the new value of max_height_ will see either the old value of
@ -357,7 +355,7 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
// the loop below. In the former case the reader will // the loop below. In the former case the reader will
// immediately drop to the next level since nullptr sorts after all // immediately drop to the next level since nullptr sorts after all
// keys. In the latter case the reader will use the new node. // keys. In the latter case the reader will use the new node.
max_height_.NoBarrier_Store(reinterpret_cast<void*>(height)); max_height_.store(height, std::memory_order_relaxed);
} }
x = NewNode(key, height); x = NewNode(key, height);

View File

@ -3,14 +3,18 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/skiplist.h" #include "db/skiplist.h"
#include <atomic>
#include <set> #include <set>
#include "gtest/gtest.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "port/port.h" #include "port/port.h"
#include "port/thread_annotations.h" #include "port/thread_annotations.h"
#include "util/arena.h" #include "util/arena.h"
#include "util/hash.h" #include "util/hash.h"
#include "util/random.h" #include "util/random.h"
#include "util/testharness.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
@ -28,8 +32,6 @@ struct Comparator {
} }
}; };
class SkipTest { };
TEST(SkipTest, Empty) { TEST(SkipTest, Empty) {
Arena arena; Arena arena;
Comparator cmp; Comparator cmp;
@ -114,8 +116,7 @@ TEST(SkipTest, InsertAndLookup) {
// Compare against model iterator // Compare against model iterator
for (std::set<Key>::reverse_iterator model_iter = keys.rbegin(); for (std::set<Key>::reverse_iterator model_iter = keys.rbegin();
model_iter != keys.rend(); model_iter != keys.rend(); ++model_iter) {
++model_iter) {
ASSERT_TRUE(iter.Valid()); ASSERT_TRUE(iter.Valid());
ASSERT_EQ(*model_iter, iter.key()); ASSERT_EQ(*model_iter, iter.key());
iter.Prev(); iter.Prev();
@ -128,7 +129,7 @@ TEST(SkipTest, InsertAndLookup) {
// concurrent readers (with no synchronization other than when a // concurrent readers (with no synchronization other than when a
// reader's iterator is created), the reader always observes all the // reader's iterator is created), the reader always observes all the
// data that was present in the skip list when the iterator was // data that was present in the skip list when the iterator was
// constructor. Because insertions are happening concurrently, we may // constructed. Because insertions are happening concurrently, we may
// also observe new values that were inserted since the iterator was // also observe new values that were inserted since the iterator was
// constructed, but we should never miss any values that were present // constructed, but we should never miss any values that were present
// at iterator construction time. // at iterator construction time.
@ -162,7 +163,7 @@ class ConcurrentTest {
} }
static Key MakeKey(uint64_t k, uint64_t g) { static Key MakeKey(uint64_t k, uint64_t g) {
assert(sizeof(Key) == sizeof(uint64_t)); static_assert(sizeof(Key) == sizeof(uint64_t), "");
assert(k <= K); // We sometimes pass K to seek to the end of the skiplist assert(k <= K); // We sometimes pass K to seek to the end of the skiplist
assert(g <= 0xffffffffu); assert(g <= 0xffffffffu);
return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff)); return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff));
@ -188,13 +189,11 @@ class ConcurrentTest {
// Per-key generation // Per-key generation
struct State { struct State {
port::AtomicPointer generation[K]; std::atomic<int> generation[K];
void Set(int k, intptr_t v) { void Set(int k, int v) {
generation[k].Release_Store(reinterpret_cast<void*>(v)); generation[k].store(v, std::memory_order_release);
}
intptr_t Get(int k) {
return reinterpret_cast<intptr_t>(generation[k].Acquire_Load());
} }
int Get(int k) { return generation[k].load(std::memory_order_acquire); }
State() { State() {
for (int k = 0; k < K; k++) { for (int k = 0; k < K; k++) {
@ -252,11 +251,9 @@ class ConcurrentTest {
// Note that generation 0 is never inserted, so it is ok if // Note that generation 0 is never inserted, so it is ok if
// <*,0,*> is missing. // <*,0,*> is missing.
ASSERT_TRUE((gen(pos) == 0) || ASSERT_TRUE((gen(pos) == 0) ||
(gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))) (gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))))
) << "key: " << key(pos) << "key: " << key(pos) << "; gen: " << gen(pos)
<< "; gen: " << gen(pos) << "; initgen: " << initial_state.Get(key(pos));
<< "; initgen: "
<< initial_state.Get(key(pos));
// Advance to next key in the valid key space // Advance to next key in the valid key space
if (key(pos) < key(current)) { if (key(pos) < key(current)) {
@ -300,19 +297,12 @@ class TestState {
public: public:
ConcurrentTest t_; ConcurrentTest t_;
int seed_; int seed_;
port::AtomicPointer quit_flag_; std::atomic<bool> quit_flag_;
enum ReaderState { enum ReaderState { STARTING, RUNNING, DONE };
STARTING,
RUNNING,
DONE
};
explicit TestState(int s) explicit TestState(int s)
: seed_(s), : seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {}
quit_flag_(nullptr),
state_(STARTING),
state_cv_(&mu_) {}
void Wait(ReaderState s) LOCKS_EXCLUDED(mu_) { void Wait(ReaderState s) LOCKS_EXCLUDED(mu_) {
mu_.Lock(); mu_.Lock();
@ -340,7 +330,7 @@ static void ConcurrentReader(void* arg) {
Random rnd(state->seed_); Random rnd(state->seed_);
int64_t reads = 0; int64_t reads = 0;
state->Change(TestState::RUNNING); state->Change(TestState::RUNNING);
while (!state->quit_flag_.Acquire_Load()) { while (!state->quit_flag_.load(std::memory_order_acquire)) {
state->t_.ReadStep(&rnd); state->t_.ReadStep(&rnd);
++reads; ++reads;
} }
@ -362,7 +352,7 @@ static void RunConcurrent(int run) {
for (int i = 0; i < kSize; i++) { for (int i = 0; i < kSize; i++) {
state.t_.WriteStep(&rnd); state.t_.WriteStep(&rnd);
} }
state.quit_flag_.Release_Store(&state); // Any non-null arg will do state.quit_flag_.store(true, std::memory_order_release);
state.Wait(TestState::DONE); state.Wait(TestState::DONE);
} }
} }
@ -376,5 +366,6 @@ TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -44,8 +44,14 @@ class SnapshotList {
} }
bool empty() const { return head_.next_ == &head_; } bool empty() const { return head_.next_ == &head_; }
SnapshotImpl* oldest() const { assert(!empty()); return head_.next_; } SnapshotImpl* oldest() const {
SnapshotImpl* newest() const { assert(!empty()); return head_.prev_; } assert(!empty());
return head_.next_;
}
SnapshotImpl* newest() const {
assert(!empty());
return head_.prev_;
}
// Creates a SnapshotImpl and appends it to the end of the list. // Creates a SnapshotImpl and appends it to the end of the list.
SnapshotImpl* New(SequenceNumber sequence_number) { SnapshotImpl* New(SequenceNumber sequence_number) {

View File

@ -29,18 +29,14 @@ static void UnrefEntry(void* arg1, void* arg2) {
cache->Release(h); cache->Release(h);
} }
TableCache::TableCache(const std::string& dbname, TableCache::TableCache(const std::string& dbname, const Options& options,
const Options& options,
int entries) int entries)
: env_(options.env), : env_(options.env),
dbname_(dbname), dbname_(dbname),
options_(options), options_(options),
cache_(NewLRUCache(entries)) { cache_(NewLRUCache(entries)) {}
}
TableCache::~TableCache() { TableCache::~TableCache() { delete cache_; }
delete cache_;
}
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
Cache::Handle** handle) { Cache::Handle** handle) {
@ -80,8 +76,7 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
} }
Iterator* TableCache::NewIterator(const ReadOptions& options, Iterator* TableCache::NewIterator(const ReadOptions& options,
uint64_t file_number, uint64_t file_number, uint64_t file_size,
uint64_t file_size,
Table** tableptr) { Table** tableptr) {
if (tableptr != nullptr) { if (tableptr != nullptr) {
*tableptr = nullptr; *tableptr = nullptr;
@ -102,17 +97,15 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
return result; return result;
} }
Status TableCache::Get(const ReadOptions& options, Status TableCache::Get(const ReadOptions& options, uint64_t file_number,
uint64_t file_number, uint64_t file_size, const Slice& k, void* arg,
uint64_t file_size, void (*handle_result)(void*, const Slice&,
const Slice& k, const Slice&)) {
void* arg,
void (*saver)(void*, const Slice&, const Slice&)) {
Cache::Handle* handle = nullptr; Cache::Handle* handle = nullptr;
Status s = FindTable(file_number, file_size, &handle); Status s = FindTable(file_number, file_size, &handle);
if (s.ok()) { if (s.ok()) {
Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table; Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
s = t->InternalGet(options, k, arg, saver); s = t->InternalGet(options, k, arg, handle_result);
cache_->Release(handle); cache_->Release(handle);
} }
return s; return s;

View File

@ -7,8 +7,10 @@
#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
#include <string>
#include <stdint.h> #include <stdint.h>
#include <string>
#include "db/dbformat.h" #include "db/dbformat.h"
#include "leveldb/cache.h" #include "leveldb/cache.h"
#include "leveldb/table.h" #include "leveldb/table.h"
@ -30,30 +32,25 @@ class TableCache {
// underlies the returned iterator. The returned "*tableptr" object is owned // underlies the returned iterator. The returned "*tableptr" object is owned
// by the cache and should not be deleted, and is valid for as long as the // by the cache and should not be deleted, and is valid for as long as the
// returned iterator is live. // returned iterator is live.
Iterator* NewIterator(const ReadOptions& options, Iterator* NewIterator(const ReadOptions& options, uint64_t file_number,
uint64_t file_number, uint64_t file_size, Table** tableptr = nullptr);
uint64_t file_size,
Table** tableptr = nullptr);
// If a seek to internal key "k" in specified file finds an entry, // If a seek to internal key "k" in specified file finds an entry,
// call (*handle_result)(arg, found_key, found_value). // call (*handle_result)(arg, found_key, found_value).
Status Get(const ReadOptions& options, Status Get(const ReadOptions& options, uint64_t file_number,
uint64_t file_number, uint64_t file_size, const Slice& k, void* arg,
uint64_t file_size,
const Slice& k,
void* arg,
void (*handle_result)(void*, const Slice&, const Slice&)); void (*handle_result)(void*, const Slice&, const Slice&));
// Evict any entry for the specified file number // Evict any entry for the specified file number
void Evict(uint64_t file_number); void Evict(uint64_t file_number);
private: private:
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
Env* const env_; Env* const env_;
const std::string dbname_; const std::string dbname_;
const Options& options_; const Options& options_;
Cache* cache_; Cache* cache_;
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
}; };
} // namespace leveldb } // namespace leveldb

View File

@ -66,12 +66,10 @@ void VersionEdit::EncodeTo(std::string* dst) const {
PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode()); PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode());
} }
for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); for (const auto& deleted_file_kvp : deleted_files_) {
iter != deleted_files_.end();
++iter) {
PutVarint32(dst, kDeletedFile); PutVarint32(dst, kDeletedFile);
PutVarint32(dst, iter->first); // level PutVarint32(dst, deleted_file_kvp.first); // level
PutVarint64(dst, iter->second); // file number PutVarint64(dst, deleted_file_kvp.second); // file number
} }
for (size_t i = 0; i < new_files_.size(); i++) { for (size_t i = 0; i < new_files_.size(); i++) {
@ -88,8 +86,7 @@ void VersionEdit::EncodeTo(std::string* dst) const {
static bool GetInternalKey(Slice* input, InternalKey* dst) { static bool GetInternalKey(Slice* input, InternalKey* dst) {
Slice str; Slice str;
if (GetLengthPrefixedSlice(input, &str)) { if (GetLengthPrefixedSlice(input, &str)) {
dst->DecodeFrom(str); return dst->DecodeFrom(str);
return true;
} else { } else {
return false; return false;
} }
@ -97,8 +94,7 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) {
static bool GetLevel(Slice* input, int* level) { static bool GetLevel(Slice* input, int* level) {
uint32_t v; uint32_t v;
if (GetVarint32(input, &v) && if (GetVarint32(input, &v) && v < config::kNumLevels) {
v < config::kNumLevels) {
*level = v; *level = v;
return true; return true;
} else { } else {
@ -163,8 +159,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
break; break;
case kCompactPointer: case kCompactPointer:
if (GetLevel(&input, &level) && if (GetLevel(&input, &level) && GetInternalKey(&input, &key)) {
GetInternalKey(&input, &key)) {
compact_pointers_.push_back(std::make_pair(level, key)); compact_pointers_.push_back(std::make_pair(level, key));
} else { } else {
msg = "compaction pointer"; msg = "compaction pointer";
@ -172,8 +167,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
break; break;
case kDeletedFile: case kDeletedFile:
if (GetLevel(&input, &level) && if (GetLevel(&input, &level) && GetVarint64(&input, &number)) {
GetVarint64(&input, &number)) {
deleted_files_.insert(std::make_pair(level, number)); deleted_files_.insert(std::make_pair(level, number));
} else { } else {
msg = "deleted file"; msg = "deleted file";
@ -181,8 +175,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
break; break;
case kNewFile: case kNewFile:
if (GetLevel(&input, &level) && if (GetLevel(&input, &level) && GetVarint64(&input, &f.number) &&
GetVarint64(&input, &f.number) &&
GetVarint64(&input, &f.file_size) && GetVarint64(&input, &f.file_size) &&
GetInternalKey(&input, &f.smallest) && GetInternalKey(&input, &f.smallest) &&
GetInternalKey(&input, &f.largest)) { GetInternalKey(&input, &f.largest)) {
@ -238,13 +231,11 @@ std::string VersionEdit::DebugString() const {
r.append(" "); r.append(" ");
r.append(compact_pointers_[i].second.DebugString()); r.append(compact_pointers_[i].second.DebugString());
} }
for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); for (const auto& deleted_files_kvp : deleted_files_) {
iter != deleted_files_.end();
++iter) {
r.append("\n DeleteFile: "); r.append("\n DeleteFile: ");
AppendNumberTo(&r, iter->first); AppendNumberTo(&r, deleted_files_kvp.first);
r.append(" "); r.append(" ");
AppendNumberTo(&r, iter->second); AppendNumberTo(&r, deleted_files_kvp.second);
} }
for (size_t i = 0; i < new_files_.size(); i++) { for (size_t i = 0; i < new_files_.size(); i++) {
const FileMetaData& f = new_files_[i].second; const FileMetaData& f = new_files_[i].second;

View File

@ -8,6 +8,7 @@
#include <set> #include <set>
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "db/dbformat.h" #include "db/dbformat.h"
namespace leveldb { namespace leveldb {
@ -15,20 +16,20 @@ namespace leveldb {
class VersionSet; class VersionSet;
struct FileMetaData { struct FileMetaData {
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {}
int refs; int refs;
int allowed_seeks; // Seeks allowed until compaction int allowed_seeks; // Seeks allowed until compaction
uint64_t number; uint64_t number;
uint64_t file_size; // File size in bytes uint64_t file_size; // File size in bytes
InternalKey smallest; // Smallest internal key served by table InternalKey smallest; // Smallest internal key served by table
InternalKey largest; // Largest internal key served by table InternalKey largest; // Largest internal key served by table
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
}; };
class VersionEdit { class VersionEdit {
public: public:
VersionEdit() { Clear(); } VersionEdit() { Clear(); }
~VersionEdit() { } ~VersionEdit() = default;
void Clear(); void Clear();
@ -59,10 +60,8 @@ class VersionEdit {
// Add the specified file at the specified number. // Add the specified file at the specified number.
// REQUIRES: This version has not been saved (see VersionSet::SaveTo) // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
void AddFile(int level, uint64_t file, void AddFile(int level, uint64_t file, uint64_t file_size,
uint64_t file_size, const InternalKey& smallest, const InternalKey& largest) {
const InternalKey& smallest,
const InternalKey& largest) {
FileMetaData f; FileMetaData f;
f.number = file; f.number = file;
f.file_size = file_size; f.file_size = file_size;

View File

@ -3,7 +3,8 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/version_edit.h" #include "db/version_edit.h"
#include "util/testharness.h"
#include "gtest/gtest.h"
namespace leveldb { namespace leveldb {
@ -17,8 +18,6 @@ static void TestEncodeDecode(const VersionEdit& edit) {
ASSERT_EQ(encoded, encoded2); ASSERT_EQ(encoded, encoded2);
} }
class VersionEditTest { };
TEST(VersionEditTest, EncodeDecode) { TEST(VersionEditTest, EncodeDecode) {
static const uint64_t kBig = 1ull << 50; static const uint64_t kBig = 1ull << 50;
@ -42,5 +41,6 @@ TEST(VersionEditTest, EncodeDecode) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -4,8 +4,10 @@
#include "db/version_set.h" #include "db/version_set.h"
#include <algorithm>
#include <stdio.h> #include <stdio.h>
#include <algorithm>
#include "db/filename.h" #include "db/filename.h"
#include "db/log_reader.h" #include "db/log_reader.h"
#include "db/log_writer.h" #include "db/log_writer.h"
@ -84,8 +86,7 @@ Version::~Version() {
} }
int FindFile(const InternalKeyComparator& icmp, int FindFile(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& files, const std::vector<FileMetaData*>& files, const Slice& key) {
const Slice& key) {
uint32_t left = 0; uint32_t left = 0;
uint32_t right = files.size(); uint32_t right = files.size();
while (left < right) { while (left < right) {
@ -104,22 +105,21 @@ int FindFile(const InternalKeyComparator& icmp,
return right; return right;
} }
static bool AfterFile(const Comparator* ucmp, static bool AfterFile(const Comparator* ucmp, const Slice* user_key,
const Slice* user_key, const FileMetaData* f) { const FileMetaData* f) {
// null user_key occurs before all keys and is therefore never after *f // null user_key occurs before all keys and is therefore never after *f
return (user_key != nullptr && return (user_key != nullptr &&
ucmp->Compare(*user_key, f->largest.user_key()) > 0); ucmp->Compare(*user_key, f->largest.user_key()) > 0);
} }
static bool BeforeFile(const Comparator* ucmp, static bool BeforeFile(const Comparator* ucmp, const Slice* user_key,
const Slice* user_key, const FileMetaData* f) { const FileMetaData* f) {
// null user_key occurs after all keys and is therefore never before *f // null user_key occurs after all keys and is therefore never before *f
return (user_key != nullptr && return (user_key != nullptr &&
ucmp->Compare(*user_key, f->smallest.user_key()) < 0); ucmp->Compare(*user_key, f->smallest.user_key()) < 0);
} }
bool SomeFileOverlapsRange( bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
const InternalKeyComparator& icmp,
bool disjoint_sorted_files, bool disjoint_sorted_files,
const std::vector<FileMetaData*>& files, const std::vector<FileMetaData*>& files,
const Slice* smallest_user_key, const Slice* smallest_user_key,
@ -143,8 +143,9 @@ bool SomeFileOverlapsRange(
uint32_t index = 0; uint32_t index = 0;
if (smallest_user_key != nullptr) { if (smallest_user_key != nullptr) {
// Find the earliest possible internal key for smallest_user_key // Find the earliest possible internal key for smallest_user_key
InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); InternalKey small_key(*smallest_user_key, kMaxSequenceNumber,
index = FindFile(icmp, files, small.Encode()); kValueTypeForSeek);
index = FindFile(icmp, files, small_key.Encode());
} }
if (index >= files.size()) { if (index >= files.size()) {
@ -164,25 +165,21 @@ class Version::LevelFileNumIterator : public Iterator {
public: public:
LevelFileNumIterator(const InternalKeyComparator& icmp, LevelFileNumIterator(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>* flist) const std::vector<FileMetaData*>* flist)
: icmp_(icmp), : icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid
flist_(flist),
index_(flist->size()) { // Marks as invalid
} }
virtual bool Valid() const { bool Valid() const override { return index_ < flist_->size(); }
return index_ < flist_->size(); void Seek(const Slice& target) override {
}
virtual void Seek(const Slice& target) {
index_ = FindFile(icmp_, *flist_, target); index_ = FindFile(icmp_, *flist_, target);
} }
virtual void SeekToFirst() { index_ = 0; } void SeekToFirst() override { index_ = 0; }
virtual void SeekToLast() { void SeekToLast() override {
index_ = flist_->empty() ? 0 : flist_->size() - 1; index_ = flist_->empty() ? 0 : flist_->size() - 1;
} }
virtual void Next() { void Next() override {
assert(Valid()); assert(Valid());
index_++; index_++;
} }
virtual void Prev() { void Prev() override {
assert(Valid()); assert(Valid());
if (index_ == 0) { if (index_ == 0) {
index_ = flist_->size(); // Marks as invalid index_ = flist_->size(); // Marks as invalid
@ -190,17 +187,18 @@ class Version::LevelFileNumIterator : public Iterator {
index_--; index_--;
} }
} }
Slice key() const { Slice key() const override {
assert(Valid()); assert(Valid());
return (*flist_)[index_]->largest.Encode(); return (*flist_)[index_]->largest.Encode();
} }
Slice value() const { Slice value() const override {
assert(Valid()); assert(Valid());
EncodeFixed64(value_buf_, (*flist_)[index_]->number); EncodeFixed64(value_buf_, (*flist_)[index_]->number);
EncodeFixed64(value_buf_ + 8, (*flist_)[index_]->file_size); EncodeFixed64(value_buf_ + 8, (*flist_)[index_]->file_size);
return Slice(value_buf_, sizeof(value_buf_)); return Slice(value_buf_, sizeof(value_buf_));
} }
virtual Status status() const { return Status::OK(); } Status status() const override { return Status::OK(); }
private: private:
const InternalKeyComparator icmp_; const InternalKeyComparator icmp_;
const std::vector<FileMetaData*>* const flist_; const std::vector<FileMetaData*>* const flist_;
@ -210,16 +208,14 @@ class Version::LevelFileNumIterator : public Iterator {
mutable char value_buf_[16]; mutable char value_buf_[16];
}; };
static Iterator* GetFileIterator(void* arg, static Iterator* GetFileIterator(void* arg, const ReadOptions& options,
const ReadOptions& options,
const Slice& file_value) { const Slice& file_value) {
TableCache* cache = reinterpret_cast<TableCache*>(arg); TableCache* cache = reinterpret_cast<TableCache*>(arg);
if (file_value.size() != 16) { if (file_value.size() != 16) {
return NewErrorIterator( return NewErrorIterator(
Status::Corruption("FileReader invoked with unexpected value")); Status::Corruption("FileReader invoked with unexpected value"));
} else { } else {
return cache->NewIterator(options, return cache->NewIterator(options, DecodeFixed64(file_value.data()),
DecodeFixed64(file_value.data()),
DecodeFixed64(file_value.data() + 8)); DecodeFixed64(file_value.data() + 8));
} }
} }
@ -227,16 +223,15 @@ static Iterator* GetFileIterator(void* arg,
Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
int level) const { int level) const {
return NewTwoLevelIterator( return NewTwoLevelIterator(
new LevelFileNumIterator(vset_->icmp_, &files_[level]), new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator,
&GetFileIterator, vset_->table_cache_, options); vset_->table_cache_, options);
} }
void Version::AddIterators(const ReadOptions& options, void Version::AddIterators(const ReadOptions& options,
std::vector<Iterator*>* iters) { std::vector<Iterator*>* iters) {
// Merge all level zero files together since they may overlap // Merge all level zero files together since they may overlap
for (size_t i = 0; i < files_[0].size(); i++) { for (size_t i = 0; i < files_[0].size(); i++) {
iters->push_back( iters->push_back(vset_->table_cache_->NewIterator(
vset_->table_cache_->NewIterator(
options, files_[0][i]->number, files_[0][i]->file_size)); options, files_[0][i]->number, files_[0][i]->file_size));
} }
@ -264,7 +259,7 @@ struct Saver {
Slice user_key; Slice user_key;
std::string* value; std::string* value;
}; };
} } // namespace
static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { static void SaveValue(void* arg, const Slice& ikey, const Slice& v) {
Saver* s = reinterpret_cast<Saver*>(arg); Saver* s = reinterpret_cast<Saver*>(arg);
ParsedInternalKey parsed_key; ParsedInternalKey parsed_key;
@ -284,10 +279,8 @@ static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
return a->number > b->number; return a->number > b->number;
} }
void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
void* arg,
bool (*func)(void*, int, FileMetaData*)) { bool (*func)(void*, int, FileMetaData*)) {
// TODO(sanjay): Change Version::Get() to use this function.
const Comparator* ucmp = vset_->icmp_.user_comparator(); const Comparator* ucmp = vset_->icmp_.user_comparator();
// Search level-0 in order from newest to oldest. // Search level-0 in order from newest to oldest.
@ -329,103 +322,82 @@ void Version::ForEachOverlapping(Slice user_key, Slice internal_key,
} }
} }
Status Version::Get(const ReadOptions& options, Status Version::Get(const ReadOptions& options, const LookupKey& k,
const LookupKey& k, std::string* value, GetStats* stats) {
std::string* value,
GetStats* stats) {
Slice ikey = k.internal_key();
Slice user_key = k.user_key();
const Comparator* ucmp = vset_->icmp_.user_comparator();
Status s;
stats->seek_file = nullptr; stats->seek_file = nullptr;
stats->seek_file_level = -1; stats->seek_file_level = -1;
FileMetaData* last_file_read = nullptr;
int last_file_read_level = -1;
// We can search level-by-level since entries never hop across
// levels. Therefore we are guaranteed that if we find data
// in an smaller level, later levels are irrelevant.
std::vector<FileMetaData*> tmp;
FileMetaData* tmp2;
for (int level = 0; level < config::kNumLevels; level++) {
size_t num_files = files_[level].size();
if (num_files == 0) continue;
// Get the list of files to search in this level
FileMetaData* const* files = &files_[level][0];
if (level == 0) {
// Level-0 files may overlap each other. Find all files that
// overlap user_key and process them in order from newest to oldest.
tmp.reserve(num_files);
for (uint32_t i = 0; i < num_files; i++) {
FileMetaData* f = files[i];
if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
tmp.push_back(f);
}
}
if (tmp.empty()) continue;
std::sort(tmp.begin(), tmp.end(), NewestFirst);
files = &tmp[0];
num_files = tmp.size();
} else {
// Binary search to find earliest index whose largest key >= ikey.
uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
if (index >= num_files) {
files = nullptr;
num_files = 0;
} else {
tmp2 = files[index];
if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
// All of "tmp2" is past any data for user_key
files = nullptr;
num_files = 0;
} else {
files = &tmp2;
num_files = 1;
}
}
}
for (uint32_t i = 0; i < num_files; ++i) {
if (last_file_read != nullptr && stats->seek_file == nullptr) {
// We have had more than one seek for this read. Charge the 1st file.
stats->seek_file = last_file_read;
stats->seek_file_level = last_file_read_level;
}
FileMetaData* f = files[i];
last_file_read = f;
last_file_read_level = level;
struct State {
Saver saver; Saver saver;
saver.state = kNotFound; GetStats* stats;
saver.ucmp = ucmp; const ReadOptions* options;
saver.user_key = user_key; Slice ikey;
saver.value = value; FileMetaData* last_file_read;
s = vset_->table_cache_->Get(options, f->number, f->file_size, int last_file_read_level;
ikey, &saver, SaveValue);
if (!s.ok()) { VersionSet* vset;
return s; Status s;
} bool found;
switch (saver.state) {
case kNotFound: static bool Match(void* arg, int level, FileMetaData* f) {
break; // Keep searching in other files State* state = reinterpret_cast<State*>(arg);
case kFound:
return s; if (state->stats->seek_file == nullptr &&
case kDeleted: state->last_file_read != nullptr) {
s = Status::NotFound(Slice()); // Use empty error message for speed // We have had more than one seek for this read. Charge the 1st file.
return s; state->stats->seek_file = state->last_file_read;
case kCorrupt: state->stats->seek_file_level = state->last_file_read_level;
s = Status::Corruption("corrupted key for ", user_key);
return s;
}
}
} }
return Status::NotFound(Slice()); // Use an empty error message for speed state->last_file_read = f;
state->last_file_read_level = level;
state->s = state->vset->table_cache_->Get(*state->options, f->number,
f->file_size, state->ikey,
&state->saver, SaveValue);
if (!state->s.ok()) {
state->found = true;
return false;
}
switch (state->saver.state) {
case kNotFound:
return true; // Keep searching in other files
case kFound:
state->found = true;
return false;
case kDeleted:
return false;
case kCorrupt:
state->s =
Status::Corruption("corrupted key for ", state->saver.user_key);
state->found = true;
return false;
}
// Not reached. Added to avoid false compilation warnings of
// "control reaches end of non-void function".
return false;
}
};
State state;
state.found = false;
state.stats = stats;
state.last_file_read = nullptr;
state.last_file_read_level = -1;
state.options = &options;
state.ikey = k.internal_key();
state.vset = vset_;
state.saver.state = kNotFound;
state.saver.ucmp = vset_->icmp_.user_comparator();
state.saver.user_key = k.user_key();
state.saver.value = value;
ForEachOverlapping(state.saver.user_key, state.ikey, &state, &State::Match);
return state.found ? state.s : Status::NotFound(Slice());
} }
bool Version::UpdateStats(const GetStats& stats) { bool Version::UpdateStats(const GetStats& stats) {
@ -479,9 +451,7 @@ bool Version::RecordReadSample(Slice internal_key) {
return false; return false;
} }
void Version::Ref() { void Version::Ref() { ++refs_; }
++refs_;
}
void Version::Unref() { void Version::Unref() {
assert(this != &vset_->dummy_versions_); assert(this != &vset_->dummy_versions_);
@ -492,15 +462,13 @@ void Version::Unref() {
} }
} }
bool Version::OverlapInLevel(int level, bool Version::OverlapInLevel(int level, const Slice* smallest_user_key,
const Slice* smallest_user_key,
const Slice* largest_user_key) { const Slice* largest_user_key) {
return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level], return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level],
smallest_user_key, largest_user_key); smallest_user_key, largest_user_key);
} }
int Version::PickLevelForMemTableOutput( int Version::PickLevelForMemTableOutput(const Slice& smallest_user_key,
const Slice& smallest_user_key,
const Slice& largest_user_key) { const Slice& largest_user_key) {
int level = 0; int level = 0;
if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) { if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) {
@ -528,9 +496,7 @@ int Version::PickLevelForMemTableOutput(
} }
// Store in "*inputs" all files in "level" that overlap [begin,end] // Store in "*inputs" all files in "level" that overlap [begin,end]
void Version::GetOverlappingInputs( void Version::GetOverlappingInputs(int level, const InternalKey* begin,
int level,
const InternalKey* begin,
const InternalKey* end, const InternalKey* end,
std::vector<FileMetaData*>* inputs) { std::vector<FileMetaData*>* inputs) {
assert(level >= 0); assert(level >= 0);
@ -561,8 +527,8 @@ void Version::GetOverlappingInputs(
user_begin = file_start; user_begin = file_start;
inputs->clear(); inputs->clear();
i = 0; i = 0;
} else if (end != nullptr && user_cmp->Compare(file_limit, } else if (end != nullptr &&
user_end) > 0) { user_cmp->Compare(file_limit, user_end) > 0) {
user_end = file_limit; user_end = file_limit;
inputs->clear(); inputs->clear();
i = 0; i = 0;
@ -630,9 +596,7 @@ class VersionSet::Builder {
public: public:
// Initialize a builder with the files from *base and other info from *vset // Initialize a builder with the files from *base and other info from *vset
Builder(VersionSet* vset, Version* base) Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) {
: vset_(vset),
base_(base) {
base_->Ref(); base_->Ref();
BySmallestKey cmp; BySmallestKey cmp;
cmp.internal_comparator = &vset_->icmp_; cmp.internal_comparator = &vset_->icmp_;
@ -646,8 +610,8 @@ class VersionSet::Builder {
const FileSet* added = levels_[level].added_files; const FileSet* added = levels_[level].added_files;
std::vector<FileMetaData*> to_unref; std::vector<FileMetaData*> to_unref;
to_unref.reserve(added->size()); to_unref.reserve(added->size());
for (FileSet::const_iterator it = added->begin(); for (FileSet::const_iterator it = added->begin(); it != added->end();
it != added->end(); ++it) { ++it) {
to_unref.push_back(*it); to_unref.push_back(*it);
} }
delete added; delete added;
@ -672,12 +636,9 @@ class VersionSet::Builder {
} }
// Delete files // Delete files
const VersionEdit::DeletedFileSet& del = edit->deleted_files_; for (const auto& deleted_file_set_kvp : edit->deleted_files_) {
for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin(); const int level = deleted_file_set_kvp.first;
iter != del.end(); const uint64_t number = deleted_file_set_kvp.second;
++iter) {
const int level = iter->first;
const uint64_t number = iter->second;
levels_[level].deleted_files.insert(number); levels_[level].deleted_files.insert(number);
} }
@ -700,7 +661,7 @@ class VersionSet::Builder {
// same as the compaction of 40KB of data. We are a little // same as the compaction of 40KB of data. We are a little
// conservative and allow approximately one seek for every 16KB // conservative and allow approximately one seek for every 16KB
// of data before triggering a compaction. // of data before triggering a compaction.
f->allowed_seeks = (f->file_size / 16384); f->allowed_seeks = static_cast<int>((f->file_size / 16384U));
if (f->allowed_seeks < 100) f->allowed_seeks = 100; if (f->allowed_seeks < 100) f->allowed_seeks = 100;
levels_[level].deleted_files.erase(f->number); levels_[level].deleted_files.erase(f->number);
@ -718,20 +679,17 @@ class VersionSet::Builder {
const std::vector<FileMetaData*>& base_files = base_->files_[level]; const std::vector<FileMetaData*>& base_files = base_->files_[level];
std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin(); std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
std::vector<FileMetaData*>::const_iterator base_end = base_files.end(); std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
const FileSet* added = levels_[level].added_files; const FileSet* added_files = levels_[level].added_files;
v->files_[level].reserve(base_files.size() + added->size()); v->files_[level].reserve(base_files.size() + added_files->size());
for (FileSet::const_iterator added_iter = added->begin(); for (const auto& added_file : *added_files) {
added_iter != added->end();
++added_iter) {
// Add all smaller files listed in base_ // Add all smaller files listed in base_
for (std::vector<FileMetaData*>::const_iterator bpos for (std::vector<FileMetaData*>::const_iterator bpos =
= std::upper_bound(base_iter, base_end, *added_iter, cmp); std::upper_bound(base_iter, base_end, added_file, cmp);
base_iter != bpos; base_iter != bpos; ++base_iter) {
++base_iter) {
MaybeAddFile(v, level, *base_iter); MaybeAddFile(v, level, *base_iter);
} }
MaybeAddFile(v, level, *added_iter); MaybeAddFile(v, level, added_file);
} }
// Add remaining base files // Add remaining base files
@ -773,8 +731,7 @@ class VersionSet::Builder {
} }
}; };
VersionSet::VersionSet(const std::string& dbname, VersionSet::VersionSet(const std::string& dbname, const Options* options,
const Options* options,
TableCache* table_cache, TableCache* table_cache,
const InternalKeyComparator* cmp) const InternalKeyComparator* cmp)
: env_(options->env), : env_(options->env),
@ -906,7 +863,7 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
Status VersionSet::Recover(bool* save_manifest) { Status VersionSet::Recover(bool* save_manifest) {
struct LogReporter : public log::Reader::Reporter { struct LogReporter : public log::Reader::Reporter {
Status* status; Status* status;
virtual void Corruption(size_t bytes, const Status& s) { void Corruption(size_t bytes, const Status& s) override {
if (this->status->ok()) *this->status = s; if (this->status->ok()) *this->status = s;
} }
}; };
@ -927,8 +884,8 @@ Status VersionSet::Recover(bool *save_manifest) {
s = env_->NewSequentialFile(dscname, &file); s = env_->NewSequentialFile(dscname, &file);
if (!s.ok()) { if (!s.ok()) {
if (s.IsNotFound()) { if (s.IsNotFound()) {
return Status::Corruption( return Status::Corruption("CURRENT points to a non-existent file",
"CURRENT points to a non-existent file", s.ToString()); s.ToString());
} }
return s; return s;
} }
@ -946,7 +903,8 @@ Status VersionSet::Recover(bool *save_manifest) {
{ {
LogReporter reporter; LogReporter reporter;
reporter.status = &s; reporter.status = &s;
log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); log::Reader reader(file, &reporter, true /*checksum*/,
0 /*initial_offset*/);
Slice record; Slice record;
std::string scratch; std::string scratch;
while (reader.ReadRecord(&record, &scratch) && s.ok()) { while (reader.ReadRecord(&record, &scratch) && s.ok()) {
@ -1142,16 +1100,12 @@ int VersionSet::NumLevelFiles(int level) const {
const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
// Update code if kNumLevels changes // Update code if kNumLevels changes
assert(config::kNumLevels == 7); static_assert(config::kNumLevels == 7, "");
snprintf(scratch->buffer, sizeof(scratch->buffer), snprintf(scratch->buffer, sizeof(scratch->buffer),
"files[ %d %d %d %d %d %d %d ]", "files[ %d %d %d %d %d %d %d ]", int(current_->files_[0].size()),
int(current_->files_[0].size()), int(current_->files_[1].size()), int(current_->files_[2].size()),
int(current_->files_[1].size()), int(current_->files_[3].size()), int(current_->files_[4].size()),
int(current_->files_[2].size()), int(current_->files_[5].size()), int(current_->files_[6].size()));
int(current_->files_[3].size()),
int(current_->files_[4].size()),
int(current_->files_[5].size()),
int(current_->files_[6].size()));
return scratch->buffer; return scratch->buffer;
} }
@ -1188,8 +1142,7 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
} }
void VersionSet::AddLiveFiles(std::set<uint64_t>* live) { void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
for (Version* v = dummy_versions_.next_; for (Version* v = dummy_versions_.next_; v != &dummy_versions_;
v != &dummy_versions_;
v = v->next_) { v = v->next_) {
for (int level = 0; level < config::kNumLevels; level++) { for (int level = 0; level < config::kNumLevels; level++) {
const std::vector<FileMetaData*>& files = v->files_[level]; const std::vector<FileMetaData*>& files = v->files_[level];
@ -1227,8 +1180,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() {
// *smallest, *largest. // *smallest, *largest.
// REQUIRES: inputs is not empty // REQUIRES: inputs is not empty
void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs, void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
InternalKey* smallest, InternalKey* smallest, InternalKey* largest) {
InternalKey* largest) {
assert(!inputs.empty()); assert(!inputs.empty());
smallest->Clear(); smallest->Clear();
largest->Clear(); largest->Clear();
@ -1253,8 +1205,7 @@ void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
// REQUIRES: inputs is not empty // REQUIRES: inputs is not empty
void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1, void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
const std::vector<FileMetaData*>& inputs2, const std::vector<FileMetaData*>& inputs2,
InternalKey* smallest, InternalKey* smallest, InternalKey* largest) {
InternalKey* largest) {
std::vector<FileMetaData*> all = inputs1; std::vector<FileMetaData*> all = inputs1;
all.insert(all.end(), inputs2.begin(), inputs2.end()); all.insert(all.end(), inputs2.begin(), inputs2.end());
GetRange(all, smallest, largest); GetRange(all, smallest, largest);
@ -1276,8 +1227,8 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
if (c->level() + which == 0) { if (c->level() + which == 0) {
const std::vector<FileMetaData*>& files = c->inputs_[which]; const std::vector<FileMetaData*>& files = c->inputs_[which];
for (size_t i = 0; i < files.size(); i++) { for (size_t i = 0; i < files.size(); i++) {
list[num++] = table_cache_->NewIterator( list[num++] = table_cache_->NewIterator(options, files[i]->number,
options, files[i]->number, files[i]->file_size); files[i]->file_size);
} }
} else { } else {
// Create concatenating iterator for the files from this level // Create concatenating iterator for the files from this level
@ -1347,12 +1298,94 @@ Compaction* VersionSet::PickCompaction() {
return c; return c;
} }
// Finds the largest key in a vector of files. Returns true if files it not
// empty.
bool FindLargestKey(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& files,
InternalKey* largest_key) {
if (files.empty()) {
return false;
}
*largest_key = files[0]->largest;
for (size_t i = 1; i < files.size(); ++i) {
FileMetaData* f = files[i];
if (icmp.Compare(f->largest, *largest_key) > 0) {
*largest_key = f->largest;
}
}
return true;
}
// Finds minimum file b2=(l2, u2) in level file for which l2 > u1 and
// user_key(l2) = user_key(u1)
FileMetaData* FindSmallestBoundaryFile(
const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& level_files,
const InternalKey& largest_key) {
const Comparator* user_cmp = icmp.user_comparator();
FileMetaData* smallest_boundary_file = nullptr;
for (size_t i = 0; i < level_files.size(); ++i) {
FileMetaData* f = level_files[i];
if (icmp.Compare(f->smallest, largest_key) > 0 &&
user_cmp->Compare(f->smallest.user_key(), largest_key.user_key()) ==
0) {
if (smallest_boundary_file == nullptr ||
icmp.Compare(f->smallest, smallest_boundary_file->smallest) < 0) {
smallest_boundary_file = f;
}
}
}
return smallest_boundary_file;
}
// Extracts the largest file b1 from |compaction_files| and then searches for a
// b2 in |level_files| for which user_key(u1) = user_key(l2). If it finds such a
// file b2 (known as a boundary file) it adds it to |compaction_files| and then
// searches again using this new upper bound.
//
// If there are two blocks, b1=(l1, u1) and b2=(l2, u2) and
// user_key(u1) = user_key(l2), and if we compact b1 but not b2 then a
// subsequent get operation will yield an incorrect result because it will
// return the record from b2 in level i rather than from b1 because it searches
// level by level for records matching the supplied user key.
//
// parameters:
// in level_files: List of files to search for boundary files.
// in/out compaction_files: List of files to extend by adding boundary files.
void AddBoundaryInputs(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& level_files,
std::vector<FileMetaData*>* compaction_files) {
InternalKey largest_key;
// Quick return if compaction_files is empty.
if (!FindLargestKey(icmp, *compaction_files, &largest_key)) {
return;
}
bool continue_searching = true;
while (continue_searching) {
FileMetaData* smallest_boundary_file =
FindSmallestBoundaryFile(icmp, level_files, largest_key);
// If a boundary file was found advance largest_key, otherwise we're done.
if (smallest_boundary_file != NULL) {
compaction_files->push_back(smallest_boundary_file);
largest_key = smallest_boundary_file->largest;
} else {
continue_searching = false;
}
}
}
void VersionSet::SetupOtherInputs(Compaction* c) { void VersionSet::SetupOtherInputs(Compaction* c) {
const int level = c->level(); const int level = c->level();
InternalKey smallest, largest; InternalKey smallest, largest;
AddBoundaryInputs(icmp_, current_->files_[level], &c->inputs_[0]);
GetRange(c->inputs_[0], &smallest, &largest); GetRange(c->inputs_[0], &smallest, &largest);
current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]); current_->GetOverlappingInputs(level + 1, &smallest, &largest,
&c->inputs_[1]);
// Get entire range covered by compaction // Get entire range covered by compaction
InternalKey all_start, all_limit; InternalKey all_start, all_limit;
@ -1363,6 +1396,7 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
if (!c->inputs_[1].empty()) { if (!c->inputs_[1].empty()) {
std::vector<FileMetaData*> expanded0; std::vector<FileMetaData*> expanded0;
current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0); current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0);
AddBoundaryInputs(icmp_, current_->files_[level], &expanded0);
const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
const int64_t expanded0_size = TotalFileSize(expanded0); const int64_t expanded0_size = TotalFileSize(expanded0);
@ -1377,13 +1411,9 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
if (expanded1.size() == c->inputs_[1].size()) { if (expanded1.size() == c->inputs_[1].size()) {
Log(options_->info_log, Log(options_->info_log,
"Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n", "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
level, level, int(c->inputs_[0].size()), int(c->inputs_[1].size()),
int(c->inputs_[0].size()), long(inputs0_size), long(inputs1_size), int(expanded0.size()),
int(c->inputs_[1].size()), int(expanded1.size()), long(expanded0_size), long(inputs1_size));
long(inputs0_size), long(inputs1_size),
int(expanded0.size()),
int(expanded1.size()),
long(expanded0_size), long(inputs1_size));
smallest = new_start; smallest = new_start;
largest = new_limit; largest = new_limit;
c->inputs_[0] = expanded0; c->inputs_[0] = expanded0;
@ -1408,9 +1438,7 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
c->edit_.SetCompactPointer(level, largest); c->edit_.SetCompactPointer(level, largest);
} }
Compaction* VersionSet::CompactRange( Compaction* VersionSet::CompactRange(int level, const InternalKey* begin,
int level,
const InternalKey* begin,
const InternalKey* end) { const InternalKey* end) {
std::vector<FileMetaData*> inputs; std::vector<FileMetaData*> inputs;
current_->GetOverlappingInputs(level, begin, end, &inputs); current_->GetOverlappingInputs(level, begin, end, &inputs);
@ -1484,7 +1512,7 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) { for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
const std::vector<FileMetaData*>& files = input_version_->files_[lvl]; const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
for (; level_ptrs_[lvl] < files.size(); ) { while (level_ptrs_[lvl] < files.size()) {
FileMetaData* f = files[level_ptrs_[lvl]]; FileMetaData* f = files[level_ptrs_[lvl]];
if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
// We've advanced far enough // We've advanced far enough
@ -1506,7 +1534,8 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
const InternalKeyComparator* icmp = &vset->icmp_; const InternalKeyComparator* icmp = &vset->icmp_;
while (grandparent_index_ < grandparents_.size() && while (grandparent_index_ < grandparents_.size() &&
icmp->Compare(internal_key, icmp->Compare(internal_key,
grandparents_[grandparent_index_]->largest.Encode()) > 0) { grandparents_[grandparent_index_]->largest.Encode()) >
0) {
if (seen_key_) { if (seen_key_) {
overlapped_bytes_ += grandparents_[grandparent_index_]->file_size; overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
} }

View File

@ -18,6 +18,7 @@
#include <map> #include <map>
#include <set> #include <set>
#include <vector> #include <vector>
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/version_edit.h" #include "db/version_edit.h"
#include "port/port.h" #include "port/port.h"
@ -25,7 +26,9 @@
namespace leveldb { namespace leveldb {
namespace log { class Writer; } namespace log {
class Writer;
}
class Compaction; class Compaction;
class Iterator; class Iterator;
@ -40,8 +43,7 @@ class WritableFile;
// Return files.size() if there is no such file. // Return files.size() if there is no such file.
// REQUIRES: "files" contains a sorted list of non-overlapping files. // REQUIRES: "files" contains a sorted list of non-overlapping files.
int FindFile(const InternalKeyComparator& icmp, int FindFile(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& files, const std::vector<FileMetaData*>& files, const Slice& key);
const Slice& key);
// Returns true iff some file in "files" overlaps the user key range // Returns true iff some file in "files" overlaps the user key range
// [*smallest,*largest]. // [*smallest,*largest].
@ -57,11 +59,6 @@ bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
class Version { class Version {
public: public:
// Append to *iters a sequence of iterators that will
// yield the contents of this Version when merged together.
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
// Lookup the value for key. If found, store it in *val and // Lookup the value for key. If found, store it in *val and
// return OK. Else return a non-OK status. Fills *stats. // return OK. Else return a non-OK status. Fills *stats.
// REQUIRES: lock is not held // REQUIRES: lock is not held
@ -69,6 +66,12 @@ class Version {
FileMetaData* seek_file; FileMetaData* seek_file;
int seek_file_level; int seek_file_level;
}; };
// Append to *iters a sequence of iterators that will
// yield the contents of this Version when merged together.
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
Status Get(const ReadOptions&, const LookupKey& key, std::string* val, Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
GetStats* stats); GetStats* stats);
@ -98,8 +101,7 @@ class Version {
// some part of [*smallest_user_key,*largest_user_key]. // some part of [*smallest_user_key,*largest_user_key].
// smallest_user_key==nullptr represents a key smaller than all the DB's keys. // smallest_user_key==nullptr represents a key smaller than all the DB's keys.
// largest_user_key==nullptr represents a key largest than all the DB's keys. // largest_user_key==nullptr represents a key largest than all the DB's keys.
bool OverlapInLevel(int level, bool OverlapInLevel(int level, const Slice* smallest_user_key,
const Slice* smallest_user_key,
const Slice* largest_user_key); const Slice* largest_user_key);
// Return the level at which we should place a new memtable compaction // Return the level at which we should place a new memtable compaction
@ -117,6 +119,22 @@ class Version {
friend class VersionSet; friend class VersionSet;
class LevelFileNumIterator; class LevelFileNumIterator;
explicit Version(VersionSet* vset)
: vset_(vset),
next_(this),
prev_(this),
refs_(0),
file_to_compact_(nullptr),
file_to_compact_level_(-1),
compaction_score_(-1),
compaction_level_(-1) {}
Version(const Version&) = delete;
Version& operator=(const Version&) = delete;
~Version();
Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const; Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
// Call func(arg, level, f) for every file that overlaps user_key in // Call func(arg, level, f) for every file that overlaps user_key in
@ -124,8 +142,7 @@ class Version {
// false, makes no more calls. // false, makes no more calls.
// //
// REQUIRES: user portion of internal_key == user_key. // REQUIRES: user portion of internal_key == user_key.
void ForEachOverlapping(Slice user_key, Slice internal_key, void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
void* arg,
bool (*func)(void*, int, FileMetaData*)); bool (*func)(void*, int, FileMetaData*));
VersionSet* vset_; // VersionSet to which this Version belongs VersionSet* vset_; // VersionSet to which this Version belongs
@ -145,28 +162,15 @@ class Version {
// are initialized by Finalize(). // are initialized by Finalize().
double compaction_score_; double compaction_score_;
int compaction_level_; int compaction_level_;
explicit Version(VersionSet* vset)
: vset_(vset), next_(this), prev_(this), refs_(0),
file_to_compact_(nullptr),
file_to_compact_level_(-1),
compaction_score_(-1),
compaction_level_(-1) {
}
~Version();
// No copying allowed
Version(const Version&);
void operator=(const Version&);
}; };
class VersionSet { class VersionSet {
public: public:
VersionSet(const std::string& dbname, VersionSet(const std::string& dbname, const Options* options,
const Options* options, TableCache* table_cache, const InternalKeyComparator*);
TableCache* table_cache, VersionSet(const VersionSet&) = delete;
const InternalKeyComparator*); VersionSet& operator=(const VersionSet&) = delete;
~VersionSet(); ~VersionSet();
// Apply *edit to the current version to form a new descriptor that // Apply *edit to the current version to form a new descriptor that
@ -233,9 +237,7 @@ class VersionSet {
// the specified level. Returns nullptr if there is nothing in that // the specified level. Returns nullptr if there is nothing in that
// level that overlaps the specified range. Caller should delete // level that overlaps the specified range. Caller should delete
// the result. // the result.
Compaction* CompactRange( Compaction* CompactRange(int level, const InternalKey* begin,
int level,
const InternalKey* begin,
const InternalKey* end); const InternalKey* end);
// Return the maximum overlapping data (in bytes) at next level for any // Return the maximum overlapping data (in bytes) at next level for any
@ -277,14 +279,12 @@ class VersionSet {
void Finalize(Version* v); void Finalize(Version* v);
void GetRange(const std::vector<FileMetaData*>& inputs, void GetRange(const std::vector<FileMetaData*>& inputs, InternalKey* smallest,
InternalKey* smallest,
InternalKey* largest); InternalKey* largest);
void GetRange2(const std::vector<FileMetaData*>& inputs1, void GetRange2(const std::vector<FileMetaData*>& inputs1,
const std::vector<FileMetaData*>& inputs2, const std::vector<FileMetaData*>& inputs2,
InternalKey* smallest, InternalKey* smallest, InternalKey* largest);
InternalKey* largest);
void SetupOtherInputs(Compaction* c); void SetupOtherInputs(Compaction* c);
@ -313,10 +313,6 @@ class VersionSet {
// Per-level key at which the next compaction at that level should start. // Per-level key at which the next compaction at that level should start.
// Either an empty string, or a valid InternalKey. // Either an empty string, or a valid InternalKey.
std::string compact_pointer_[config::kNumLevels]; std::string compact_pointer_[config::kNumLevels];
// No copying allowed
VersionSet(const VersionSet&);
void operator=(const VersionSet&);
}; };
// A Compaction encapsulates information about a compaction. // A Compaction encapsulates information about a compaction.
@ -375,7 +371,7 @@ class Compaction {
// Each compaction reads inputs from "level_" and "level_+1" // Each compaction reads inputs from "level_" and "level_+1"
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
// State used to check for number of of overlapping grandparent files // State used to check for number of overlapping grandparent files
// (parent == level_ + 1, grandparent == level_ + 2) // (parent == level_ + 1, grandparent == level_ + 2)
std::vector<FileMetaData*> grandparents_; std::vector<FileMetaData*> grandparents_;
size_t grandparent_index_; // Index in grandparent_starts_ size_t grandparent_index_; // Index in grandparent_starts_

View File

@ -3,17 +3,15 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/version_set.h" #include "db/version_set.h"
#include "gtest/gtest.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
class FindFileTest { class FindFileTest : public testing::Test {
public: public:
std::vector<FileMetaData*> files_;
bool disjoint_sorted_files_;
FindFileTest() : disjoint_sorted_files_(true) {} FindFileTest() : disjoint_sorted_files_(true) {}
~FindFileTest() { ~FindFileTest() {
@ -46,9 +44,14 @@ class FindFileTest {
(smallest != nullptr ? &s : nullptr), (smallest != nullptr ? &s : nullptr),
(largest != nullptr ? &l : nullptr)); (largest != nullptr ? &l : nullptr));
} }
bool disjoint_sorted_files_;
private:
std::vector<FileMetaData*> files_;
}; };
TEST(FindFileTest, Empty) { TEST_F(FindFileTest, Empty) {
ASSERT_EQ(0, Find("foo")); ASSERT_EQ(0, Find("foo"));
ASSERT_TRUE(!Overlaps("a", "z")); ASSERT_TRUE(!Overlaps("a", "z"));
ASSERT_TRUE(!Overlaps(nullptr, "z")); ASSERT_TRUE(!Overlaps(nullptr, "z"));
@ -56,7 +59,7 @@ TEST(FindFileTest, Empty) {
ASSERT_TRUE(!Overlaps(nullptr, nullptr)); ASSERT_TRUE(!Overlaps(nullptr, nullptr));
} }
TEST(FindFileTest, Single) { TEST_F(FindFileTest, Single) {
Add("p", "q"); Add("p", "q");
ASSERT_EQ(0, Find("a")); ASSERT_EQ(0, Find("a"));
ASSERT_EQ(0, Find("p")); ASSERT_EQ(0, Find("p"));
@ -86,8 +89,7 @@ TEST(FindFileTest, Single) {
ASSERT_TRUE(Overlaps(nullptr, nullptr)); ASSERT_TRUE(Overlaps(nullptr, nullptr));
} }
TEST_F(FindFileTest, Multiple) {
TEST(FindFileTest, Multiple) {
Add("150", "200"); Add("150", "200");
Add("200", "250"); Add("200", "250");
Add("300", "350"); Add("300", "350");
@ -125,7 +127,7 @@ TEST(FindFileTest, Multiple) {
ASSERT_TRUE(Overlaps("450", "500")); ASSERT_TRUE(Overlaps("450", "500"));
} }
TEST(FindFileTest, MultipleNullBoundaries) { TEST_F(FindFileTest, MultipleNullBoundaries) {
Add("150", "200"); Add("150", "200");
Add("200", "250"); Add("200", "250");
Add("300", "350"); Add("300", "350");
@ -145,7 +147,7 @@ TEST(FindFileTest, MultipleNullBoundaries) {
ASSERT_TRUE(Overlaps("450", nullptr)); ASSERT_TRUE(Overlaps("450", nullptr));
} }
TEST(FindFileTest, OverlapSequenceChecks) { TEST_F(FindFileTest, OverlapSequenceChecks) {
Add("200", "200", 5000, 3000); Add("200", "200", 5000, 3000);
ASSERT_TRUE(!Overlaps("199", "199")); ASSERT_TRUE(!Overlaps("199", "199"));
ASSERT_TRUE(!Overlaps("201", "300")); ASSERT_TRUE(!Overlaps("201", "300"));
@ -154,7 +156,7 @@ TEST(FindFileTest, OverlapSequenceChecks) {
ASSERT_TRUE(Overlaps("200", "210")); ASSERT_TRUE(Overlaps("200", "210"));
} }
TEST(FindFileTest, OverlappingFiles) { TEST_F(FindFileTest, OverlappingFiles) {
Add("150", "600"); Add("150", "600");
Add("400", "500"); Add("400", "500");
disjoint_sorted_files_ = false; disjoint_sorted_files_ = false;
@ -172,8 +174,163 @@ TEST(FindFileTest, OverlappingFiles) {
ASSERT_TRUE(Overlaps("600", "700")); ASSERT_TRUE(Overlaps("600", "700"));
} }
void AddBoundaryInputs(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& level_files,
std::vector<FileMetaData*>* compaction_files);
class AddBoundaryInputsTest : public testing::Test {
public:
std::vector<FileMetaData*> level_files_;
std::vector<FileMetaData*> compaction_files_;
std::vector<FileMetaData*> all_files_;
InternalKeyComparator icmp_;
AddBoundaryInputsTest() : icmp_(BytewiseComparator()) {}
~AddBoundaryInputsTest() {
for (size_t i = 0; i < all_files_.size(); ++i) {
delete all_files_[i];
}
all_files_.clear();
}
FileMetaData* CreateFileMetaData(uint64_t number, InternalKey smallest,
InternalKey largest) {
FileMetaData* f = new FileMetaData();
f->number = number;
f->smallest = smallest;
f->largest = largest;
all_files_.push_back(f);
return f;
}
};
TEST_F(AddBoundaryInputsTest, TestEmptyFileSets) {
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
ASSERT_TRUE(compaction_files_.empty());
ASSERT_TRUE(level_files_.empty());
}
TEST_F(AddBoundaryInputsTest, TestEmptyLevelFiles) {
FileMetaData* f1 =
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
InternalKey(InternalKey("100", 1, kTypeValue)));
compaction_files_.push_back(f1);
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
ASSERT_EQ(1, compaction_files_.size());
ASSERT_EQ(f1, compaction_files_[0]);
ASSERT_TRUE(level_files_.empty());
}
TEST_F(AddBoundaryInputsTest, TestEmptyCompactionFiles) {
FileMetaData* f1 =
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
InternalKey(InternalKey("100", 1, kTypeValue)));
level_files_.push_back(f1);
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
ASSERT_TRUE(compaction_files_.empty());
ASSERT_EQ(1, level_files_.size());
ASSERT_EQ(f1, level_files_[0]);
}
TEST_F(AddBoundaryInputsTest, TestNoBoundaryFiles) {
FileMetaData* f1 =
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
InternalKey(InternalKey("100", 1, kTypeValue)));
FileMetaData* f2 =
CreateFileMetaData(1, InternalKey("200", 2, kTypeValue),
InternalKey(InternalKey("200", 1, kTypeValue)));
FileMetaData* f3 =
CreateFileMetaData(1, InternalKey("300", 2, kTypeValue),
InternalKey(InternalKey("300", 1, kTypeValue)));
level_files_.push_back(f3);
level_files_.push_back(f2);
level_files_.push_back(f1);
compaction_files_.push_back(f2);
compaction_files_.push_back(f3);
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
ASSERT_EQ(2, compaction_files_.size());
}
TEST_F(AddBoundaryInputsTest, TestOneBoundaryFiles) {
FileMetaData* f1 =
CreateFileMetaData(1, InternalKey("100", 3, kTypeValue),
InternalKey(InternalKey("100", 2, kTypeValue)));
FileMetaData* f2 =
CreateFileMetaData(1, InternalKey("100", 1, kTypeValue),
InternalKey(InternalKey("200", 3, kTypeValue)));
FileMetaData* f3 =
CreateFileMetaData(1, InternalKey("300", 2, kTypeValue),
InternalKey(InternalKey("300", 1, kTypeValue)));
level_files_.push_back(f3);
level_files_.push_back(f2);
level_files_.push_back(f1);
compaction_files_.push_back(f1);
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
ASSERT_EQ(2, compaction_files_.size());
ASSERT_EQ(f1, compaction_files_[0]);
ASSERT_EQ(f2, compaction_files_[1]);
}
TEST_F(AddBoundaryInputsTest, TestTwoBoundaryFiles) {
FileMetaData* f1 =
CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
InternalKey(InternalKey("100", 5, kTypeValue)));
FileMetaData* f2 =
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
InternalKey(InternalKey("300", 1, kTypeValue)));
FileMetaData* f3 =
CreateFileMetaData(1, InternalKey("100", 4, kTypeValue),
InternalKey(InternalKey("100", 3, kTypeValue)));
level_files_.push_back(f2);
level_files_.push_back(f3);
level_files_.push_back(f1);
compaction_files_.push_back(f1);
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
ASSERT_EQ(3, compaction_files_.size());
ASSERT_EQ(f1, compaction_files_[0]);
ASSERT_EQ(f3, compaction_files_[1]);
ASSERT_EQ(f2, compaction_files_[2]);
}
TEST_F(AddBoundaryInputsTest, TestDisjoinFilePointers) {
FileMetaData* f1 =
CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
InternalKey(InternalKey("100", 5, kTypeValue)));
FileMetaData* f2 =
CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
InternalKey(InternalKey("100", 5, kTypeValue)));
FileMetaData* f3 =
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
InternalKey(InternalKey("300", 1, kTypeValue)));
FileMetaData* f4 =
CreateFileMetaData(1, InternalKey("100", 4, kTypeValue),
InternalKey(InternalKey("100", 3, kTypeValue)));
level_files_.push_back(f2);
level_files_.push_back(f3);
level_files_.push_back(f4);
compaction_files_.push_back(f1);
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
ASSERT_EQ(3, compaction_files_.size());
ASSERT_EQ(f1, compaction_files_[0]);
ASSERT_EQ(f4, compaction_files_[1]);
ASSERT_EQ(f3, compaction_files_[2]);
}
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -15,10 +15,10 @@
#include "leveldb/write_batch.h" #include "leveldb/write_batch.h"
#include "leveldb/db.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/memtable.h" #include "db/memtable.h"
#include "db/write_batch_internal.h" #include "db/write_batch_internal.h"
#include "leveldb/db.h"
#include "util/coding.h" #include "util/coding.h"
namespace leveldb { namespace leveldb {
@ -26,22 +26,18 @@ namespace leveldb {
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count. // WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
static const size_t kHeader = 12; static const size_t kHeader = 12;
WriteBatch::WriteBatch() { WriteBatch::WriteBatch() { Clear(); }
Clear();
}
WriteBatch::~WriteBatch() { } WriteBatch::~WriteBatch() = default;
WriteBatch::Handler::~Handler() { } WriteBatch::Handler::~Handler() = default;
void WriteBatch::Clear() { void WriteBatch::Clear() {
rep_.clear(); rep_.clear();
rep_.resize(kHeader); rep_.resize(kHeader);
} }
size_t WriteBatch::ApproximateSize() { size_t WriteBatch::ApproximateSize() const { return rep_.size(); }
return rep_.size();
}
Status WriteBatch::Iterate(Handler* handler) const { Status WriteBatch::Iterate(Handler* handler) const {
Slice input(rep_); Slice input(rep_);
@ -112,25 +108,28 @@ void WriteBatch::Delete(const Slice& key) {
PutLengthPrefixedSlice(&rep_, key); PutLengthPrefixedSlice(&rep_, key);
} }
void WriteBatch::Append(const WriteBatch& source) {
WriteBatchInternal::Append(this, &source);
}
namespace { namespace {
class MemTableInserter : public WriteBatch::Handler { class MemTableInserter : public WriteBatch::Handler {
public: public:
SequenceNumber sequence_; SequenceNumber sequence_;
MemTable* mem_; MemTable* mem_;
virtual void Put(const Slice& key, const Slice& value) { void Put(const Slice& key, const Slice& value) override {
mem_->Add(sequence_, kTypeValue, key, value); mem_->Add(sequence_, kTypeValue, key, value);
sequence_++; sequence_++;
} }
virtual void Delete(const Slice& key) { void Delete(const Slice& key) override {
mem_->Add(sequence_, kTypeDeletion, key, Slice()); mem_->Add(sequence_, kTypeDeletion, key, Slice());
sequence_++; sequence_++;
} }
}; };
} // namespace } // namespace
Status WriteBatchInternal::InsertInto(const WriteBatch* b, Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) {
MemTable* memtable) {
MemTableInserter inserter; MemTableInserter inserter;
inserter.sequence_ = WriteBatchInternal::Sequence(b); inserter.sequence_ = WriteBatchInternal::Sequence(b);
inserter.mem_ = memtable; inserter.mem_ = memtable;

View File

@ -29,13 +29,9 @@ class WriteBatchInternal {
// this batch. // this batch.
static void SetSequence(WriteBatch* batch, SequenceNumber seq); static void SetSequence(WriteBatch* batch, SequenceNumber seq);
static Slice Contents(const WriteBatch* batch) { static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); }
return Slice(batch->rep_);
}
static size_t ByteSize(const WriteBatch* batch) { static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); }
return batch->rep_.size();
}
static void SetContents(WriteBatch* batch, const Slice& contents); static void SetContents(WriteBatch* batch, const Slice& contents);
@ -46,5 +42,4 @@ class WriteBatchInternal {
} // namespace leveldb } // namespace leveldb
#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_

View File

@ -2,13 +2,12 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "leveldb/db.h" #include "gtest/gtest.h"
#include "db/memtable.h" #include "db/memtable.h"
#include "db/write_batch_internal.h" #include "db/write_batch_internal.h"
#include "leveldb/db.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
namespace leveldb { namespace leveldb {
@ -22,7 +21,7 @@ static std::string PrintContents(WriteBatch* b) {
Iterator* iter = mem->NewIterator(); Iterator* iter = mem->NewIterator();
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ParsedInternalKey ikey; ParsedInternalKey ikey;
ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey)); EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey));
switch (ikey.type) { switch (ikey.type) {
case kTypeValue: case kTypeValue:
state.append("Put("); state.append("Put(");
@ -52,8 +51,6 @@ static std::string PrintContents(WriteBatch* b) {
return state; return state;
} }
class WriteBatchTest { };
TEST(WriteBatchTest, Empty) { TEST(WriteBatchTest, Empty) {
WriteBatch batch; WriteBatch batch;
ASSERT_EQ("", PrintContents(&batch)); ASSERT_EQ("", PrintContents(&batch));
@ -68,7 +65,8 @@ TEST(WriteBatchTest, Multiple) {
WriteBatchInternal::SetSequence(&batch, 100); WriteBatchInternal::SetSequence(&batch, 100);
ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch));
ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); ASSERT_EQ(3, WriteBatchInternal::Count(&batch));
ASSERT_EQ("Put(baz, boo)@102" ASSERT_EQ(
"Put(baz, boo)@102"
"Delete(box)@101" "Delete(box)@101"
"Put(foo, bar)@100", "Put(foo, bar)@100",
PrintContents(&batch)); PrintContents(&batch));
@ -82,7 +80,8 @@ TEST(WriteBatchTest, Corruption) {
Slice contents = WriteBatchInternal::Contents(&batch); Slice contents = WriteBatchInternal::Contents(&batch);
WriteBatchInternal::SetContents(&batch, WriteBatchInternal::SetContents(&batch,
Slice(contents.data(), contents.size() - 1)); Slice(contents.data(), contents.size() - 1));
ASSERT_EQ("Put(foo, bar)@200" ASSERT_EQ(
"Put(foo, bar)@200"
"ParseError()", "ParseError()",
PrintContents(&batch)); PrintContents(&batch));
} }
@ -91,22 +90,22 @@ TEST(WriteBatchTest, Append) {
WriteBatch b1, b2; WriteBatch b1, b2;
WriteBatchInternal::SetSequence(&b1, 200); WriteBatchInternal::SetSequence(&b1, 200);
WriteBatchInternal::SetSequence(&b2, 300); WriteBatchInternal::SetSequence(&b2, 300);
WriteBatchInternal::Append(&b1, &b2); b1.Append(b2);
ASSERT_EQ("", ASSERT_EQ("", PrintContents(&b1));
PrintContents(&b1));
b2.Put("a", "va"); b2.Put("a", "va");
WriteBatchInternal::Append(&b1, &b2); b1.Append(b2);
ASSERT_EQ("Put(a, va)@200", ASSERT_EQ("Put(a, va)@200", PrintContents(&b1));
PrintContents(&b1));
b2.Clear(); b2.Clear();
b2.Put("b", "vb"); b2.Put("b", "vb");
WriteBatchInternal::Append(&b1, &b2); b1.Append(b2);
ASSERT_EQ("Put(a, va)@200" ASSERT_EQ(
"Put(a, va)@200"
"Put(b, vb)@201", "Put(b, vb)@201",
PrintContents(&b1)); PrintContents(&b1));
b2.Delete("foo"); b2.Delete("foo");
WriteBatchInternal::Append(&b1, &b2); b1.Append(b2);
ASSERT_EQ("Put(a, va)@200" ASSERT_EQ(
"Put(a, va)@200"
"Put(b, vb)@202" "Put(b, vb)@202"
"Put(b, vb)@201" "Put(b, vb)@201"
"Delete(foo)@203", "Delete(foo)@203",
@ -133,5 +132,6 @@ TEST(WriteBatchTest, ApproximateSize) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -90,9 +90,9 @@ div.bsql {
<h4>Benchmark Source Code</h4> <h4>Benchmark Source Code</h4>
<p>We wrote benchmark tools for SQLite and Kyoto TreeDB based on LevelDB's <span class="code">db_bench</span>. The code for each of the benchmarks resides here:</p> <p>We wrote benchmark tools for SQLite and Kyoto TreeDB based on LevelDB's <span class="code">db_bench</span>. The code for each of the benchmarks resides here:</p>
<ul> <ul>
<li> <b>LevelDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/trunk/db/db_bench.cc">db/db_bench.cc</a>.</li> <li> <b>LevelDB:</b> <a href="https://github.com/google/leveldb/blob/master/benchmarks/db_bench.cc">benchmarks/db_bench.cc</a>.</li>
<li> <b>SQLite:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_sqlite3.cc">doc/bench/db_bench_sqlite3.cc</a>.</li> <li> <b>SQLite:</b> <a href="https://github.com/google/leveldb/blob/master/benchmarks/db_bench_sqlite3.cc">benchmarks/db_bench_sqlite3.cc</a>.</li>
<li> <b>Kyoto TreeDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_tree_db.cc">doc/bench/db_bench_tree_db.cc</a>.</li> <li> <b>Kyoto TreeDB:</b> <a href="https://github.com/google/leveldb/blob/master/benchmarks/db_bench_tree_db.cc">benchmarks/db_bench_tree_db.cc</a>.</li>
</ul> </ul>
<h4>Custom Build Specifications</h4> <h4>Custom Build Specifications</h4>

View File

@ -307,7 +307,7 @@ version numbers found in the keys to decide how to interpret them.
## Performance ## Performance
Performance can be tuned by changing the default values of the types defined in Performance can be tuned by changing the default values of the types defined in
`include/leveldb/options.h`. `include/options.h`.
### Block size ### Block size

View File

@ -27,6 +27,10 @@ class FileState {
// and the caller must call Ref() at least once. // and the caller must call Ref() at least once.
FileState() : refs_(0), size_(0) {} FileState() : refs_(0), size_(0) {}
// No copying allowed.
FileState(const FileState&) = delete;
FileState& operator=(const FileState&) = delete;
// Increase the reference count. // Increase the reference count.
void Ref() { void Ref() {
MutexLock lock(&refs_mutex_); MutexLock lock(&refs_mutex_);
@ -51,9 +55,22 @@ class FileState {
} }
} }
uint64_t Size() const { return size_; } uint64_t Size() const {
MutexLock lock(&blocks_mutex_);
return size_;
}
void Truncate() {
MutexLock lock(&blocks_mutex_);
for (char*& block : blocks_) {
delete[] block;
}
blocks_.clear();
size_ = 0;
}
Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
MutexLock lock(&blocks_mutex_);
if (offset > size_) { if (offset > size_) {
return Status::IOError("Offset greater than file size."); return Status::IOError("Offset greater than file size.");
} }
@ -69,13 +86,6 @@ class FileState {
assert(offset / kBlockSize <= std::numeric_limits<size_t>::max()); assert(offset / kBlockSize <= std::numeric_limits<size_t>::max());
size_t block = static_cast<size_t>(offset / kBlockSize); size_t block = static_cast<size_t>(offset / kBlockSize);
size_t block_offset = offset % kBlockSize; size_t block_offset = offset % kBlockSize;
if (n <= kBlockSize - block_offset) {
// The requested bytes are all in the first block.
*result = Slice(blocks_[block] + block_offset, n);
return Status::OK();
}
size_t bytes_to_copy = n; size_t bytes_to_copy = n;
char* dst = scratch; char* dst = scratch;
@ -100,6 +110,7 @@ class FileState {
const char* src = data.data(); const char* src = data.data();
size_t src_len = data.size(); size_t src_len = data.size();
MutexLock lock(&blocks_mutex_);
while (src_len > 0) { while (src_len > 0) {
size_t avail; size_t avail;
size_t offset = size_ % kBlockSize; size_t offset = size_ % kBlockSize;
@ -126,28 +137,17 @@ class FileState {
} }
private: private:
// Private since only Unref() should be used to delete it. enum { kBlockSize = 8 * 1024 };
~FileState() {
for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
++i) {
delete [] *i;
}
}
// No copying allowed. // Private since only Unref() should be used to delete it.
FileState(const FileState&); ~FileState() { Truncate(); }
void operator=(const FileState&);
port::Mutex refs_mutex_; port::Mutex refs_mutex_;
int refs_ GUARDED_BY(refs_mutex_); int refs_ GUARDED_BY(refs_mutex_);
// The following fields are not protected by any mutex. They are only mutable mutable port::Mutex blocks_mutex_;
// while the file is being written, and concurrent access is not allowed std::vector<char*> blocks_ GUARDED_BY(blocks_mutex_);
// to writable files. uint64_t size_ GUARDED_BY(blocks_mutex_);
std::vector<char*> blocks_;
uint64_t size_;
enum { kBlockSize = 8 * 1024 };
}; };
class SequentialFileImpl : public SequentialFile { class SequentialFileImpl : public SequentialFile {
@ -156,11 +156,9 @@ class SequentialFileImpl : public SequentialFile {
file_->Ref(); file_->Ref();
} }
~SequentialFileImpl() { ~SequentialFileImpl() override { file_->Unref(); }
file_->Unref();
}
virtual Status Read(size_t n, Slice* result, char* scratch) { Status Read(size_t n, Slice* result, char* scratch) override {
Status s = file_->Read(pos_, n, result, scratch); Status s = file_->Read(pos_, n, result, scratch);
if (s.ok()) { if (s.ok()) {
pos_ += result->size(); pos_ += result->size();
@ -168,7 +166,7 @@ class SequentialFileImpl : public SequentialFile {
return s; return s;
} }
virtual Status Skip(uint64_t n) { Status Skip(uint64_t n) override {
if (pos_ > file_->Size()) { if (pos_ > file_->Size()) {
return Status::IOError("pos_ > file_->Size()"); return Status::IOError("pos_ > file_->Size()");
} }
@ -187,16 +185,12 @@ class SequentialFileImpl : public SequentialFile {
class RandomAccessFileImpl : public RandomAccessFile { class RandomAccessFileImpl : public RandomAccessFile {
public: public:
explicit RandomAccessFileImpl(FileState* file) : file_(file) { explicit RandomAccessFileImpl(FileState* file) : file_(file) { file_->Ref(); }
file_->Ref();
}
~RandomAccessFileImpl() { ~RandomAccessFileImpl() override { file_->Unref(); }
file_->Unref();
}
virtual Status Read(uint64_t offset, size_t n, Slice* result, Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const { char* scratch) const override {
return file_->Read(offset, n, result, scratch); return file_->Read(offset, n, result, scratch);
} }
@ -206,21 +200,15 @@ class RandomAccessFileImpl : public RandomAccessFile {
class WritableFileImpl : public WritableFile { class WritableFileImpl : public WritableFile {
public: public:
WritableFileImpl(FileState* file) : file_(file) { WritableFileImpl(FileState* file) : file_(file) { file_->Ref(); }
file_->Ref();
}
~WritableFileImpl() { ~WritableFileImpl() override { file_->Unref(); }
file_->Unref();
}
virtual Status Append(const Slice& data) { Status Append(const Slice& data) override { return file_->Append(data); }
return file_->Append(data);
}
virtual Status Close() { return Status::OK(); } Status Close() override { return Status::OK(); }
virtual Status Flush() { return Status::OK(); } Status Flush() override { return Status::OK(); }
virtual Status Sync() { return Status::OK(); } Status Sync() override { return Status::OK(); }
private: private:
FileState* file_; FileState* file_;
@ -228,22 +216,22 @@ class WritableFileImpl : public WritableFile {
class NoOpLogger : public Logger { class NoOpLogger : public Logger {
public: public:
virtual void Logv(const char* format, va_list ap) { } void Logv(const char* format, va_list ap) override {}
}; };
class InMemoryEnv : public EnvWrapper { class InMemoryEnv : public EnvWrapper {
public: public:
explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) {} explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) {}
virtual ~InMemoryEnv() { ~InMemoryEnv() override {
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ for (const auto& kvp : file_map_) {
i->second->Unref(); kvp.second->Unref();
} }
} }
// Partial implementation of the Env interface. // Partial implementation of the Env interface.
virtual Status NewSequentialFile(const std::string& fname, Status NewSequentialFile(const std::string& fname,
SequentialFile** result) { SequentialFile** result) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(fname) == file_map_.end()) { if (file_map_.find(fname) == file_map_.end()) {
*result = nullptr; *result = nullptr;
@ -254,8 +242,8 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK(); return Status::OK();
} }
virtual Status NewRandomAccessFile(const std::string& fname, Status NewRandomAccessFile(const std::string& fname,
RandomAccessFile** result) { RandomAccessFile** result) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(fname) == file_map_.end()) { if (file_map_.find(fname) == file_map_.end()) {
*result = nullptr; *result = nullptr;
@ -266,23 +254,28 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK(); return Status::OK();
} }
virtual Status NewWritableFile(const std::string& fname, Status NewWritableFile(const std::string& fname,
WritableFile** result) { WritableFile** result) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(fname) != file_map_.end()) { FileSystem::iterator it = file_map_.find(fname);
DeleteFileInternal(fname);
}
FileState* file = new FileState(); FileState* file;
if (it == file_map_.end()) {
// File is not currently open.
file = new FileState();
file->Ref(); file->Ref();
file_map_[fname] = file; file_map_[fname] = file;
} else {
file = it->second;
file->Truncate();
}
*result = new WritableFileImpl(file); *result = new WritableFileImpl(file);
return Status::OK(); return Status::OK();
} }
virtual Status NewAppendableFile(const std::string& fname, Status NewAppendableFile(const std::string& fname,
WritableFile** result) { WritableFile** result) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
FileState** sptr = &file_map_[fname]; FileState** sptr = &file_map_[fname];
FileState* file = *sptr; FileState* file = *sptr;
@ -294,18 +287,18 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK(); return Status::OK();
} }
virtual bool FileExists(const std::string& fname) { bool FileExists(const std::string& fname) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
return file_map_.find(fname) != file_map_.end(); return file_map_.find(fname) != file_map_.end();
} }
virtual Status GetChildren(const std::string& dir, Status GetChildren(const std::string& dir,
std::vector<std::string>* result) { std::vector<std::string>* result) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
result->clear(); result->clear();
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ for (const auto& kvp : file_map_) {
const std::string& filename = i->first; const std::string& filename = kvp.first;
if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' && if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
Slice(filename).starts_with(Slice(dir))) { Slice(filename).starts_with(Slice(dir))) {
@ -326,7 +319,7 @@ class InMemoryEnv : public EnvWrapper {
file_map_.erase(fname); file_map_.erase(fname);
} }
virtual Status DeleteFile(const std::string& fname) { Status DeleteFile(const std::string& fname) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(fname) == file_map_.end()) { if (file_map_.find(fname) == file_map_.end()) {
return Status::IOError(fname, "File not found"); return Status::IOError(fname, "File not found");
@ -336,15 +329,11 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK(); return Status::OK();
} }
virtual Status CreateDir(const std::string& dirname) { Status CreateDir(const std::string& dirname) override { return Status::OK(); }
return Status::OK();
}
virtual Status DeleteDir(const std::string& dirname) { Status DeleteDir(const std::string& dirname) override { return Status::OK(); }
return Status::OK();
}
virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) { Status GetFileSize(const std::string& fname, uint64_t* file_size) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(fname) == file_map_.end()) { if (file_map_.find(fname) == file_map_.end()) {
return Status::IOError(fname, "File not found"); return Status::IOError(fname, "File not found");
@ -354,8 +343,8 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK(); return Status::OK();
} }
virtual Status RenameFile(const std::string& src, Status RenameFile(const std::string& src,
const std::string& target) { const std::string& target) override {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(src) == file_map_.end()) { if (file_map_.find(src) == file_map_.end()) {
return Status::IOError(src, "File not found"); return Status::IOError(src, "File not found");
@ -367,22 +356,22 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK(); return Status::OK();
} }
virtual Status LockFile(const std::string& fname, FileLock** lock) { Status LockFile(const std::string& fname, FileLock** lock) override {
*lock = new FileLock; *lock = new FileLock;
return Status::OK(); return Status::OK();
} }
virtual Status UnlockFile(FileLock* lock) { Status UnlockFile(FileLock* lock) override {
delete lock; delete lock;
return Status::OK(); return Status::OK();
} }
virtual Status GetTestDirectory(std::string* path) { Status GetTestDirectory(std::string* path) override {
*path = "/test"; *path = "/test";
return Status::OK(); return Status::OK();
} }
virtual Status NewLogger(const std::string& fname, Logger** result) { Status NewLogger(const std::string& fname, Logger** result) override {
*result = new NoOpLogger; *result = new NoOpLogger;
return Status::OK(); return Status::OK();
} }
@ -390,14 +379,13 @@ class InMemoryEnv : public EnvWrapper {
private: private:
// Map from filenames to FileState objects, representing a simple file system. // Map from filenames to FileState objects, representing a simple file system.
typedef std::map<std::string, FileState*> FileSystem; typedef std::map<std::string, FileState*> FileSystem;
port::Mutex mutex_; port::Mutex mutex_;
FileSystem file_map_ GUARDED_BY(mutex_); FileSystem file_map_ GUARDED_BY(mutex_);
}; };
} // namespace } // namespace
Env* NewMemEnv(Env* base_env) { Env* NewMemEnv(Env* base_env) { return new InMemoryEnv(base_env); }
return new InMemoryEnv(base_env);
}
} // namespace leveldb } // namespace leveldb

View File

@ -4,76 +4,74 @@
#include "helpers/memenv/memenv.h" #include "helpers/memenv/memenv.h"
#include "db/db_impl.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "util/testharness.h"
#include <string> #include <string>
#include <vector> #include <vector>
#include "gtest/gtest.h"
#include "db/db_impl.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "util/testutil.h"
namespace leveldb { namespace leveldb {
class MemEnvTest { class MemEnvTest : public testing::Test {
public: public:
Env* env_; MemEnvTest() : env_(NewMemEnv(Env::Default())) {}
~MemEnvTest() { delete env_; }
MemEnvTest() Env* env_;
: env_(NewMemEnv(Env::Default())) {
}
~MemEnvTest() {
delete env_;
}
}; };
TEST(MemEnvTest, Basics) { TEST_F(MemEnvTest, Basics) {
uint64_t file_size; uint64_t file_size;
WritableFile* writable_file; WritableFile* writable_file;
std::vector<std::string> children; std::vector<std::string> children;
ASSERT_OK(env_->CreateDir("/dir")); ASSERT_LEVELDB_OK(env_->CreateDir("/dir"));
// Check that the directory is empty. // Check that the directory is empty.
ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); ASSERT_TRUE(!env_->FileExists("/dir/non_existent"));
ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children));
ASSERT_EQ(0, children.size()); ASSERT_EQ(0, children.size());
// Create a file. // Create a file.
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
ASSERT_EQ(0, file_size); ASSERT_EQ(0, file_size);
delete writable_file; delete writable_file;
// Check that the file exists. // Check that the file exists.
ASSERT_TRUE(env_->FileExists("/dir/f")); ASSERT_TRUE(env_->FileExists("/dir/f"));
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
ASSERT_EQ(0, file_size); ASSERT_EQ(0, file_size);
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children));
ASSERT_EQ(1, children.size()); ASSERT_EQ(1, children.size());
ASSERT_EQ("f", children[0]); ASSERT_EQ("f", children[0]);
// Write to the file. // Write to the file.
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
ASSERT_OK(writable_file->Append("abc")); ASSERT_LEVELDB_OK(writable_file->Append("abc"));
delete writable_file; delete writable_file;
// Check that append works. // Check that append works.
ASSERT_OK(env_->NewAppendableFile("/dir/f", &writable_file)); ASSERT_LEVELDB_OK(env_->NewAppendableFile("/dir/f", &writable_file));
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
ASSERT_EQ(3, file_size); ASSERT_EQ(3, file_size);
ASSERT_OK(writable_file->Append("hello")); ASSERT_LEVELDB_OK(writable_file->Append("hello"));
delete writable_file; delete writable_file;
// Check for expected size. // Check for expected size.
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
ASSERT_EQ(8, file_size); ASSERT_EQ(8, file_size);
// Check that renaming works. // Check that renaming works.
ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); ASSERT_LEVELDB_OK(env_->RenameFile("/dir/f", "/dir/g"));
ASSERT_TRUE(!env_->FileExists("/dir/f")); ASSERT_TRUE(!env_->FileExists("/dir/f"));
ASSERT_TRUE(env_->FileExists("/dir/g")); ASSERT_TRUE(env_->FileExists("/dir/g"));
ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/g", &file_size));
ASSERT_EQ(8, file_size); ASSERT_EQ(8, file_size);
// Check that opening non-existent file fails. // Check that opening non-existent file fails.
@ -86,48 +84,49 @@ TEST(MemEnvTest, Basics) {
// Check that deleting works. // Check that deleting works.
ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
ASSERT_OK(env_->DeleteFile("/dir/g")); ASSERT_LEVELDB_OK(env_->DeleteFile("/dir/g"));
ASSERT_TRUE(!env_->FileExists("/dir/g")); ASSERT_TRUE(!env_->FileExists("/dir/g"));
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children));
ASSERT_EQ(0, children.size()); ASSERT_EQ(0, children.size());
ASSERT_OK(env_->DeleteDir("/dir")); ASSERT_LEVELDB_OK(env_->DeleteDir("/dir"));
} }
TEST(MemEnvTest, ReadWrite) { TEST_F(MemEnvTest, ReadWrite) {
WritableFile* writable_file; WritableFile* writable_file;
SequentialFile* seq_file; SequentialFile* seq_file;
RandomAccessFile* rand_file; RandomAccessFile* rand_file;
Slice result; Slice result;
char scratch[100]; char scratch[100];
ASSERT_OK(env_->CreateDir("/dir")); ASSERT_LEVELDB_OK(env_->CreateDir("/dir"));
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
ASSERT_OK(writable_file->Append("hello ")); ASSERT_LEVELDB_OK(writable_file->Append("hello "));
ASSERT_OK(writable_file->Append("world")); ASSERT_LEVELDB_OK(writable_file->Append("world"));
delete writable_file; delete writable_file;
// Read sequentially. // Read sequentially.
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); ASSERT_LEVELDB_OK(env_->NewSequentialFile("/dir/f", &seq_file));
ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello". ASSERT_LEVELDB_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
ASSERT_EQ(0, result.compare("hello")); ASSERT_EQ(0, result.compare("hello"));
ASSERT_OK(seq_file->Skip(1)); ASSERT_LEVELDB_OK(seq_file->Skip(1));
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world". ASSERT_LEVELDB_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
ASSERT_EQ(0, result.compare("world")); ASSERT_EQ(0, result.compare("world"));
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF. ASSERT_LEVELDB_OK(
seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
ASSERT_EQ(0, result.size()); ASSERT_EQ(0, result.size());
ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file. ASSERT_LEVELDB_OK(seq_file->Skip(100)); // Try to skip past end of file.
ASSERT_OK(seq_file->Read(1000, &result, scratch)); ASSERT_LEVELDB_OK(seq_file->Read(1000, &result, scratch));
ASSERT_EQ(0, result.size()); ASSERT_EQ(0, result.size());
delete seq_file; delete seq_file;
// Random reads. // Random reads.
ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file)); ASSERT_LEVELDB_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world". ASSERT_LEVELDB_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
ASSERT_EQ(0, result.compare("world")); ASSERT_EQ(0, result.compare("world"));
ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello". ASSERT_LEVELDB_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
ASSERT_EQ(0, result.compare("hello")); ASSERT_EQ(0, result.compare("hello"));
ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d". ASSERT_LEVELDB_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
ASSERT_EQ(0, result.compare("d")); ASSERT_EQ(0, result.compare("d"));
// Too high offset. // Too high offset.
@ -135,30 +134,30 @@ TEST(MemEnvTest, ReadWrite) {
delete rand_file; delete rand_file;
} }
TEST(MemEnvTest, Locks) { TEST_F(MemEnvTest, Locks) {
FileLock* lock; FileLock* lock;
// These are no-ops, but we test they return success. // These are no-ops, but we test they return success.
ASSERT_OK(env_->LockFile("some file", &lock)); ASSERT_LEVELDB_OK(env_->LockFile("some file", &lock));
ASSERT_OK(env_->UnlockFile(lock)); ASSERT_LEVELDB_OK(env_->UnlockFile(lock));
} }
TEST(MemEnvTest, Misc) { TEST_F(MemEnvTest, Misc) {
std::string test_dir; std::string test_dir;
ASSERT_OK(env_->GetTestDirectory(&test_dir)); ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));
ASSERT_TRUE(!test_dir.empty()); ASSERT_TRUE(!test_dir.empty());
WritableFile* writable_file; WritableFile* writable_file;
ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file)); ASSERT_LEVELDB_OK(env_->NewWritableFile("/a/b", &writable_file));
// These are no-ops, but we test they return success. // These are no-ops, but we test they return success.
ASSERT_OK(writable_file->Sync()); ASSERT_LEVELDB_OK(writable_file->Sync());
ASSERT_OK(writable_file->Flush()); ASSERT_LEVELDB_OK(writable_file->Flush());
ASSERT_OK(writable_file->Close()); ASSERT_LEVELDB_OK(writable_file->Close());
delete writable_file; delete writable_file;
} }
TEST(MemEnvTest, LargeWrite) { TEST_F(MemEnvTest, LargeWrite) {
const size_t kWriteSize = 300 * 1024; const size_t kWriteSize = 300 * 1024;
char* scratch = new char[kWriteSize * 2]; char* scratch = new char[kWriteSize * 2];
@ -168,21 +167,21 @@ TEST(MemEnvTest, LargeWrite) {
} }
WritableFile* writable_file; WritableFile* writable_file;
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
ASSERT_OK(writable_file->Append("foo")); ASSERT_LEVELDB_OK(writable_file->Append("foo"));
ASSERT_OK(writable_file->Append(write_data)); ASSERT_LEVELDB_OK(writable_file->Append(write_data));
delete writable_file; delete writable_file;
SequentialFile* seq_file; SequentialFile* seq_file;
Slice result; Slice result;
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); ASSERT_LEVELDB_OK(env_->NewSequentialFile("/dir/f", &seq_file));
ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo". ASSERT_LEVELDB_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
ASSERT_EQ(0, result.compare("foo")); ASSERT_EQ(0, result.compare("foo"));
size_t read = 0; size_t read = 0;
std::string read_data; std::string read_data;
while (read < kWriteSize) { while (read < kWriteSize) {
ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch)); ASSERT_LEVELDB_OK(seq_file->Read(kWriteSize - read, &result, scratch));
read_data.append(result.data(), result.size()); read_data.append(result.data(), result.size());
read += result.size(); read += result.size();
} }
@ -191,7 +190,30 @@ TEST(MemEnvTest, LargeWrite) {
delete[] scratch; delete[] scratch;
} }
TEST(MemEnvTest, DBTest) { TEST_F(MemEnvTest, OverwriteOpenFile) {
const char kWrite1Data[] = "Write #1 data";
const size_t kFileDataLen = sizeof(kWrite1Data) - 1;
const std::string kTestFileName = testing::TempDir() + "leveldb-TestFile.dat";
ASSERT_LEVELDB_OK(WriteStringToFile(env_, kWrite1Data, kTestFileName));
RandomAccessFile* rand_file;
ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(kTestFileName, &rand_file));
const char kWrite2Data[] = "Write #2 data";
ASSERT_LEVELDB_OK(WriteStringToFile(env_, kWrite2Data, kTestFileName));
// Verify that overwriting an open file will result in the new file data
// being read from files opened before the write.
Slice result;
char scratch[kFileDataLen];
ASSERT_LEVELDB_OK(rand_file->Read(0, kFileDataLen, &result, scratch));
ASSERT_EQ(0, result.compare(kWrite2Data));
delete rand_file;
}
TEST_F(MemEnvTest, DBTest) {
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
options.env = env_; options.env = env_;
@ -200,14 +222,14 @@ TEST(MemEnvTest, DBTest) {
const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")};
const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")};
ASSERT_OK(DB::Open(options, "/dir/db", &db)); ASSERT_LEVELDB_OK(DB::Open(options, "/dir/db", &db));
for (size_t i = 0; i < 3; ++i) { for (size_t i = 0; i < 3; ++i) {
ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); ASSERT_LEVELDB_OK(db->Put(WriteOptions(), keys[i], vals[i]));
} }
for (size_t i = 0; i < 3; ++i) { for (size_t i = 0; i < 3; ++i) {
std::string res; std::string res;
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); ASSERT_LEVELDB_OK(db->Get(ReadOptions(), keys[i], &res));
ASSERT_TRUE(res == vals[i]); ASSERT_TRUE(res == vals[i]);
} }
@ -223,11 +245,11 @@ TEST(MemEnvTest, DBTest) {
delete iterator; delete iterator;
DBImpl* dbi = reinterpret_cast<DBImpl*>(db); DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
ASSERT_OK(dbi->TEST_CompactMemTable()); ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable());
for (size_t i = 0; i < 3; ++i) { for (size_t i = 0; i < 3; ++i) {
std::string res; std::string res;
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); ASSERT_LEVELDB_OK(db->Get(ReadOptions(), keys[i], &res));
ASSERT_TRUE(res == vals[i]); ASSERT_TRUE(res == vals[i]);
} }
@ -237,5 +259,6 @@ TEST(MemEnvTest, DBTest) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -32,7 +32,7 @@
On failure, leveldb frees the old value of *errptr and On failure, leveldb frees the old value of *errptr and
set *errptr to a malloc()ed error message. set *errptr to a malloc()ed error message.
(4) Bools have the type unsigned char (0 == false; rest == true) (4) Bools have the type uint8_t (0 == false; rest == true)
(5) All of the pointer arguments must be non-NULL. (5) All of the pointer arguments must be non-NULL.
*/ */
@ -47,6 +47,7 @@ extern "C" {
#include <stdarg.h> #include <stdarg.h>
#include <stddef.h> #include <stddef.h>
#include <stdint.h> #include <stdint.h>
#include "leveldb/export.h" #include "leveldb/export.h"
/* Exported types */ /* Exported types */
@ -130,7 +131,7 @@ LEVELDB_EXPORT void leveldb_repair_db(const leveldb_options_t* options,
/* Iterator */ /* Iterator */
LEVELDB_EXPORT void leveldb_iter_destroy(leveldb_iterator_t*); LEVELDB_EXPORT void leveldb_iter_destroy(leveldb_iterator_t*);
LEVELDB_EXPORT unsigned char leveldb_iter_valid(const leveldb_iterator_t*); LEVELDB_EXPORT uint8_t leveldb_iter_valid(const leveldb_iterator_t*);
LEVELDB_EXPORT void leveldb_iter_seek_to_first(leveldb_iterator_t*); LEVELDB_EXPORT void leveldb_iter_seek_to_first(leveldb_iterator_t*);
LEVELDB_EXPORT void leveldb_iter_seek_to_last(leveldb_iterator_t*); LEVELDB_EXPORT void leveldb_iter_seek_to_last(leveldb_iterator_t*);
LEVELDB_EXPORT void leveldb_iter_seek(leveldb_iterator_t*, const char* k, LEVELDB_EXPORT void leveldb_iter_seek(leveldb_iterator_t*, const char* k,
@ -146,7 +147,7 @@ LEVELDB_EXPORT void leveldb_iter_get_error(const leveldb_iterator_t*,
/* Write batch */ /* Write batch */
LEVELDB_EXPORT leveldb_writebatch_t* leveldb_writebatch_create(); LEVELDB_EXPORT leveldb_writebatch_t* leveldb_writebatch_create(void);
LEVELDB_EXPORT void leveldb_writebatch_destroy(leveldb_writebatch_t*); LEVELDB_EXPORT void leveldb_writebatch_destroy(leveldb_writebatch_t*);
LEVELDB_EXPORT void leveldb_writebatch_clear(leveldb_writebatch_t*); LEVELDB_EXPORT void leveldb_writebatch_clear(leveldb_writebatch_t*);
LEVELDB_EXPORT void leveldb_writebatch_put(leveldb_writebatch_t*, LEVELDB_EXPORT void leveldb_writebatch_put(leveldb_writebatch_t*,
@ -155,24 +156,26 @@ LEVELDB_EXPORT void leveldb_writebatch_put(leveldb_writebatch_t*,
LEVELDB_EXPORT void leveldb_writebatch_delete(leveldb_writebatch_t*, LEVELDB_EXPORT void leveldb_writebatch_delete(leveldb_writebatch_t*,
const char* key, size_t klen); const char* key, size_t klen);
LEVELDB_EXPORT void leveldb_writebatch_iterate( LEVELDB_EXPORT void leveldb_writebatch_iterate(
leveldb_writebatch_t*, void* state, const leveldb_writebatch_t*, void* state,
void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
void (*deleted)(void*, const char* k, size_t klen)); void (*deleted)(void*, const char* k, size_t klen));
LEVELDB_EXPORT void leveldb_writebatch_append(
leveldb_writebatch_t* destination, const leveldb_writebatch_t* source);
/* Options */ /* Options */
LEVELDB_EXPORT leveldb_options_t* leveldb_options_create(); LEVELDB_EXPORT leveldb_options_t* leveldb_options_create(void);
LEVELDB_EXPORT void leveldb_options_destroy(leveldb_options_t*); LEVELDB_EXPORT void leveldb_options_destroy(leveldb_options_t*);
LEVELDB_EXPORT void leveldb_options_set_comparator(leveldb_options_t*, LEVELDB_EXPORT void leveldb_options_set_comparator(leveldb_options_t*,
leveldb_comparator_t*); leveldb_comparator_t*);
LEVELDB_EXPORT void leveldb_options_set_filter_policy(leveldb_options_t*, LEVELDB_EXPORT void leveldb_options_set_filter_policy(leveldb_options_t*,
leveldb_filterpolicy_t*); leveldb_filterpolicy_t*);
LEVELDB_EXPORT void leveldb_options_set_create_if_missing(leveldb_options_t*, LEVELDB_EXPORT void leveldb_options_set_create_if_missing(leveldb_options_t*,
unsigned char); uint8_t);
LEVELDB_EXPORT void leveldb_options_set_error_if_exists(leveldb_options_t*, LEVELDB_EXPORT void leveldb_options_set_error_if_exists(leveldb_options_t*,
unsigned char); uint8_t);
LEVELDB_EXPORT void leveldb_options_set_paranoid_checks(leveldb_options_t*, LEVELDB_EXPORT void leveldb_options_set_paranoid_checks(leveldb_options_t*,
unsigned char); uint8_t);
LEVELDB_EXPORT void leveldb_options_set_env(leveldb_options_t*, leveldb_env_t*); LEVELDB_EXPORT void leveldb_options_set_env(leveldb_options_t*, leveldb_env_t*);
LEVELDB_EXPORT void leveldb_options_set_info_log(leveldb_options_t*, LEVELDB_EXPORT void leveldb_options_set_info_log(leveldb_options_t*,
leveldb_logger_t*); leveldb_logger_t*);
@ -187,10 +190,7 @@ LEVELDB_EXPORT void leveldb_options_set_block_restart_interval(
LEVELDB_EXPORT void leveldb_options_set_max_file_size(leveldb_options_t*, LEVELDB_EXPORT void leveldb_options_set_max_file_size(leveldb_options_t*,
size_t); size_t);
enum { enum { leveldb_no_compression = 0, leveldb_snappy_compression = 1 };
leveldb_no_compression = 0,
leveldb_snappy_compression = 1
};
LEVELDB_EXPORT void leveldb_options_set_compression(leveldb_options_t*, int); LEVELDB_EXPORT void leveldb_options_set_compression(leveldb_options_t*, int);
/* Comparator */ /* Comparator */
@ -209,7 +209,7 @@ LEVELDB_EXPORT leveldb_filterpolicy_t* leveldb_filterpolicy_create(
char* (*create_filter)(void*, const char* const* key_array, char* (*create_filter)(void*, const char* const* key_array,
const size_t* key_length_array, int num_keys, const size_t* key_length_array, int num_keys,
size_t* filter_length), size_t* filter_length),
unsigned char (*key_may_match)(void*, const char* key, size_t length, uint8_t (*key_may_match)(void*, const char* key, size_t length,
const char* filter, size_t filter_length), const char* filter, size_t filter_length),
const char* (*name)(void*)); const char* (*name)(void*));
LEVELDB_EXPORT void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t*); LEVELDB_EXPORT void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t*);
@ -219,21 +219,21 @@ LEVELDB_EXPORT leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(
/* Read options */ /* Read options */
LEVELDB_EXPORT leveldb_readoptions_t* leveldb_readoptions_create(); LEVELDB_EXPORT leveldb_readoptions_t* leveldb_readoptions_create(void);
LEVELDB_EXPORT void leveldb_readoptions_destroy(leveldb_readoptions_t*); LEVELDB_EXPORT void leveldb_readoptions_destroy(leveldb_readoptions_t*);
LEVELDB_EXPORT void leveldb_readoptions_set_verify_checksums( LEVELDB_EXPORT void leveldb_readoptions_set_verify_checksums(
leveldb_readoptions_t*, unsigned char); leveldb_readoptions_t*, uint8_t);
LEVELDB_EXPORT void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t*, LEVELDB_EXPORT void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t*,
unsigned char); uint8_t);
LEVELDB_EXPORT void leveldb_readoptions_set_snapshot(leveldb_readoptions_t*, LEVELDB_EXPORT void leveldb_readoptions_set_snapshot(leveldb_readoptions_t*,
const leveldb_snapshot_t*); const leveldb_snapshot_t*);
/* Write options */ /* Write options */
LEVELDB_EXPORT leveldb_writeoptions_t* leveldb_writeoptions_create(); LEVELDB_EXPORT leveldb_writeoptions_t* leveldb_writeoptions_create(void);
LEVELDB_EXPORT void leveldb_writeoptions_destroy(leveldb_writeoptions_t*); LEVELDB_EXPORT void leveldb_writeoptions_destroy(leveldb_writeoptions_t*);
LEVELDB_EXPORT void leveldb_writeoptions_set_sync(leveldb_writeoptions_t*, LEVELDB_EXPORT void leveldb_writeoptions_set_sync(leveldb_writeoptions_t*,
unsigned char); uint8_t);
/* Cache */ /* Cache */
@ -242,7 +242,7 @@ LEVELDB_EXPORT void leveldb_cache_destroy(leveldb_cache_t* cache);
/* Env */ /* Env */
LEVELDB_EXPORT leveldb_env_t* leveldb_create_default_env(); LEVELDB_EXPORT leveldb_env_t* leveldb_create_default_env(void);
LEVELDB_EXPORT void leveldb_env_destroy(leveldb_env_t*); LEVELDB_EXPORT void leveldb_env_destroy(leveldb_env_t*);
/* If not NULL, the returned buffer must be released using leveldb_free(). */ /* If not NULL, the returned buffer must be released using leveldb_free(). */
@ -258,10 +258,10 @@ LEVELDB_EXPORT char* leveldb_env_get_test_directory(leveldb_env_t*);
LEVELDB_EXPORT void leveldb_free(void* ptr); LEVELDB_EXPORT void leveldb_free(void* ptr);
/* Return the major version number for this release. */ /* Return the major version number for this release. */
LEVELDB_EXPORT int leveldb_major_version(); LEVELDB_EXPORT int leveldb_major_version(void);
/* Return the minor version number for this release. */ /* Return the minor version number for this release. */
LEVELDB_EXPORT int leveldb_minor_version(); LEVELDB_EXPORT int leveldb_minor_version(void);
#ifdef __cplusplus #ifdef __cplusplus
} /* end extern "C" */ } /* end extern "C" */

View File

@ -19,6 +19,7 @@
#define STORAGE_LEVELDB_INCLUDE_CACHE_H_ #define STORAGE_LEVELDB_INCLUDE_CACHE_H_
#include <stdint.h> #include <stdint.h>
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/slice.h" #include "leveldb/slice.h"

View File

@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ #define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
#include <string> #include <string>
#include "leveldb/export.h" #include "leveldb/export.h"
namespace leveldb { namespace leveldb {
@ -44,8 +45,7 @@ class LEVELDB_EXPORT Comparator {
// If *start < limit, changes *start to a short string in [start,limit). // If *start < limit, changes *start to a short string in [start,limit).
// Simple comparator implementations may return with *start unchanged, // Simple comparator implementations may return with *start unchanged,
// i.e., an implementation of this method that does nothing is correct. // i.e., an implementation of this method that does nothing is correct.
virtual void FindShortestSeparator( virtual void FindShortestSeparator(std::string* start,
std::string* start,
const Slice& limit) const = 0; const Slice& limit) const = 0;
// Changes *key to a short string >= *key. // Changes *key to a short string >= *key.

View File

@ -7,15 +7,16 @@
#include <stdint.h> #include <stdint.h>
#include <stdio.h> #include <stdio.h>
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/iterator.h" #include "leveldb/iterator.h"
#include "leveldb/options.h" #include "leveldb/options.h"
namespace leveldb { namespace leveldb {
// Update Makefile if you change these // Update CMakeLists.txt if you change these
static const int kMajorVersion = 1; static const int kMajorVersion = 1;
static const int kMinorVersion = 20; static const int kMinorVersion = 22;
struct Options; struct Options;
struct ReadOptions; struct ReadOptions;
@ -32,11 +33,11 @@ class LEVELDB_EXPORT Snapshot {
// A range of keys // A range of keys
struct LEVELDB_EXPORT Range { struct LEVELDB_EXPORT Range {
Range() = default;
Range(const Slice& s, const Slice& l) : start(s), limit(l) {}
Slice start; // Included in the range Slice start; // Included in the range
Slice limit; // Not included in the range Slice limit; // Not included in the range
Range() { }
Range(const Slice& s, const Slice& l) : start(s), limit(l) { }
}; };
// A DB is a persistent ordered map from keys to values. // A DB is a persistent ordered map from keys to values.
@ -49,8 +50,7 @@ class LEVELDB_EXPORT DB {
// OK on success. // OK on success.
// Stores nullptr in *dbptr and returns a non-OK status on error. // Stores nullptr in *dbptr and returns a non-OK status on error.
// Caller should delete *dbptr when it is no longer needed. // Caller should delete *dbptr when it is no longer needed.
static Status Open(const Options& options, static Status Open(const Options& options, const std::string& name,
const std::string& name,
DB** dbptr); DB** dbptr);
DB() = default; DB() = default;
@ -63,8 +63,7 @@ class LEVELDB_EXPORT DB {
// Set the database entry for "key" to "value". Returns OK on success, // Set the database entry for "key" to "value". Returns OK on success,
// and a non-OK status on error. // and a non-OK status on error.
// Note: consider setting options.sync = true. // Note: consider setting options.sync = true.
virtual Status Put(const WriteOptions& options, virtual Status Put(const WriteOptions& options, const Slice& key,
const Slice& key,
const Slice& value) = 0; const Slice& value) = 0;
// Remove the database entry (if any) for "key". Returns OK on // Remove the database entry (if any) for "key". Returns OK on
@ -85,8 +84,8 @@ class LEVELDB_EXPORT DB {
// a status for which Status::IsNotFound() returns true. // a status for which Status::IsNotFound() returns true.
// //
// May return some other Status on an error. // May return some other Status on an error.
virtual Status Get(const ReadOptions& options, virtual Status Get(const ReadOptions& options, const Slice& key,
const Slice& key, std::string* value) = 0; std::string* value) = 0;
// Return a heap-allocated iterator over the contents of the database. // Return a heap-allocated iterator over the contents of the database.
// The result of NewIterator() is initially invalid (caller must // The result of NewIterator() is initially invalid (caller must

View File

@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ #define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
#include <string> #include <string>
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/status.h" #include "leveldb/status.h"

View File

@ -15,11 +15,34 @@
#include <stdarg.h> #include <stdarg.h>
#include <stdint.h> #include <stdint.h>
#include <string> #include <string>
#include <vector> #include <vector>
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/status.h" #include "leveldb/status.h"
#if defined(_WIN32)
// The leveldb::Env class below contains a DeleteFile method.
// At the same time, <windows.h>, a fairly popular header
// file for Windows applications, defines a DeleteFile macro.
//
// Without any intervention on our part, the result of this
// unfortunate coincidence is that the name of the
// leveldb::Env::DeleteFile method seen by the compiler depends on
// whether <windows.h> was included before or after the LevelDB
// headers.
//
// To avoid headaches, we undefined DeleteFile (if defined) and
// redefine it at the bottom of this file. This way <windows.h>
// can be included before this file (or not at all) and the
// exported method will always be leveldb::Env::DeleteFile.
#if defined(DeleteFile)
#undef DeleteFile
#define LEVELDB_DELETEFILE_UNDEFINED
#endif // defined(DeleteFile)
#endif // defined(_WIN32)
namespace leveldb { namespace leveldb {
class FileLock; class FileLock;
@ -45,7 +68,7 @@ class LEVELDB_EXPORT Env {
// The result of Default() belongs to leveldb and must never be deleted. // The result of Default() belongs to leveldb and must never be deleted.
static Env* Default(); static Env* Default();
// Create a brand new sequentially-readable file with the specified name. // Create an object that sequentially reads the file with the specified name.
// On success, stores a pointer to the new file in *result and returns OK. // On success, stores a pointer to the new file in *result and returns OK.
// On failure stores nullptr in *result and returns non-OK. If the file does // On failure stores nullptr in *result and returns non-OK. If the file does
// not exist, returns a non-OK status. Implementations should return a // not exist, returns a non-OK status. Implementations should return a
@ -55,7 +78,7 @@ class LEVELDB_EXPORT Env {
virtual Status NewSequentialFile(const std::string& fname, virtual Status NewSequentialFile(const std::string& fname,
SequentialFile** result) = 0; SequentialFile** result) = 0;
// Create a brand new random access read-only file with the // Create an object supporting random-access reads from the file with the
// specified name. On success, stores a pointer to the new file in // specified name. On success, stores a pointer to the new file in
// *result and returns OK. On failure stores nullptr in *result and // *result and returns OK. On failure stores nullptr in *result and
// returns non-OK. If the file does not exist, returns a non-OK // returns non-OK. If the file does not exist, returns a non-OK
@ -143,16 +166,14 @@ class LEVELDB_EXPORT Env {
// added to the same Env may run concurrently in different threads. // added to the same Env may run concurrently in different threads.
// I.e., the caller may not assume that background work items are // I.e., the caller may not assume that background work items are
// serialized. // serialized.
virtual void Schedule( virtual void Schedule(void (*function)(void* arg), void* arg) = 0;
void (*function)(void* arg),
void* arg) = 0;
// Start a new thread, invoking "function(arg)" within the new thread. // Start a new thread, invoking "function(arg)" within the new thread.
// When "function(arg)" returns, the thread will be destroyed. // When "function(arg)" returns, the thread will be destroyed.
virtual void StartThread(void (*function)(void* arg), void* arg) = 0; virtual void StartThread(void (*function)(void* arg), void* arg) = 0;
// *path is set to a temporary directory that can be used for testing. It may // *path is set to a temporary directory that can be used for testing. It may
// or many not have just been created. The directory may or may not differ // or may not have just been created. The directory may or may not differ
// between runs of the same process, but subsequent calls will return the // between runs of the same process, but subsequent calls will return the
// same directory. // same directory.
virtual Status GetTestDirectory(std::string* path) = 0; virtual Status GetTestDirectory(std::string* path) = 0;
@ -343,9 +364,7 @@ class LEVELDB_EXPORT EnvWrapper : public Env {
Status NewLogger(const std::string& fname, Logger** result) override { Status NewLogger(const std::string& fname, Logger** result) override {
return target_->NewLogger(fname, result); return target_->NewLogger(fname, result);
} }
uint64_t NowMicros() override { uint64_t NowMicros() override { return target_->NowMicros(); }
return target_->NowMicros();
}
void SleepForMicroseconds(int micros) override { void SleepForMicroseconds(int micros) override {
target_->SleepForMicroseconds(micros); target_->SleepForMicroseconds(micros);
} }
@ -356,4 +375,13 @@ class LEVELDB_EXPORT EnvWrapper : public Env {
} // namespace leveldb } // namespace leveldb
// Redefine DeleteFile if necessary.
#if defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED)
#if defined(UNICODE)
#define DeleteFile DeleteFileW
#else
#define DeleteFile DeleteFileA
#endif // defined(UNICODE)
#endif // defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED)
#endif // STORAGE_LEVELDB_INCLUDE_ENV_H_ #endif // STORAGE_LEVELDB_INCLUDE_ENV_H_

View File

@ -17,6 +17,7 @@
#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ #define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
#include <string> #include <string>
#include "leveldb/export.h" #include "leveldb/export.h"
namespace leveldb { namespace leveldb {
@ -39,8 +40,8 @@ class LEVELDB_EXPORT FilterPolicy {
// //
// Warning: do not change the initial contents of *dst. Instead, // Warning: do not change the initial contents of *dst. Instead,
// append the newly constructed filter to *dst. // append the newly constructed filter to *dst.
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) virtual void CreateFilter(const Slice* keys, int n,
const = 0; std::string* dst) const = 0;
// "filter" contains the data appended by a preceding call to // "filter" contains the data appended by a preceding call to
// CreateFilter() on this class. This method must return true if // CreateFilter() on this class. This method must return true if

View File

@ -84,16 +84,19 @@ class LEVELDB_EXPORT Iterator {
// Cleanup functions are stored in a single-linked list. // Cleanup functions are stored in a single-linked list.
// The list's head node is inlined in the iterator. // The list's head node is inlined in the iterator.
struct CleanupNode { struct CleanupNode {
// True if the node is not used. Only head nodes might be unused.
bool IsEmpty() const { return function == nullptr; }
// Invokes the cleanup function.
void Run() {
assert(function != nullptr);
(*function)(arg1, arg2);
}
// The head node is used if the function pointer is not null. // The head node is used if the function pointer is not null.
CleanupFunction function; CleanupFunction function;
void* arg1; void* arg1;
void* arg2; void* arg2;
CleanupNode* next; CleanupNode* next;
// True if the node is not used. Only head nodes might be unused.
bool IsEmpty() const { return function == nullptr; }
// Invokes the cleanup function.
void Run() { assert(function != nullptr); (*function)(arg1, arg2); }
}; };
CleanupNode cleanup_head_; CleanupNode cleanup_head_;
}; };

View File

@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ #define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
#include <stddef.h> #include <stddef.h>
#include "leveldb/export.h" #include "leveldb/export.h"
namespace leveldb { namespace leveldb {
@ -30,6 +31,9 @@ enum CompressionType {
// Options to control the behavior of a database (passed to DB::Open) // Options to control the behavior of a database (passed to DB::Open)
struct LEVELDB_EXPORT Options { struct LEVELDB_EXPORT Options {
// Create an Options object with default values for all fields.
Options();
// ------------------- // -------------------
// Parameters that affect behavior // Parameters that affect behavior
@ -42,20 +46,17 @@ struct LEVELDB_EXPORT Options {
const Comparator* comparator; const Comparator* comparator;
// If true, the database will be created if it is missing. // If true, the database will be created if it is missing.
// Default: false bool create_if_missing = false;
bool create_if_missing;
// If true, an error is raised if the database already exists. // If true, an error is raised if the database already exists.
// Default: false bool error_if_exists = false;
bool error_if_exists;
// If true, the implementation will do aggressive checking of the // If true, the implementation will do aggressive checking of the
// data it is processing and will stop early if it detects any // data it is processing and will stop early if it detects any
// errors. This may have unforeseen ramifications: for example, a // errors. This may have unforeseen ramifications: for example, a
// corruption of one DB entry may cause a large number of entries to // corruption of one DB entry may cause a large number of entries to
// become unreadable or for the entire DB to become unopenable. // become unreadable or for the entire DB to become unopenable.
// Default: false bool paranoid_checks = false;
bool paranoid_checks;
// Use the specified object to interact with the environment, // Use the specified object to interact with the environment,
// e.g. to read/write files, schedule background work, etc. // e.g. to read/write files, schedule background work, etc.
@ -65,8 +66,7 @@ struct LEVELDB_EXPORT Options {
// Any internal progress/error information generated by the db will // Any internal progress/error information generated by the db will
// be written to info_log if it is non-null, or to a file stored // be written to info_log if it is non-null, or to a file stored
// in the same directory as the DB contents if info_log is null. // in the same directory as the DB contents if info_log is null.
// Default: nullptr Logger* info_log = nullptr;
Logger* info_log;
// ------------------- // -------------------
// Parameters that affect performance // Parameters that affect performance
@ -79,39 +79,30 @@ struct LEVELDB_EXPORT Options {
// so you may wish to adjust this parameter to control memory usage. // so you may wish to adjust this parameter to control memory usage.
// Also, a larger write buffer will result in a longer recovery time // Also, a larger write buffer will result in a longer recovery time
// the next time the database is opened. // the next time the database is opened.
// size_t write_buffer_size = 4 * 1024 * 1024;
// Default: 4MB
size_t write_buffer_size;
// Number of open files that can be used by the DB. You may need to // Number of open files that can be used by the DB. You may need to
// increase this if your database has a large working set (budget // increase this if your database has a large working set (budget
// one open file per 2MB of working set). // one open file per 2MB of working set).
// int max_open_files = 1000;
// Default: 1000
int max_open_files;
// Control over blocks (user data is stored in a set of blocks, and // Control over blocks (user data is stored in a set of blocks, and
// a block is the unit of reading from disk). // a block is the unit of reading from disk).
// If non-null, use the specified cache for blocks. // If non-null, use the specified cache for blocks.
// If null, leveldb will automatically create and use an 8MB internal cache. // If null, leveldb will automatically create and use an 8MB internal cache.
// Default: nullptr Cache* block_cache = nullptr;
Cache* block_cache;
// Approximate size of user data packed per block. Note that the // Approximate size of user data packed per block. Note that the
// block size specified here corresponds to uncompressed data. The // block size specified here corresponds to uncompressed data. The
// actual size of the unit read from disk may be smaller if // actual size of the unit read from disk may be smaller if
// compression is enabled. This parameter can be changed dynamically. // compression is enabled. This parameter can be changed dynamically.
// size_t block_size = 4 * 1024;
// Default: 4K
size_t block_size;
// Number of keys between restart points for delta encoding of keys. // Number of keys between restart points for delta encoding of keys.
// This parameter can be changed dynamically. Most clients should // This parameter can be changed dynamically. Most clients should
// leave this parameter alone. // leave this parameter alone.
// int block_restart_interval = 16;
// Default: 16
int block_restart_interval;
// Leveldb will write up to this amount of bytes to a file before // Leveldb will write up to this amount of bytes to a file before
// switching to a new one. // switching to a new one.
@ -121,9 +112,7 @@ struct LEVELDB_EXPORT Options {
// compactions and hence longer latency/performance hiccups. // compactions and hence longer latency/performance hiccups.
// Another reason to increase this parameter might be when you are // Another reason to increase this parameter might be when you are
// initially populating a large database. // initially populating a large database.
// size_t max_file_size = 2 * 1024 * 1024;
// Default: 2MB
size_t max_file_size;
// Compress blocks using the specified compression algorithm. This // Compress blocks using the specified compression algorithm. This
// parameter can be changed dynamically. // parameter can be changed dynamically.
@ -139,53 +128,43 @@ struct LEVELDB_EXPORT Options {
// worth switching to kNoCompression. Even if the input data is // worth switching to kNoCompression. Even if the input data is
// incompressible, the kSnappyCompression implementation will // incompressible, the kSnappyCompression implementation will
// efficiently detect that and will switch to uncompressed mode. // efficiently detect that and will switch to uncompressed mode.
CompressionType compression; CompressionType compression = kSnappyCompression;
// EXPERIMENTAL: If true, append to existing MANIFEST and log files // EXPERIMENTAL: If true, append to existing MANIFEST and log files
// when a database is opened. This can significantly speed up open. // when a database is opened. This can significantly speed up open.
// //
// Default: currently false, but may become true later. // Default: currently false, but may become true later.
bool reuse_logs; bool reuse_logs = false;
// If non-null, use the specified filter policy to reduce disk reads. // If non-null, use the specified filter policy to reduce disk reads.
// Many applications will benefit from passing the result of // Many applications will benefit from passing the result of
// NewBloomFilterPolicy() here. // NewBloomFilterPolicy() here.
// const FilterPolicy* filter_policy = nullptr;
// Default: nullptr
const FilterPolicy* filter_policy;
// Create an Options object with default values for all fields.
Options();
}; };
// Options that control read operations // Options that control read operations
struct LEVELDB_EXPORT ReadOptions { struct LEVELDB_EXPORT ReadOptions {
ReadOptions() = default;
// If true, all data read from underlying storage will be // If true, all data read from underlying storage will be
// verified against corresponding checksums. // verified against corresponding checksums.
// Default: false bool verify_checksums = false;
bool verify_checksums;
// Should the data read for this iteration be cached in memory? // Should the data read for this iteration be cached in memory?
// Callers may wish to set this field to false for bulk scans. // Callers may wish to set this field to false for bulk scans.
// Default: true bool fill_cache = true;
bool fill_cache;
// If "snapshot" is non-null, read as of the supplied snapshot // If "snapshot" is non-null, read as of the supplied snapshot
// (which must belong to the DB that is being read and which must // (which must belong to the DB that is being read and which must
// not have been released). If "snapshot" is null, use an implicit // not have been released). If "snapshot" is null, use an implicit
// snapshot of the state at the beginning of this read operation. // snapshot of the state at the beginning of this read operation.
// Default: nullptr const Snapshot* snapshot = nullptr;
const Snapshot* snapshot;
ReadOptions()
: verify_checksums(false),
fill_cache(true),
snapshot(nullptr) {
}
}; };
// Options that control write operations // Options that control write operations
struct LEVELDB_EXPORT WriteOptions { struct LEVELDB_EXPORT WriteOptions {
WriteOptions() = default;
// If true, the write will be flushed from the operating system // If true, the write will be flushed from the operating system
// buffer cache (by calling WritableFile::Sync()) before the write // buffer cache (by calling WritableFile::Sync()) before the write
// is considered complete. If this flag is true, writes will be // is considered complete. If this flag is true, writes will be
@ -200,13 +179,7 @@ struct LEVELDB_EXPORT WriteOptions {
// crash semantics as the "write()" system call. A DB write // crash semantics as the "write()" system call. A DB write
// with sync==true has similar crash semantics to a "write()" // with sync==true has similar crash semantics to a "write()"
// system call followed by "fsync()". // system call followed by "fsync()".
// bool sync = false;
// Default: false
bool sync;
WriteOptions()
: sync(false) {
}
}; };
} // namespace leveldb } // namespace leveldb

View File

@ -18,7 +18,9 @@
#include <assert.h> #include <assert.h>
#include <stddef.h> #include <stddef.h>
#include <string.h> #include <string.h>
#include <string> #include <string>
#include "leveldb/export.h" #include "leveldb/export.h"
namespace leveldb { namespace leveldb {
@ -58,7 +60,10 @@ class LEVELDB_EXPORT Slice {
} }
// Change this slice to refer to an empty array // Change this slice to refer to an empty array
void clear() { data_ = ""; size_ = 0; } void clear() {
data_ = "";
size_ = 0;
}
// Drop the first "n" bytes from this slice. // Drop the first "n" bytes from this slice.
void remove_prefix(size_t n) { void remove_prefix(size_t n) {
@ -78,8 +83,7 @@ class LEVELDB_EXPORT Slice {
// Return true iff "x" is a prefix of "*this" // Return true iff "x" is a prefix of "*this"
bool starts_with(const Slice& x) const { bool starts_with(const Slice& x) const {
return ((size_ >= x.size_) && return ((size_ >= x.size_) && (memcmp(data_, x.data_, x.size_) == 0));
(memcmp(data_, x.data_, x.size_) == 0));
} }
private: private:
@ -92,21 +96,20 @@ inline bool operator==(const Slice& x, const Slice& y) {
(memcmp(x.data(), y.data(), x.size()) == 0)); (memcmp(x.data(), y.data(), x.size()) == 0));
} }
inline bool operator!=(const Slice& x, const Slice& y) { inline bool operator!=(const Slice& x, const Slice& y) { return !(x == y); }
return !(x == y);
}
inline int Slice::compare(const Slice& b) const { inline int Slice::compare(const Slice& b) const {
const size_t min_len = (size_ < b.size_) ? size_ : b.size_; const size_t min_len = (size_ < b.size_) ? size_ : b.size_;
int r = memcmp(data_, b.data_, min_len); int r = memcmp(data_, b.data_, min_len);
if (r == 0) { if (r == 0) {
if (size_ < b.size_) r = -1; if (size_ < b.size_)
else if (size_ > b.size_) r = +1; r = -1;
else if (size_ > b.size_)
r = +1;
} }
return r; return r;
} }
} // namespace leveldb } // namespace leveldb
#endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_ #endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_

View File

@ -15,6 +15,7 @@
#include <algorithm> #include <algorithm>
#include <string> #include <string>
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/slice.h" #include "leveldb/slice.h"
@ -75,13 +76,6 @@ class LEVELDB_EXPORT Status {
std::string ToString() const; std::string ToString() const;
private: private:
// OK status has a null state_. Otherwise, state_ is a new[] array
// of the following form:
// state_[0..3] == length of message
// state_[4] == code
// state_[5..] == message
const char* state_;
enum Code { enum Code {
kOk = 0, kOk = 0,
kNotFound = 1, kNotFound = 1,
@ -97,6 +91,13 @@ class LEVELDB_EXPORT Status {
Status(Code code, const Slice& msg, const Slice& msg2); Status(Code code, const Slice& msg, const Slice& msg2);
static const char* CopyState(const char* s); static const char* CopyState(const char* s);
// OK status has a null state_. Otherwise, state_ is a new[] array
// of the following form:
// state_[0..3] == length of message
// state_[4] == code
// state_[5..] == message
const char* state_;
}; };
inline Status::Status(const Status& rhs) { inline Status::Status(const Status& rhs) {

View File

@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_INCLUDE_TABLE_H_ #define STORAGE_LEVELDB_INCLUDE_TABLE_H_
#include <stdint.h> #include <stdint.h>
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/iterator.h" #include "leveldb/iterator.h"
@ -36,13 +37,11 @@ class LEVELDB_EXPORT Table {
// for the duration of the returned table's lifetime. // for the duration of the returned table's lifetime.
// //
// *file must remain live while this Table is in use. // *file must remain live while this Table is in use.
static Status Open(const Options& options, static Status Open(const Options& options, RandomAccessFile* file,
RandomAccessFile* file, uint64_t file_size, Table** table);
uint64_t file_size,
Table** table);
Table(const Table&) = delete; Table(const Table&) = delete;
void operator=(const Table&) = delete; Table& operator=(const Table&) = delete;
~Table(); ~Table();
@ -60,24 +59,24 @@ class LEVELDB_EXPORT Table {
uint64_t ApproximateOffsetOf(const Slice& key) const; uint64_t ApproximateOffsetOf(const Slice& key) const;
private: private:
friend class TableCache;
struct Rep; struct Rep;
Rep* rep_;
explicit Table(Rep* rep) { rep_ = rep; }
static Iterator* BlockReader(void*, const ReadOptions&, const Slice&); static Iterator* BlockReader(void*, const ReadOptions&, const Slice&);
explicit Table(Rep* rep) : rep_(rep) {}
// Calls (*handle_result)(arg, ...) with the entry found after a call // Calls (*handle_result)(arg, ...) with the entry found after a call
// to Seek(key). May not make such a call if filter policy says // to Seek(key). May not make such a call if filter policy says
// that key is not present. // that key is not present.
friend class TableCache; Status InternalGet(const ReadOptions&, const Slice& key, void* arg,
Status InternalGet( void (*handle_result)(void* arg, const Slice& k,
const ReadOptions&, const Slice& key, const Slice& v));
void* arg,
void (*handle_result)(void* arg, const Slice& k, const Slice& v));
void ReadMeta(const Footer& footer); void ReadMeta(const Footer& footer);
void ReadFilter(const Slice& filter_handle_value); void ReadFilter(const Slice& filter_handle_value);
Rep* const rep_;
}; };
} // namespace leveldb } // namespace leveldb

View File

@ -14,6 +14,7 @@
#define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ #define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
#include <stdint.h> #include <stdint.h>
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/options.h" #include "leveldb/options.h"
#include "leveldb/status.h" #include "leveldb/status.h"
@ -32,7 +33,7 @@ class LEVELDB_EXPORT TableBuilder {
TableBuilder(const Options& options, WritableFile* file); TableBuilder(const Options& options, WritableFile* file);
TableBuilder(const TableBuilder&) = delete; TableBuilder(const TableBuilder&) = delete;
void operator=(const TableBuilder&) = delete; TableBuilder& operator=(const TableBuilder&) = delete;
// REQUIRES: Either Finish() or Abandon() has been called. // REQUIRES: Either Finish() or Abandon() has been called.
~TableBuilder(); ~TableBuilder();

View File

@ -22,6 +22,7 @@
#define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ #define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
#include <string> #include <string>
#include "leveldb/export.h" #include "leveldb/export.h"
#include "leveldb/status.h" #include "leveldb/status.h"
@ -31,6 +32,13 @@ class Slice;
class LEVELDB_EXPORT WriteBatch { class LEVELDB_EXPORT WriteBatch {
public: public:
class LEVELDB_EXPORT Handler {
public:
virtual ~Handler();
virtual void Put(const Slice& key, const Slice& value) = 0;
virtual void Delete(const Slice& key) = 0;
};
WriteBatch(); WriteBatch();
// Intentionally copyable. // Intentionally copyable.
@ -52,15 +60,16 @@ class LEVELDB_EXPORT WriteBatch {
// //
// This number is tied to implementation details, and may change across // This number is tied to implementation details, and may change across
// releases. It is intended for LevelDB usage metrics. // releases. It is intended for LevelDB usage metrics.
size_t ApproximateSize(); size_t ApproximateSize() const;
// Copies the operations in "source" to this batch.
//
// This runs in O(source size) time. However, the constant factor is better
// than calling Iterate() over the source batch with a Handler that replicates
// the operations into this batch.
void Append(const WriteBatch& source);
// Support for iterating over the contents of a batch. // Support for iterating over the contents of a batch.
class Handler {
public:
virtual ~Handler();
virtual void Put(const Slice& key, const Slice& value) = 0;
virtual void Delete(const Slice& key) = 0;
};
Status Iterate(Handler* handler) const; Status Iterate(Handler* handler) const;
private: private:

View File

@ -3,13 +3,14 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
// Test for issue 178: a manual compaction causes deleted data to reappear. // Test for issue 178: a manual compaction causes deleted data to reappear.
#include <cstdlib>
#include <iostream> #include <iostream>
#include <sstream> #include <sstream>
#include <cstdlib>
#include "gtest/gtest.h"
#include "leveldb/db.h" #include "leveldb/db.h"
#include "leveldb/write_batch.h" #include "leveldb/write_batch.h"
#include "util/testharness.h" #include "util/testutil.h"
namespace { namespace {
@ -21,15 +22,11 @@ std::string Key1(int i) {
return buf; return buf;
} }
std::string Key2(int i) { std::string Key2(int i) { return Key1(i) + "_xxx"; }
return Key1(i) + "_xxx";
}
class Issue178 { };
TEST(Issue178, Test) { TEST(Issue178, Test) {
// Get rid of any state from an old run. // Get rid of any state from an old run.
std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test"; std::string dbpath = testing::TempDir() + "leveldb_cbug_test";
DestroyDB(dbpath, leveldb::Options()); DestroyDB(dbpath, leveldb::Options());
// Open database. Disable compression since it affects the creation // Open database. Disable compression since it affects the creation
@ -39,28 +36,28 @@ TEST(Issue178, Test) {
leveldb::Options db_options; leveldb::Options db_options;
db_options.create_if_missing = true; db_options.create_if_missing = true;
db_options.compression = leveldb::kNoCompression; db_options.compression = leveldb::kNoCompression;
ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db)); ASSERT_LEVELDB_OK(leveldb::DB::Open(db_options, dbpath, &db));
// create first key range // create first key range
leveldb::WriteBatch batch; leveldb::WriteBatch batch;
for (size_t i = 0; i < kNumKeys; i++) { for (size_t i = 0; i < kNumKeys; i++) {
batch.Put(Key1(i), "value for range 1 key"); batch.Put(Key1(i), "value for range 1 key");
} }
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch));
// create second key range // create second key range
batch.Clear(); batch.Clear();
for (size_t i = 0; i < kNumKeys; i++) { for (size_t i = 0; i < kNumKeys; i++) {
batch.Put(Key2(i), "value for range 2 key"); batch.Put(Key2(i), "value for range 2 key");
} }
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch));
// delete second key range // delete second key range
batch.Clear(); batch.Clear();
for (size_t i = 0; i < kNumKeys; i++) { for (size_t i = 0; i < kNumKeys; i++) {
batch.Delete(Key2(i)); batch.Delete(Key2(i));
} }
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch));
// compact database // compact database
std::string start_key = Key1(0); std::string start_key = Key1(0);
@ -88,5 +85,6 @@ TEST(Issue178, Test) {
} // anonymous namespace } // anonymous namespace
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -6,35 +6,34 @@
// to forward, the current key can be yielded unexpectedly if a new // to forward, the current key can be yielded unexpectedly if a new
// mutation has been added just before the current key. // mutation has been added just before the current key.
#include "gtest/gtest.h"
#include "leveldb/db.h" #include "leveldb/db.h"
#include "util/testharness.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
class Issue200 { };
TEST(Issue200, Test) { TEST(Issue200, Test) {
// Get rid of any state from an old run. // Get rid of any state from an old run.
std::string dbpath = test::TmpDir() + "/leveldb_issue200_test"; std::string dbpath = testing::TempDir() + "leveldb_issue200_test";
DestroyDB(dbpath, Options()); DestroyDB(dbpath, Options());
DB* db; DB* db;
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
ASSERT_OK(DB::Open(options, dbpath, &db)); ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db));
WriteOptions write_options; WriteOptions write_options;
ASSERT_OK(db->Put(write_options, "1", "b")); ASSERT_LEVELDB_OK(db->Put(write_options, "1", "b"));
ASSERT_OK(db->Put(write_options, "2", "c")); ASSERT_LEVELDB_OK(db->Put(write_options, "2", "c"));
ASSERT_OK(db->Put(write_options, "3", "d")); ASSERT_LEVELDB_OK(db->Put(write_options, "3", "d"));
ASSERT_OK(db->Put(write_options, "4", "e")); ASSERT_LEVELDB_OK(db->Put(write_options, "4", "e"));
ASSERT_OK(db->Put(write_options, "5", "f")); ASSERT_LEVELDB_OK(db->Put(write_options, "5", "f"));
ReadOptions read_options; ReadOptions read_options;
Iterator* iter = db->NewIterator(read_options); Iterator* iter = db->NewIterator(read_options);
// Add an element that should not be reflected in the iterator. // Add an element that should not be reflected in the iterator.
ASSERT_OK(db->Put(write_options, "25", "cd")); ASSERT_LEVELDB_OK(db->Put(write_options, "25", "cd"));
iter->Seek("5"); iter->Seek("5");
ASSERT_EQ(iter->key().ToString(), "5"); ASSERT_EQ(iter->key().ToString(), "5");
@ -55,5 +54,6 @@ TEST(Issue200, Test) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

131
issues/issue320_test.cc Normal file
View File

@ -0,0 +1,131 @@
// Copyright (c) 2019 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <cstdint>
#include <cstdlib>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
#include "gtest/gtest.h"
#include "leveldb/db.h"
#include "leveldb/write_batch.h"
#include "util/testutil.h"
namespace leveldb {
namespace {
// Creates a random number in the range of [0, max).
int GenerateRandomNumber(int max) { return std::rand() % max; }
std::string CreateRandomString(int32_t index) {
static const size_t len = 1024;
char bytes[len];
size_t i = 0;
while (i < 8) {
bytes[i] = 'a' + ((index >> (4 * i)) & 0xf);
++i;
}
while (i < sizeof(bytes)) {
bytes[i] = 'a' + GenerateRandomNumber(26);
++i;
}
return std::string(bytes, sizeof(bytes));
}
} // namespace
TEST(Issue320, Test) {
std::srand(0);
bool delete_before_put = false;
bool keep_snapshots = true;
std::vector<std::unique_ptr<std::pair<std::string, std::string>>> test_map(
10000);
std::vector<Snapshot const*> snapshots(100, nullptr);
DB* db;
Options options;
options.create_if_missing = true;
std::string dbpath = testing::TempDir() + "leveldb_issue320_test";
ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db));
uint32_t target_size = 10000;
uint32_t num_items = 0;
uint32_t count = 0;
std::string key;
std::string value, old_value;
WriteOptions writeOptions;
ReadOptions readOptions;
while (count < 200000) {
if ((++count % 1000) == 0) {
std::cout << "count: " << count << std::endl;
}
int index = GenerateRandomNumber(test_map.size());
WriteBatch batch;
if (test_map[index] == nullptr) {
num_items++;
test_map[index].reset(new std::pair<std::string, std::string>(
CreateRandomString(index), CreateRandomString(index)));
batch.Put(test_map[index]->first, test_map[index]->second);
} else {
ASSERT_LEVELDB_OK(
db->Get(readOptions, test_map[index]->first, &old_value));
if (old_value != test_map[index]->second) {
std::cout << "ERROR incorrect value returned by Get" << std::endl;
std::cout << " count=" << count << std::endl;
std::cout << " old value=" << old_value << std::endl;
std::cout << " test_map[index]->second=" << test_map[index]->second
<< std::endl;
std::cout << " test_map[index]->first=" << test_map[index]->first
<< std::endl;
std::cout << " index=" << index << std::endl;
ASSERT_EQ(old_value, test_map[index]->second);
}
if (num_items >= target_size && GenerateRandomNumber(100) > 30) {
batch.Delete(test_map[index]->first);
test_map[index] = nullptr;
--num_items;
} else {
test_map[index]->second = CreateRandomString(index);
if (delete_before_put) batch.Delete(test_map[index]->first);
batch.Put(test_map[index]->first, test_map[index]->second);
}
}
ASSERT_LEVELDB_OK(db->Write(writeOptions, &batch));
if (keep_snapshots && GenerateRandomNumber(10) == 0) {
int i = GenerateRandomNumber(snapshots.size());
if (snapshots[i] != nullptr) {
db->ReleaseSnapshot(snapshots[i]);
}
snapshots[i] = db->GetSnapshot();
}
}
for (Snapshot const* snapshot : snapshots) {
if (snapshot) {
db->ReleaseSnapshot(snapshot);
}
}
delete db;
DestroyDB(dbpath, options);
}
} // namespace leveldb
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

View File

@ -1,175 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
// AtomicPointer provides storage for a lock-free pointer.
// Platform-dependent implementation of AtomicPointer:
// - If the platform provides a cheap barrier, we use it with raw pointers
// - If <atomic> is present (on newer versions of gcc, it is), we use
// a <atomic>-based AtomicPointer. However we prefer the memory
// barrier based version, because at least on a gcc 4.4 32-bit build
// on linux, we have encountered a buggy <atomic> implementation.
// Also, some <atomic> implementations are much slower than a memory-barrier
// based implementation (~16ns for <atomic> based acquire-load vs. ~1ns for
// a barrier based acquire-load).
// This code is based on atomicops-internals-* in Google's perftools:
// http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase
#ifndef PORT_ATOMIC_POINTER_H_
#define PORT_ATOMIC_POINTER_H_
#include <stdint.h>
#include <atomic>
#ifdef OS_WIN
#include <windows.h>
#endif
#if defined(_M_X64) || defined(__x86_64__)
#define ARCH_CPU_X86_FAMILY 1
#elif defined(_M_IX86) || defined(__i386__) || defined(__i386)
#define ARCH_CPU_X86_FAMILY 1
#elif defined(__ARMEL__)
#define ARCH_CPU_ARM_FAMILY 1
#elif defined(__aarch64__)
#define ARCH_CPU_ARM64_FAMILY 1
#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
#define ARCH_CPU_PPC_FAMILY 1
#elif defined(__mips__)
#define ARCH_CPU_MIPS_FAMILY 1
#endif
namespace leveldb {
namespace port {
// Define MemoryBarrier() if available
// Windows on x86
#if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY)
// windows.h already provides a MemoryBarrier(void) macro
// http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx
#define LEVELDB_HAVE_MEMORY_BARRIER
// Mac OS
#elif defined(__APPLE__)
inline void MemoryBarrier() {
std::atomic_thread_fence(std::memory_order_seq_cst);
}
#define LEVELDB_HAVE_MEMORY_BARRIER
// Gcc on x86
#elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__)
inline void MemoryBarrier() {
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
__asm__ __volatile__("" : : : "memory");
}
#define LEVELDB_HAVE_MEMORY_BARRIER
// Sun Studio
#elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC)
inline void MemoryBarrier() {
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
asm volatile("" : : : "memory");
}
#define LEVELDB_HAVE_MEMORY_BARRIER
// ARM Linux
#elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__)
typedef void (*LinuxKernelMemoryBarrierFunc)(void);
// The Linux ARM kernel provides a highly optimized device-specific memory
// barrier function at a fixed memory address that is mapped in every
// user-level process.
//
// This beats using CPU-specific instructions which are, on single-core
// devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more
// than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking
// shows that the extra function call cost is completely negligible on
// multi-core devices.
//
inline void MemoryBarrier() {
(*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)();
}
#define LEVELDB_HAVE_MEMORY_BARRIER
// ARM64
#elif defined(ARCH_CPU_ARM64_FAMILY)
inline void MemoryBarrier() {
asm volatile("dmb sy" : : : "memory");
}
#define LEVELDB_HAVE_MEMORY_BARRIER
// PPC
#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
inline void MemoryBarrier() {
// TODO for some powerpc expert: is there a cheaper suitable variant?
// Perhaps by having separate barriers for acquire and release ops.
asm volatile("sync" : : : "memory");
}
#define LEVELDB_HAVE_MEMORY_BARRIER
// MIPS
#elif defined(ARCH_CPU_MIPS_FAMILY) && defined(__GNUC__)
inline void MemoryBarrier() {
__asm__ __volatile__("sync" : : : "memory");
}
#define LEVELDB_HAVE_MEMORY_BARRIER
#endif
// AtomicPointer built using platform-specific MemoryBarrier().
#if defined(LEVELDB_HAVE_MEMORY_BARRIER)
class AtomicPointer {
private:
void* rep_;
public:
AtomicPointer() { }
explicit AtomicPointer(void* p) : rep_(p) {}
inline void* NoBarrier_Load() const { return rep_; }
inline void NoBarrier_Store(void* v) { rep_ = v; }
inline void* Acquire_Load() const {
void* result = rep_;
MemoryBarrier();
return result;
}
inline void Release_Store(void* v) {
MemoryBarrier();
rep_ = v;
}
};
// AtomicPointer based on C++11 <atomic>.
#else
class AtomicPointer {
private:
std::atomic<void*> rep_;
public:
AtomicPointer() { }
explicit AtomicPointer(void* v) : rep_(v) { }
inline void* Acquire_Load() const {
return rep_.load(std::memory_order_acquire);
}
inline void Release_Store(void* v) {
rep_.store(v, std::memory_order_release);
}
inline void* NoBarrier_Load() const {
return rep_.load(std::memory_order_relaxed);
}
inline void NoBarrier_Store(void* v) {
rep_.store(v, std::memory_order_relaxed);
}
};
#endif
#undef LEVELDB_HAVE_MEMORY_BARRIER
#undef ARCH_CPU_X86_FAMILY
#undef ARCH_CPU_ARM_FAMILY
#undef ARCH_CPU_ARM64_FAMILY
#undef ARCH_CPU_PPC_FAMILY
} // namespace port
} // namespace leveldb
#endif // PORT_ATOMIC_POINTER_H_

View File

@ -10,7 +10,7 @@
// Include the appropriate platform specific file below. If you are // Include the appropriate platform specific file below. If you are
// porting to a new platform, see "port_example.h" for documentation // porting to a new platform, see "port_example.h" for documentation
// of what the new port_<platform>.h file must provide. // of what the new port_<platform>.h file must provide.
#if defined(LEVELDB_PLATFORM_POSIX) #if defined(LEVELDB_PLATFORM_POSIX) || defined(LEVELDB_PLATFORM_WINDOWS)
#include "port/port_stdcxx.h" #include "port/port_stdcxx.h"
#elif defined(LEVELDB_PLATFORM_CHROMIUM) #elif defined(LEVELDB_PLATFORM_CHROMIUM)
#include "port/port_chromium.h" #include "port/port_chromium.h"

View File

@ -6,9 +6,19 @@
#define STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ #define STORAGE_LEVELDB_PORT_PORT_CONFIG_H_
// Define to 1 if you have a definition for fdatasync() in <unistd.h>. // Define to 1 if you have a definition for fdatasync() in <unistd.h>.
#if !defined(HAVE_FUNC_FDATASYNC) #if !defined(HAVE_FDATASYNC)
#cmakedefine01 HAVE_FUNC_FDATASYNC #cmakedefine01 HAVE_FDATASYNC
#endif // !defined(HAVE_FUNC_FDATASYNC) #endif // !defined(HAVE_FDATASYNC)
// Define to 1 if you have a definition for F_FULLFSYNC in <fcntl.h>.
#if !defined(HAVE_FULLFSYNC)
#cmakedefine01 HAVE_FULLFSYNC
#endif // !defined(HAVE_FULLFSYNC)
// Define to 1 if you have a definition for O_CLOEXEC in <fcntl.h>.
#if !defined(HAVE_O_CLOEXEC)
#cmakedefine01 HAVE_O_CLOEXEC
#endif // !defined(HAVE_O_CLOEXEC)
// Define to 1 if you have Google CRC32C. // Define to 1 if you have Google CRC32C.
#if !defined(HAVE_CRC32C) #if !defined(HAVE_CRC32C)

View File

@ -62,45 +62,6 @@ class CondVar {
void SignallAll(); void SignallAll();
}; };
// Thread-safe initialization.
// Used as follows:
// static port::OnceType init_control = LEVELDB_ONCE_INIT;
// static void Initializer() { ... do something ...; }
// ...
// port::InitOnce(&init_control, &Initializer);
typedef intptr_t OnceType;
#define LEVELDB_ONCE_INIT 0
void InitOnce(port::OnceType*, void (*initializer)());
// A type that holds a pointer that can be read or written atomically
// (i.e., without word-tearing.)
class AtomicPointer {
private:
intptr_t rep_;
public:
// Initialize to arbitrary value
AtomicPointer();
// Initialize to hold v
explicit AtomicPointer(void* v) : rep_(v) { }
// Read and return the stored pointer with the guarantee that no
// later memory access (read or write) by this thread can be
// reordered ahead of this read.
void* Acquire_Load() const;
// Set v as the stored pointer with the guarantee that no earlier
// memory access (read or write) by this thread can be reordered
// after this store.
void Release_Store(void* v);
// Read the stored pointer with no ordering guarantees.
void* NoBarrier_Load() const;
// Set va as the stored pointer with no ordering guarantees.
void NoBarrier_Store(void* v);
};
// ------------------ Compression ------------------- // ------------------ Compression -------------------
// Store the snappy compression of "input[0,input_length-1]" in *output. // Store the snappy compression of "input[0,input_length-1]" in *output.

View File

@ -29,13 +29,13 @@
#include <snappy.h> #include <snappy.h>
#endif // HAVE_SNAPPY #endif // HAVE_SNAPPY
#include <stddef.h>
#include <stdint.h>
#include <cassert> #include <cassert>
#include <condition_variable> // NOLINT #include <condition_variable> // NOLINT
#include <cstddef>
#include <cstdint>
#include <mutex> // NOLINT #include <mutex> // NOLINT
#include <string> #include <string>
#include "port/atomic_pointer.h"
#include "port/thread_annotations.h" #include "port/thread_annotations.h"
namespace leveldb { namespace leveldb {
@ -79,27 +79,25 @@ class CondVar {
} }
void Signal() { cv_.notify_one(); } void Signal() { cv_.notify_one(); }
void SignalAll() { cv_.notify_all(); } void SignalAll() { cv_.notify_all(); }
private: private:
std::condition_variable cv_; std::condition_variable cv_;
Mutex* const mu_; Mutex* const mu_;
}; };
using OnceType = std::once_flag;
#define LEVELDB_ONCE_INIT {}
// Thinly wraps std::call_once.
inline void InitOnce(OnceType* once, void (*initializer)()) {
std::call_once(*once, *initializer);
}
inline bool Snappy_Compress(const char* input, size_t length, inline bool Snappy_Compress(const char* input, size_t length,
::std::string* output) { std::string* output) {
#if HAVE_SNAPPY #if HAVE_SNAPPY
output->resize(snappy::MaxCompressedLength(length)); output->resize(snappy::MaxCompressedLength(length));
size_t outlen; size_t outlen;
snappy::RawCompress(input, length, &(*output)[0], &outlen); snappy::RawCompress(input, length, &(*output)[0], &outlen);
output->resize(outlen); output->resize(outlen);
return true; return true;
#else
// Silence compiler warnings about unused arguments.
(void)input;
(void)length;
(void)output;
#endif // HAVE_SNAPPY #endif // HAVE_SNAPPY
return false; return false;
@ -110,6 +108,10 @@ inline bool Snappy_GetUncompressedLength(const char* input, size_t length,
#if HAVE_SNAPPY #if HAVE_SNAPPY
return snappy::GetUncompressedLength(input, length, result); return snappy::GetUncompressedLength(input, length, result);
#else #else
// Silence compiler warnings about unused arguments.
(void)input;
(void)length;
(void)result;
return false; return false;
#endif // HAVE_SNAPPY #endif // HAVE_SNAPPY
} }
@ -118,11 +120,18 @@ inline bool Snappy_Uncompress(const char* input, size_t length, char* output) {
#if HAVE_SNAPPY #if HAVE_SNAPPY
return snappy::RawUncompress(input, length, output); return snappy::RawUncompress(input, length, output);
#else #else
// Silence compiler warnings about unused arguments.
(void)input;
(void)length;
(void)output;
return false; return false;
#endif // HAVE_SNAPPY #endif // HAVE_SNAPPY
} }
inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) {
// Silence compiler warnings about unused arguments.
(void)func;
(void)arg;
return false; return false;
} }
@ -130,6 +139,10 @@ inline uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) {
#if HAVE_CRC32C #if HAVE_CRC32C
return ::crc32c::Extend(crc, reinterpret_cast<const uint8_t*>(buf), size); return ::crc32c::Extend(crc, reinterpret_cast<const uint8_t*>(buf), size);
#else #else
// Silence compiler warnings about unused arguments.
(void)crc;
(void)buf;
(void)size;
return 0; return 0;
#endif // HAVE_CRC32C #endif // HAVE_CRC32C
} }

View File

@ -54,18 +54,15 @@
#endif #endif
#ifndef LOCK_RETURNED #ifndef LOCK_RETURNED
#define LOCK_RETURNED(x) \ #define LOCK_RETURNED(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))
THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))
#endif #endif
#ifndef LOCKABLE #ifndef LOCKABLE
#define LOCKABLE \ #define LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(lockable)
THREAD_ANNOTATION_ATTRIBUTE__(lockable)
#endif #endif
#ifndef SCOPED_LOCKABLE #ifndef SCOPED_LOCKABLE
#define SCOPED_LOCKABLE \ #define SCOPED_LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
#endif #endif
#ifndef EXCLUSIVE_LOCK_FUNCTION #ifndef EXCLUSIVE_LOCK_FUNCTION

View File

@ -1,24 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
// MSVC didn't ship with this file until the 2010 version.
#ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_
#define STORAGE_LEVELDB_PORT_WIN_STDINT_H_
#if !defined(_MSC_VER)
#error This file should only be included when compiling with MSVC.
#endif
// Define C99 equivalent types.
typedef signed char int8_t;
typedef signed short int16_t;
typedef signed int int32_t;
typedef signed long long int64_t;
typedef unsigned char uint8_t;
typedef unsigned short uint16_t;
typedef unsigned int uint32_t;
typedef unsigned long long uint64_t;
#endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_

View File

@ -6,8 +6,10 @@
#include "table/block.h" #include "table/block.h"
#include <vector>
#include <algorithm> #include <algorithm>
#include <cstdint>
#include <vector>
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
#include "table/format.h" #include "table/format.h"
#include "util/coding.h" #include "util/coding.h"
@ -51,13 +53,12 @@ Block::~Block() {
// If any errors are detected, returns nullptr. Otherwise, returns a // If any errors are detected, returns nullptr. Otherwise, returns a
// pointer to the key delta (just past the three decoded values). // pointer to the key delta (just past the three decoded values).
static inline const char* DecodeEntry(const char* p, const char* limit, static inline const char* DecodeEntry(const char* p, const char* limit,
uint32_t* shared, uint32_t* shared, uint32_t* non_shared,
uint32_t* non_shared,
uint32_t* value_length) { uint32_t* value_length) {
if (limit - p < 3) return nullptr; if (limit - p < 3) return nullptr;
*shared = reinterpret_cast<const unsigned char*>(p)[0]; *shared = reinterpret_cast<const uint8_t*>(p)[0];
*non_shared = reinterpret_cast<const unsigned char*>(p)[1]; *non_shared = reinterpret_cast<const uint8_t*>(p)[1];
*value_length = reinterpret_cast<const unsigned char*>(p)[2]; *value_length = reinterpret_cast<const uint8_t*>(p)[2];
if ((*shared | *non_shared | *value_length) < 128) { if ((*shared | *non_shared | *value_length) < 128) {
// Fast path: all three values are encoded in one byte each // Fast path: all three values are encoded in one byte each
p += 3; p += 3;
@ -112,9 +113,7 @@ class Block::Iter : public Iterator {
} }
public: public:
Iter(const Comparator* comparator, Iter(const Comparator* comparator, const char* data, uint32_t restarts,
const char* data,
uint32_t restarts,
uint32_t num_restarts) uint32_t num_restarts)
: comparator_(comparator), : comparator_(comparator),
data_(data), data_(data),
@ -125,23 +124,23 @@ class Block::Iter : public Iterator {
assert(num_restarts_ > 0); assert(num_restarts_ > 0);
} }
virtual bool Valid() const { return current_ < restarts_; } bool Valid() const override { return current_ < restarts_; }
virtual Status status() const { return status_; } Status status() const override { return status_; }
virtual Slice key() const { Slice key() const override {
assert(Valid()); assert(Valid());
return key_; return key_;
} }
virtual Slice value() const { Slice value() const override {
assert(Valid()); assert(Valid());
return value_; return value_;
} }
virtual void Next() { void Next() override {
assert(Valid()); assert(Valid());
ParseNextKey(); ParseNextKey();
} }
virtual void Prev() { void Prev() override {
assert(Valid()); assert(Valid());
// Scan backwards to a restart point before current_ // Scan backwards to a restart point before current_
@ -162,7 +161,7 @@ class Block::Iter : public Iterator {
} while (ParseNextKey() && NextEntryOffset() < original); } while (ParseNextKey() && NextEntryOffset() < original);
} }
virtual void Seek(const Slice& target) { void Seek(const Slice& target) override {
// Binary search in restart array to find the last restart point // Binary search in restart array to find the last restart point
// with a key < target // with a key < target
uint32_t left = 0; uint32_t left = 0;
@ -171,9 +170,9 @@ class Block::Iter : public Iterator {
uint32_t mid = (left + right + 1) / 2; uint32_t mid = (left + right + 1) / 2;
uint32_t region_offset = GetRestartPoint(mid); uint32_t region_offset = GetRestartPoint(mid);
uint32_t shared, non_shared, value_length; uint32_t shared, non_shared, value_length;
const char* key_ptr = DecodeEntry(data_ + region_offset, const char* key_ptr =
data_ + restarts_, DecodeEntry(data_ + region_offset, data_ + restarts_, &shared,
&shared, &non_shared, &value_length); &non_shared, &value_length);
if (key_ptr == nullptr || (shared != 0)) { if (key_ptr == nullptr || (shared != 0)) {
CorruptionError(); CorruptionError();
return; return;
@ -202,12 +201,12 @@ class Block::Iter : public Iterator {
} }
} }
virtual void SeekToFirst() { void SeekToFirst() override {
SeekToRestartPoint(0); SeekToRestartPoint(0);
ParseNextKey(); ParseNextKey();
} }
virtual void SeekToLast() { void SeekToLast() override {
SeekToRestartPoint(num_restarts_ - 1); SeekToRestartPoint(num_restarts_ - 1);
while (ParseNextKey() && NextEntryOffset() < restarts_) { while (ParseNextKey() && NextEntryOffset() < restarts_) {
// Keep skipping // Keep skipping
@ -253,7 +252,7 @@ class Block::Iter : public Iterator {
} }
}; };
Iterator* Block::NewIterator(const Comparator* cmp) { Iterator* Block::NewIterator(const Comparator* comparator) {
if (size_ < sizeof(uint32_t)) { if (size_ < sizeof(uint32_t)) {
return NewErrorIterator(Status::Corruption("bad block contents")); return NewErrorIterator(Status::Corruption("bad block contents"));
} }
@ -261,7 +260,7 @@ Iterator* Block::NewIterator(const Comparator* cmp) {
if (num_restarts == 0) { if (num_restarts == 0) {
return NewEmptyIterator(); return NewEmptyIterator();
} else { } else {
return new Iter(cmp, data_, restart_offset_, num_restarts); return new Iter(comparator, data_, restart_offset_, num_restarts);
} }
} }

View File

@ -7,6 +7,7 @@
#include <stddef.h> #include <stddef.h>
#include <stdint.h> #include <stdint.h>
#include "leveldb/iterator.h" #include "leveldb/iterator.h"
namespace leveldb { namespace leveldb {
@ -19,24 +20,23 @@ class Block {
// Initialize the block with the specified contents. // Initialize the block with the specified contents.
explicit Block(const BlockContents& contents); explicit Block(const BlockContents& contents);
Block(const Block&) = delete;
Block& operator=(const Block&) = delete;
~Block(); ~Block();
size_t size() const { return size_; } size_t size() const { return size_; }
Iterator* NewIterator(const Comparator* comparator); Iterator* NewIterator(const Comparator* comparator);
private: private:
class Iter;
uint32_t NumRestarts() const; uint32_t NumRestarts() const;
const char* data_; const char* data_;
size_t size_; size_t size_;
uint32_t restart_offset_; // Offset in data_ of restart array uint32_t restart_offset_; // Offset in data_ of restart array
bool owned_; // Block owns data_[] bool owned_; // Block owns data_[]
// No copying allowed
Block(const Block&);
void operator=(const Block&);
class Iter;
}; };
} // namespace leveldb } // namespace leveldb

View File

@ -28,19 +28,18 @@
#include "table/block_builder.h" #include "table/block_builder.h"
#include <algorithm>
#include <assert.h> #include <assert.h>
#include <algorithm>
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
#include "leveldb/table_builder.h" #include "leveldb/options.h"
#include "util/coding.h" #include "util/coding.h"
namespace leveldb { namespace leveldb {
BlockBuilder::BlockBuilder(const Options* options) BlockBuilder::BlockBuilder(const Options* options)
: options_(options), : options_(options), restarts_(), counter_(0), finished_(false) {
restarts_(),
counter_(0),
finished_(false) {
assert(options->block_restart_interval >= 1); assert(options->block_restart_interval >= 1);
restarts_.push_back(0); // First restart point is at offset 0 restarts_.push_back(0); // First restart point is at offset 0
} }

View File

@ -5,9 +5,10 @@
#ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ #ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_
#define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ #define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_
#include <stdint.h>
#include <vector> #include <vector>
#include <stdint.h>
#include "leveldb/slice.h" #include "leveldb/slice.h"
namespace leveldb { namespace leveldb {
@ -18,6 +19,9 @@ class BlockBuilder {
public: public:
explicit BlockBuilder(const Options* options); explicit BlockBuilder(const Options* options);
BlockBuilder(const BlockBuilder&) = delete;
BlockBuilder& operator=(const BlockBuilder&) = delete;
// Reset the contents as if the BlockBuilder was just constructed. // Reset the contents as if the BlockBuilder was just constructed.
void Reset(); void Reset();
@ -35,9 +39,7 @@ class BlockBuilder {
size_t CurrentSizeEstimate() const; size_t CurrentSizeEstimate() const;
// Return true iff no entries have been added since the last Reset() // Return true iff no entries have been added since the last Reset()
bool empty() const { bool empty() const { return buffer_.empty(); }
return buffer_.empty();
}
private: private:
const Options* options_; const Options* options_;
@ -46,10 +48,6 @@ class BlockBuilder {
int counter_; // Number of entries emitted since restart int counter_; // Number of entries emitted since restart
bool finished_; // Has Finish() been called? bool finished_; // Has Finish() been called?
std::string last_key_; std::string last_key_;
// No copying allowed
BlockBuilder(const BlockBuilder&);
void operator=(const BlockBuilder&);
}; };
} // namespace leveldb } // namespace leveldb

View File

@ -16,8 +16,7 @@ static const size_t kFilterBaseLg = 11;
static const size_t kFilterBase = 1 << kFilterBaseLg; static const size_t kFilterBase = 1 << kFilterBaseLg;
FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy) FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy)
: policy_(policy) { : policy_(policy) {}
}
void FilterBlockBuilder::StartBlock(uint64_t block_offset) { void FilterBlockBuilder::StartBlock(uint64_t block_offset) {
uint64_t filter_index = (block_offset / kFilterBase); uint64_t filter_index = (block_offset / kFilterBase);
@ -77,11 +76,7 @@ void FilterBlockBuilder::GenerateFilter() {
FilterBlockReader::FilterBlockReader(const FilterPolicy* policy, FilterBlockReader::FilterBlockReader(const FilterPolicy* policy,
const Slice& contents) const Slice& contents)
: policy_(policy), : policy_(policy), data_(nullptr), offset_(nullptr), num_(0), base_lg_(0) {
data_(nullptr),
offset_(nullptr),
num_(0),
base_lg_(0) {
size_t n = contents.size(); size_t n = contents.size();
if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array
base_lg_ = contents[n - 1]; base_lg_ = contents[n - 1];
@ -108,4 +103,4 @@ bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) {
return true; // Errors are treated as potential matches return true; // Errors are treated as potential matches
} }
} } // namespace leveldb

View File

@ -11,8 +11,10 @@
#include <stddef.h> #include <stddef.h>
#include <stdint.h> #include <stdint.h>
#include <string> #include <string>
#include <vector> #include <vector>
#include "leveldb/slice.h" #include "leveldb/slice.h"
#include "util/hash.h" #include "util/hash.h"
@ -30,6 +32,9 @@ class FilterBlockBuilder {
public: public:
explicit FilterBlockBuilder(const FilterPolicy*); explicit FilterBlockBuilder(const FilterPolicy*);
FilterBlockBuilder(const FilterBlockBuilder&) = delete;
FilterBlockBuilder& operator=(const FilterBlockBuilder&) = delete;
void StartBlock(uint64_t block_offset); void StartBlock(uint64_t block_offset);
void AddKey(const Slice& key); void AddKey(const Slice& key);
Slice Finish(); Slice Finish();
@ -43,10 +48,6 @@ class FilterBlockBuilder {
std::string result_; // Filter data computed so far std::string result_; // Filter data computed so far
std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument
std::vector<uint32_t> filter_offsets_; std::vector<uint32_t> filter_offsets_;
// No copying allowed
FilterBlockBuilder(const FilterBlockBuilder&);
void operator=(const FilterBlockBuilder&);
}; };
class FilterBlockReader { class FilterBlockReader {
@ -63,6 +64,6 @@ class FilterBlockReader {
size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file) size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file)
}; };
} } // namespace leveldb
#endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ #endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_

View File

@ -4,11 +4,11 @@
#include "table/filter_block.h" #include "table/filter_block.h"
#include "gtest/gtest.h"
#include "leveldb/filter_policy.h" #include "leveldb/filter_policy.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/hash.h" #include "util/hash.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
@ -16,18 +16,16 @@ namespace leveldb {
// For testing: emit an array with one hash value per key // For testing: emit an array with one hash value per key
class TestHashFilter : public FilterPolicy { class TestHashFilter : public FilterPolicy {
public: public:
virtual const char* Name() const { const char* Name() const override { return "TestHashFilter"; }
return "TestHashFilter";
}
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); uint32_t h = Hash(keys[i].data(), keys[i].size(), 1);
PutFixed32(dst, h); PutFixed32(dst, h);
} }
} }
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
uint32_t h = Hash(key.data(), key.size(), 1); uint32_t h = Hash(key.data(), key.size(), 1);
for (size_t i = 0; i + 4 <= filter.size(); i += 4) { for (size_t i = 0; i + 4 <= filter.size(); i += 4) {
if (h == DecodeFixed32(filter.data() + i)) { if (h == DecodeFixed32(filter.data() + i)) {
@ -38,12 +36,12 @@ class TestHashFilter : public FilterPolicy {
} }
}; };
class FilterBlockTest { class FilterBlockTest : public testing::Test {
public: public:
TestHashFilter policy_; TestHashFilter policy_;
}; };
TEST(FilterBlockTest, EmptyBuilder) { TEST_F(FilterBlockTest, EmptyBuilder) {
FilterBlockBuilder builder(&policy_); FilterBlockBuilder builder(&policy_);
Slice block = builder.Finish(); Slice block = builder.Finish();
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block));
@ -52,7 +50,7 @@ TEST(FilterBlockTest, EmptyBuilder) {
ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); ASSERT_TRUE(reader.KeyMayMatch(100000, "foo"));
} }
TEST(FilterBlockTest, SingleChunk) { TEST_F(FilterBlockTest, SingleChunk) {
FilterBlockBuilder builder(&policy_); FilterBlockBuilder builder(&policy_);
builder.StartBlock(100); builder.StartBlock(100);
builder.AddKey("foo"); builder.AddKey("foo");
@ -73,7 +71,7 @@ TEST(FilterBlockTest, SingleChunk) {
ASSERT_TRUE(!reader.KeyMayMatch(100, "other")); ASSERT_TRUE(!reader.KeyMayMatch(100, "other"));
} }
TEST(FilterBlockTest, MultiChunk) { TEST_F(FilterBlockTest, MultiChunk) {
FilterBlockBuilder builder(&policy_); FilterBlockBuilder builder(&policy_);
// First filter // First filter
@ -124,5 +122,6 @@ TEST(FilterBlockTest, MultiChunk) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

View File

@ -21,8 +21,7 @@ void BlockHandle::EncodeTo(std::string* dst) const {
} }
Status BlockHandle::DecodeFrom(Slice* input) { Status BlockHandle::DecodeFrom(Slice* input) {
if (GetVarint64(input, &offset_) && if (GetVarint64(input, &offset_) && GetVarint64(input, &size_)) {
GetVarint64(input, &size_)) {
return Status::OK(); return Status::OK();
} else { } else {
return Status::Corruption("bad block handle"); return Status::Corruption("bad block handle");
@ -62,10 +61,8 @@ Status Footer::DecodeFrom(Slice* input) {
return result; return result;
} }
Status ReadBlock(RandomAccessFile* file, Status ReadBlock(RandomAccessFile* file, const ReadOptions& options,
const ReadOptions& options, const BlockHandle& handle, BlockContents* result) {
const BlockHandle& handle,
BlockContents* result) {
result->data = Slice(); result->data = Slice();
result->cachable = false; result->cachable = false;
result->heap_allocated = false; result->heap_allocated = false;

View File

@ -5,8 +5,10 @@
#ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_ #ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_
#define STORAGE_LEVELDB_TABLE_FORMAT_H_ #define STORAGE_LEVELDB_TABLE_FORMAT_H_
#include <string>
#include <stdint.h> #include <stdint.h>
#include <string>
#include "leveldb/slice.h" #include "leveldb/slice.h"
#include "leveldb/status.h" #include "leveldb/status.h"
#include "leveldb/table_builder.h" #include "leveldb/table_builder.h"
@ -21,6 +23,9 @@ struct ReadOptions;
// block or a meta block. // block or a meta block.
class BlockHandle { class BlockHandle {
public: public:
// Maximum encoding length of a BlockHandle
enum { kMaxEncodedLength = 10 + 10 };
BlockHandle(); BlockHandle();
// The offset of the block in the file. // The offset of the block in the file.
@ -34,9 +39,6 @@ class BlockHandle {
void EncodeTo(std::string* dst) const; void EncodeTo(std::string* dst) const;
Status DecodeFrom(Slice* input); Status DecodeFrom(Slice* input);
// Maximum encoding length of a BlockHandle
enum { kMaxEncodedLength = 10 + 10 };
private: private:
uint64_t offset_; uint64_t offset_;
uint64_t size_; uint64_t size_;
@ -46,30 +48,24 @@ class BlockHandle {
// end of every table file. // end of every table file.
class Footer { class Footer {
public: public:
Footer() { } // Encoded length of a Footer. Note that the serialization of a
// Footer will always occupy exactly this many bytes. It consists
// of two block handles and a magic number.
enum { kEncodedLength = 2 * BlockHandle::kMaxEncodedLength + 8 };
Footer() = default;
// The block handle for the metaindex block of the table // The block handle for the metaindex block of the table
const BlockHandle& metaindex_handle() const { return metaindex_handle_; } const BlockHandle& metaindex_handle() const { return metaindex_handle_; }
void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; } void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; }
// The block handle for the index block of the table // The block handle for the index block of the table
const BlockHandle& index_handle() const { const BlockHandle& index_handle() const { return index_handle_; }
return index_handle_; void set_index_handle(const BlockHandle& h) { index_handle_ = h; }
}
void set_index_handle(const BlockHandle& h) {
index_handle_ = h;
}
void EncodeTo(std::string* dst) const; void EncodeTo(std::string* dst) const;
Status DecodeFrom(Slice* input); Status DecodeFrom(Slice* input);
// Encoded length of a Footer. Note that the serialization of a
// Footer will always occupy exactly this many bytes. It consists
// of two block handles and a magic number.
enum {
kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8
};
private: private:
BlockHandle metaindex_handle_; BlockHandle metaindex_handle_;
BlockHandle index_handle_; BlockHandle index_handle_;
@ -91,17 +87,13 @@ struct BlockContents {
// Read the block identified by "handle" from "file". On failure // Read the block identified by "handle" from "file". On failure
// return non-OK. On success fill *result and return OK. // return non-OK. On success fill *result and return OK.
Status ReadBlock(RandomAccessFile* file, Status ReadBlock(RandomAccessFile* file, const ReadOptions& options,
const ReadOptions& options, const BlockHandle& handle, BlockContents* result);
const BlockHandle& handle,
BlockContents* result);
// Implementation details follow. Clients should ignore, // Implementation details follow. Clients should ignore,
inline BlockHandle::BlockHandle() inline BlockHandle::BlockHandle()
: offset_(~static_cast<uint64_t>(0)), : offset_(~static_cast<uint64_t>(0)), size_(~static_cast<uint64_t>(0)) {}
size_(~static_cast<uint64_t>(0)) {
}
} // namespace leveldb } // namespace leveldb

View File

@ -51,8 +51,14 @@ class EmptyIterator : public Iterator {
void SeekToLast() override {} void SeekToLast() override {}
void Next() override { assert(false); } void Next() override { assert(false); }
void Prev() override { assert(false); } void Prev() override { assert(false); }
Slice key() const override { assert(false); return Slice(); } Slice key() const override {
Slice value() const override { assert(false); return Slice(); } assert(false);
return Slice();
}
Slice value() const override {
assert(false);
return Slice();
}
Status status() const override { return status_; } Status status() const override { return status_; }
private: private:
@ -61,9 +67,7 @@ class EmptyIterator : public Iterator {
} // anonymous namespace } // anonymous namespace
Iterator* NewEmptyIterator() { Iterator* NewEmptyIterator() { return new EmptyIterator(Status::OK()); }
return new EmptyIterator(Status::OK());
}
Iterator* NewErrorIterator(const Status& status) { Iterator* NewErrorIterator(const Status& status) {
return new EmptyIterator(status); return new EmptyIterator(status);

View File

@ -17,9 +17,7 @@ namespace leveldb {
class IteratorWrapper { class IteratorWrapper {
public: public:
IteratorWrapper() : iter_(nullptr), valid_(false) {} IteratorWrapper() : iter_(nullptr), valid_(false) {}
explicit IteratorWrapper(Iterator* iter): iter_(nullptr) { explicit IteratorWrapper(Iterator* iter) : iter_(nullptr) { Set(iter); }
Set(iter);
}
~IteratorWrapper() { delete iter_; } ~IteratorWrapper() { delete iter_; }
Iterator* iter() const { return iter_; } Iterator* iter() const { return iter_; }
@ -35,18 +33,46 @@ class IteratorWrapper {
} }
} }
// Iterator interface methods // Iterator interface methods
bool Valid() const { return valid_; } bool Valid() const { return valid_; }
Slice key() const { assert(Valid()); return key_; } Slice key() const {
Slice value() const { assert(Valid()); return iter_->value(); } assert(Valid());
return key_;
}
Slice value() const {
assert(Valid());
return iter_->value();
}
// Methods below require iter() != nullptr // Methods below require iter() != nullptr
Status status() const { assert(iter_); return iter_->status(); } Status status() const {
void Next() { assert(iter_); iter_->Next(); Update(); } assert(iter_);
void Prev() { assert(iter_); iter_->Prev(); Update(); } return iter_->status();
void Seek(const Slice& k) { assert(iter_); iter_->Seek(k); Update(); } }
void SeekToFirst() { assert(iter_); iter_->SeekToFirst(); Update(); } void Next() {
void SeekToLast() { assert(iter_); iter_->SeekToLast(); Update(); } assert(iter_);
iter_->Next();
Update();
}
void Prev() {
assert(iter_);
iter_->Prev();
Update();
}
void Seek(const Slice& k) {
assert(iter_);
iter_->Seek(k);
Update();
}
void SeekToFirst() {
assert(iter_);
iter_->SeekToFirst();
Update();
}
void SeekToLast() {
assert(iter_);
iter_->SeekToLast();
Update();
}
private: private:
void Update() { void Update() {

View File

@ -24,15 +24,11 @@ class MergingIterator : public Iterator {
} }
} }
virtual ~MergingIterator() { ~MergingIterator() override { delete[] children_; }
delete[] children_;
}
virtual bool Valid() const { bool Valid() const override { return (current_ != nullptr); }
return (current_ != nullptr);
}
virtual void SeekToFirst() { void SeekToFirst() override {
for (int i = 0; i < n_; i++) { for (int i = 0; i < n_; i++) {
children_[i].SeekToFirst(); children_[i].SeekToFirst();
} }
@ -40,7 +36,7 @@ class MergingIterator : public Iterator {
direction_ = kForward; direction_ = kForward;
} }
virtual void SeekToLast() { void SeekToLast() override {
for (int i = 0; i < n_; i++) { for (int i = 0; i < n_; i++) {
children_[i].SeekToLast(); children_[i].SeekToLast();
} }
@ -48,7 +44,7 @@ class MergingIterator : public Iterator {
direction_ = kReverse; direction_ = kReverse;
} }
virtual void Seek(const Slice& target) { void Seek(const Slice& target) override {
for (int i = 0; i < n_; i++) { for (int i = 0; i < n_; i++) {
children_[i].Seek(target); children_[i].Seek(target);
} }
@ -56,7 +52,7 @@ class MergingIterator : public Iterator {
direction_ = kForward; direction_ = kForward;
} }
virtual void Next() { void Next() override {
assert(Valid()); assert(Valid());
// Ensure that all children are positioned after key(). // Ensure that all children are positioned after key().
@ -82,7 +78,7 @@ class MergingIterator : public Iterator {
FindSmallest(); FindSmallest();
} }
virtual void Prev() { void Prev() override {
assert(Valid()); assert(Valid());
// Ensure that all children are positioned before key(). // Ensure that all children are positioned before key().
@ -111,17 +107,17 @@ class MergingIterator : public Iterator {
FindLargest(); FindLargest();
} }
virtual Slice key() const { Slice key() const override {
assert(Valid()); assert(Valid());
return current_->key(); return current_->key();
} }
virtual Slice value() const { Slice value() const override {
assert(Valid()); assert(Valid());
return current_->value(); return current_->value();
} }
virtual Status status() const { Status status() const override {
Status status; Status status;
for (int i = 0; i < n_; i++) { for (int i = 0; i < n_; i++) {
status = children_[i].status(); status = children_[i].status();
@ -133,6 +129,9 @@ class MergingIterator : public Iterator {
} }
private: private:
// Which direction is the iterator moving?
enum Direction { kForward, kReverse };
void FindSmallest(); void FindSmallest();
void FindLargest(); void FindLargest();
@ -143,12 +142,6 @@ class MergingIterator : public Iterator {
IteratorWrapper* children_; IteratorWrapper* children_;
int n_; int n_;
IteratorWrapper* current_; IteratorWrapper* current_;
// Which direction is the iterator moving?
enum Direction {
kForward,
kReverse
};
Direction direction_; Direction direction_;
}; };
@ -183,14 +176,15 @@ void MergingIterator::FindLargest() {
} }
} // namespace } // namespace
Iterator* NewMergingIterator(const Comparator* cmp, Iterator** list, int n) { Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children,
int n) {
assert(n >= 0); assert(n >= 0);
if (n == 0) { if (n == 0) {
return NewEmptyIterator(); return NewEmptyIterator();
} else if (n == 1) { } else if (n == 1) {
return list[0]; return children[0];
} else { } else {
return new MergingIterator(cmp, list, n); return new MergingIterator(comparator, children, n);
} }
} }

View File

@ -18,8 +18,8 @@ class Iterator;
// key is present in K child iterators, it will be yielded K times. // key is present in K child iterators, it will be yielded K times.
// //
// REQUIRES: n >= 0 // REQUIRES: n >= 0
Iterator* NewMergingIterator( Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children,
const Comparator* comparator, Iterator** children, int n); int n);
} // namespace leveldb } // namespace leveldb

View File

@ -35,10 +35,8 @@ struct Table::Rep {
Block* index_block; Block* index_block;
}; };
Status Table::Open(const Options& options, Status Table::Open(const Options& options, RandomAccessFile* file,
RandomAccessFile* file, uint64_t size, Table** table) {
uint64_t size,
Table** table) {
*table = nullptr; *table = nullptr;
if (size < Footer::kEncodedLength) { if (size < Footer::kEncodedLength) {
return Status::Corruption("file is too short to be an sstable"); return Status::Corruption("file is too short to be an sstable");
@ -135,9 +133,7 @@ void Table::ReadFilter(const Slice& filter_handle_value) {
rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data); rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data);
} }
Table::~Table() { Table::~Table() { delete rep_; }
delete rep_;
}
static void DeleteBlock(void* arg, void* ignored) { static void DeleteBlock(void* arg, void* ignored) {
delete reinterpret_cast<Block*>(arg); delete reinterpret_cast<Block*>(arg);
@ -156,8 +152,7 @@ static void ReleaseBlock(void* arg, void* h) {
// Convert an index iterator value (i.e., an encoded BlockHandle) // Convert an index iterator value (i.e., an encoded BlockHandle)
// into an iterator over the contents of the corresponding block. // into an iterator over the contents of the corresponding block.
Iterator* Table::BlockReader(void* arg, Iterator* Table::BlockReader(void* arg, const ReadOptions& options,
const ReadOptions& options,
const Slice& index_value) { const Slice& index_value) {
Table* table = reinterpret_cast<Table*>(arg); Table* table = reinterpret_cast<Table*>(arg);
Cache* block_cache = table->rep_->options.block_cache; Cache* block_cache = table->rep_->options.block_cache;
@ -185,8 +180,8 @@ Iterator* Table::BlockReader(void* arg,
if (s.ok()) { if (s.ok()) {
block = new Block(contents); block = new Block(contents);
if (contents.cachable && options.fill_cache) { if (contents.cachable && options.fill_cache) {
cache_handle = block_cache->Insert( cache_handle = block_cache->Insert(key, block, block->size(),
key, block, block->size(), &DeleteCachedBlock); &DeleteCachedBlock);
} }
} }
} }
@ -218,9 +213,9 @@ Iterator* Table::NewIterator(const ReadOptions& options) const {
&Table::BlockReader, const_cast<Table*>(this), options); &Table::BlockReader, const_cast<Table*>(this), options);
} }
Status Table::InternalGet(const ReadOptions& options, const Slice& k, Status Table::InternalGet(const ReadOptions& options, const Slice& k, void* arg,
void* arg, void (*handle_result)(void*, const Slice&,
void (*saver)(void*, const Slice&, const Slice&)) { const Slice&)) {
Status s; Status s;
Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator); Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator);
iiter->Seek(k); iiter->Seek(k);
@ -228,15 +223,14 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k,
Slice handle_value = iiter->value(); Slice handle_value = iiter->value();
FilterBlockReader* filter = rep_->filter; FilterBlockReader* filter = rep_->filter;
BlockHandle handle; BlockHandle handle;
if (filter != nullptr && if (filter != nullptr && handle.DecodeFrom(&handle_value).ok() &&
handle.DecodeFrom(&handle_value).ok() &&
!filter->KeyMayMatch(handle.offset(), k)) { !filter->KeyMayMatch(handle.offset(), k)) {
// Not found // Not found
} else { } else {
Iterator* block_iter = BlockReader(this, options, iiter->value()); Iterator* block_iter = BlockReader(this, options, iiter->value());
block_iter->Seek(k); block_iter->Seek(k);
if (block_iter->Valid()) { if (block_iter->Valid()) {
(*saver)(arg, block_iter->key(), block_iter->value()); (*handle_result)(arg, block_iter->key(), block_iter->value());
} }
s = block_iter->status(); s = block_iter->status();
delete block_iter; delete block_iter;
@ -249,7 +243,6 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k,
return s; return s;
} }
uint64_t Table::ApproximateOffsetOf(const Slice& key) const { uint64_t Table::ApproximateOffsetOf(const Slice& key) const {
Iterator* index_iter = Iterator* index_iter =
rep_->index_block->NewIterator(rep_->options.comparator); rep_->index_block->NewIterator(rep_->options.comparator);

View File

@ -5,6 +5,7 @@
#include "leveldb/table_builder.h" #include "leveldb/table_builder.h"
#include <assert.h> #include <assert.h>
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/filter_policy.h" #include "leveldb/filter_policy.h"
@ -18,6 +19,22 @@
namespace leveldb { namespace leveldb {
struct TableBuilder::Rep { struct TableBuilder::Rep {
Rep(const Options& opt, WritableFile* f)
: options(opt),
index_block_options(opt),
file(f),
offset(0),
data_block(&options),
index_block(&index_block_options),
num_entries(0),
closed(false),
filter_block(opt.filter_policy == nullptr
? nullptr
: new FilterBlockBuilder(opt.filter_policy)),
pending_index_entry(false) {
index_block_options.block_restart_interval = 1;
}
Options options; Options options;
Options index_block_options; Options index_block_options;
WritableFile* file; WritableFile* file;
@ -43,21 +60,6 @@ struct TableBuilder::Rep {
BlockHandle pending_handle; // Handle to add to index block BlockHandle pending_handle; // Handle to add to index block
std::string compressed_output; std::string compressed_output;
Rep(const Options& opt, WritableFile* f)
: options(opt),
index_block_options(opt),
file(f),
offset(0),
data_block(&options),
index_block(&index_block_options),
num_entries(0),
closed(false),
filter_block(opt.filter_policy == nullptr ? nullptr
: new FilterBlockBuilder(opt.filter_policy)),
pending_index_entry(false) {
index_block_options.block_restart_interval = 1;
}
}; };
TableBuilder::TableBuilder(const Options& options, WritableFile* file) TableBuilder::TableBuilder(const Options& options, WritableFile* file)
@ -173,8 +175,7 @@ void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) {
} }
void TableBuilder::WriteRawBlock(const Slice& block_contents, void TableBuilder::WriteRawBlock(const Slice& block_contents,
CompressionType type, CompressionType type, BlockHandle* handle) {
BlockHandle* handle) {
Rep* r = rep_; Rep* r = rep_;
handle->set_offset(r->offset); handle->set_offset(r->offset);
handle->set_size(block_contents.size()); handle->set_size(block_contents.size());
@ -192,9 +193,7 @@ void TableBuilder::WriteRawBlock(const Slice& block_contents,
} }
} }
Status TableBuilder::status() const { Status TableBuilder::status() const { return rep_->status; }
return rep_->status;
}
Status TableBuilder::Finish() { Status TableBuilder::Finish() {
Rep* r = rep_; Rep* r = rep_;
@ -259,12 +258,8 @@ void TableBuilder::Abandon() {
r->closed = true; r->closed = true;
} }
uint64_t TableBuilder::NumEntries() const { uint64_t TableBuilder::NumEntries() const { return rep_->num_entries; }
return rep_->num_entries;
}
uint64_t TableBuilder::FileSize() const { uint64_t TableBuilder::FileSize() const { return rep_->offset; }
return rep_->offset;
}
} // namespace leveldb } // namespace leveldb

View File

@ -6,6 +6,8 @@
#include <map> #include <map>
#include <string> #include <string>
#include "gtest/gtest.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/memtable.h" #include "db/memtable.h"
#include "db/write_batch_internal.h" #include "db/write_batch_internal.h"
@ -17,7 +19,6 @@
#include "table/block_builder.h" #include "table/block_builder.h"
#include "table/format.h" #include "table/format.h"
#include "util/random.h" #include "util/random.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
namespace leveldb { namespace leveldb {
@ -27,8 +28,8 @@ namespace leveldb {
static std::string Reverse(const Slice& key) { static std::string Reverse(const Slice& key) {
std::string str(key.ToString()); std::string str(key.ToString());
std::string rev(""); std::string rev("");
for (std::string::reverse_iterator rit = str.rbegin(); for (std::string::reverse_iterator rit = str.rbegin(); rit != str.rend();
rit != str.rend(); ++rit) { ++rit) {
rev.push_back(*rit); rev.push_back(*rit);
} }
return rev; return rev;
@ -37,24 +38,23 @@ static std::string Reverse(const Slice& key) {
namespace { namespace {
class ReverseKeyComparator : public Comparator { class ReverseKeyComparator : public Comparator {
public: public:
virtual const char* Name() const { const char* Name() const override {
return "leveldb.ReverseBytewiseComparator"; return "leveldb.ReverseBytewiseComparator";
} }
virtual int Compare(const Slice& a, const Slice& b) const { int Compare(const Slice& a, const Slice& b) const override {
return BytewiseComparator()->Compare(Reverse(a), Reverse(b)); return BytewiseComparator()->Compare(Reverse(a), Reverse(b));
} }
virtual void FindShortestSeparator( void FindShortestSeparator(std::string* start,
std::string* start, const Slice& limit) const override {
const Slice& limit) const {
std::string s = Reverse(*start); std::string s = Reverse(*start);
std::string l = Reverse(limit); std::string l = Reverse(limit);
BytewiseComparator()->FindShortestSeparator(&s, l); BytewiseComparator()->FindShortestSeparator(&s, l);
*start = Reverse(s); *start = Reverse(s);
} }
virtual void FindShortSuccessor(std::string* key) const { void FindShortSuccessor(std::string* key) const override {
std::string s = Reverse(*key); std::string s = Reverse(*key);
BytewiseComparator()->FindShortSuccessor(&s); BytewiseComparator()->FindShortSuccessor(&s);
*key = Reverse(s); *key = Reverse(s);
@ -89,15 +89,15 @@ struct STLLessThan {
class StringSink : public WritableFile { class StringSink : public WritableFile {
public: public:
~StringSink() { } ~StringSink() override = default;
const std::string& contents() const { return contents_; } const std::string& contents() const { return contents_; }
virtual Status Close() { return Status::OK(); } Status Close() override { return Status::OK(); }
virtual Status Flush() { return Status::OK(); } Status Flush() override { return Status::OK(); }
virtual Status Sync() { return Status::OK(); } Status Sync() override { return Status::OK(); }
virtual Status Append(const Slice& data) { Status Append(const Slice& data) override {
contents_.append(data.data(), data.size()); contents_.append(data.data(), data.size());
return Status::OK(); return Status::OK();
} }
@ -106,20 +106,18 @@ class StringSink: public WritableFile {
std::string contents_; std::string contents_;
}; };
class StringSource : public RandomAccessFile { class StringSource : public RandomAccessFile {
public: public:
StringSource(const Slice& contents) StringSource(const Slice& contents)
: contents_(contents.data(), contents.size()) { : contents_(contents.data(), contents.size()) {}
}
virtual ~StringSource() { } ~StringSource() override = default;
uint64_t Size() const { return contents_.size(); } uint64_t Size() const { return contents_.size(); }
virtual Status Read(uint64_t offset, size_t n, Slice* result, Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const { char* scratch) const override {
if (offset > contents_.size()) { if (offset >= contents_.size()) {
return Status::InvalidArgument("invalid Read offset"); return Status::InvalidArgument("invalid Read offset");
} }
if (offset + n > contents_.size()) { if (offset + n > contents_.size()) {
@ -141,7 +139,7 @@ typedef std::map<std::string, std::string, STLLessThan> KVMap;
class Constructor { class Constructor {
public: public:
explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) {} explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) {}
virtual ~Constructor() { } virtual ~Constructor() = default;
void Add(const std::string& key, const Slice& value) { void Add(const std::string& key, const Slice& value) {
data_[key] = value.ToString(); data_[key] = value.ToString();
@ -150,15 +148,12 @@ class Constructor {
// Finish constructing the data structure with all the keys that have // Finish constructing the data structure with all the keys that have
// been added so far. Returns the keys in sorted order in "*keys" // been added so far. Returns the keys in sorted order in "*keys"
// and stores the key/value pairs in "*kvmap" // and stores the key/value pairs in "*kvmap"
void Finish(const Options& options, void Finish(const Options& options, std::vector<std::string>* keys,
std::vector<std::string>* keys,
KVMap* kvmap) { KVMap* kvmap) {
*kvmap = data_; *kvmap = data_;
keys->clear(); keys->clear();
for (KVMap::const_iterator it = data_.begin(); for (const auto& kvp : data_) {
it != data_.end(); keys->push_back(kvp.first);
++it) {
keys->push_back(it->first);
} }
data_.clear(); data_.clear();
Status s = FinishImpl(options, *kvmap); Status s = FinishImpl(options, *kvmap);
@ -170,7 +165,7 @@ class Constructor {
virtual Iterator* NewIterator() const = 0; virtual Iterator* NewIterator() const = 0;
virtual const KVMap& data() { return data_; } const KVMap& data() const { return data_; }
virtual DB* db() const { return nullptr; } // Overridden in DBConstructor virtual DB* db() const { return nullptr; } // Overridden in DBConstructor
@ -181,21 +176,15 @@ class Constructor {
class BlockConstructor : public Constructor { class BlockConstructor : public Constructor {
public: public:
explicit BlockConstructor(const Comparator* cmp) explicit BlockConstructor(const Comparator* cmp)
: Constructor(cmp), : Constructor(cmp), comparator_(cmp), block_(nullptr) {}
comparator_(cmp), ~BlockConstructor() override { delete block_; }
block_(nullptr) { } Status FinishImpl(const Options& options, const KVMap& data) override {
~BlockConstructor() {
delete block_;
}
virtual Status FinishImpl(const Options& options, const KVMap& data) {
delete block_; delete block_;
block_ = nullptr; block_ = nullptr;
BlockBuilder builder(&options); BlockBuilder builder(&options);
for (KVMap::const_iterator it = data.begin(); for (const auto& kvp : data) {
it != data.end(); builder.Add(kvp.first, kvp.second);
++it) {
builder.Add(it->first, it->second);
} }
// Open the block // Open the block
data_ = builder.Finish().ToString(); data_ = builder.Finish().ToString();
@ -206,12 +195,12 @@ class BlockConstructor: public Constructor {
block_ = new Block(contents); block_ = new Block(contents);
return Status::OK(); return Status::OK();
} }
virtual Iterator* NewIterator() const { Iterator* NewIterator() const override {
return block_->NewIterator(comparator_); return block_->NewIterator(comparator_);
} }
private: private:
const Comparator* comparator_; const Comparator* const comparator_;
std::string data_; std::string data_;
Block* block_; Block* block_;
@ -221,27 +210,21 @@ class BlockConstructor: public Constructor {
class TableConstructor : public Constructor { class TableConstructor : public Constructor {
public: public:
TableConstructor(const Comparator* cmp) TableConstructor(const Comparator* cmp)
: Constructor(cmp), : Constructor(cmp), source_(nullptr), table_(nullptr) {}
source_(nullptr), table_(nullptr) { ~TableConstructor() override { Reset(); }
} Status FinishImpl(const Options& options, const KVMap& data) override {
~TableConstructor() {
Reset();
}
virtual Status FinishImpl(const Options& options, const KVMap& data) {
Reset(); Reset();
StringSink sink; StringSink sink;
TableBuilder builder(options, &sink); TableBuilder builder(options, &sink);
for (KVMap::const_iterator it = data.begin(); for (const auto& kvp : data) {
it != data.end(); builder.Add(kvp.first, kvp.second);
++it) { EXPECT_LEVELDB_OK(builder.status());
builder.Add(it->first, it->second);
ASSERT_TRUE(builder.status().ok());
} }
Status s = builder.Finish(); Status s = builder.Finish();
ASSERT_TRUE(s.ok()) << s.ToString(); EXPECT_LEVELDB_OK(s);
ASSERT_EQ(sink.contents().size(), builder.FileSize()); EXPECT_EQ(sink.contents().size(), builder.FileSize());
// Open the table // Open the table
source_ = new StringSource(sink.contents()); source_ = new StringSource(sink.contents());
@ -250,7 +233,7 @@ class TableConstructor: public Constructor {
return Table::Open(table_options, source_, sink.contents().size(), &table_); return Table::Open(table_options, source_, sink.contents().size(), &table_);
} }
virtual Iterator* NewIterator() const { Iterator* NewIterator() const override {
return table_->NewIterator(ReadOptions()); return table_->NewIterator(ReadOptions());
} }
@ -276,20 +259,25 @@ class TableConstructor: public Constructor {
class KeyConvertingIterator : public Iterator { class KeyConvertingIterator : public Iterator {
public: public:
explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) {} explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) {}
virtual ~KeyConvertingIterator() { delete iter_; }
virtual bool Valid() const { return iter_->Valid(); } KeyConvertingIterator(const KeyConvertingIterator&) = delete;
virtual void Seek(const Slice& target) { KeyConvertingIterator& operator=(const KeyConvertingIterator&) = delete;
~KeyConvertingIterator() override { delete iter_; }
bool Valid() const override { return iter_->Valid(); }
void Seek(const Slice& target) override {
ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue); ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue);
std::string encoded; std::string encoded;
AppendInternalKey(&encoded, ikey); AppendInternalKey(&encoded, ikey);
iter_->Seek(encoded); iter_->Seek(encoded);
} }
virtual void SeekToFirst() { iter_->SeekToFirst(); } void SeekToFirst() override { iter_->SeekToFirst(); }
virtual void SeekToLast() { iter_->SeekToLast(); } void SeekToLast() override { iter_->SeekToLast(); }
virtual void Next() { iter_->Next(); } void Next() override { iter_->Next(); }
virtual void Prev() { iter_->Prev(); } void Prev() override { iter_->Prev(); }
virtual Slice key() const { Slice key() const override {
assert(Valid()); assert(Valid());
ParsedInternalKey key; ParsedInternalKey key;
if (!ParseInternalKey(iter_->key(), &key)) { if (!ParseInternalKey(iter_->key(), &key)) {
@ -299,86 +287,72 @@ class KeyConvertingIterator: public Iterator {
return key.user_key; return key.user_key;
} }
virtual Slice value() const { return iter_->value(); } Slice value() const override { return iter_->value(); }
virtual Status status() const { Status status() const override {
return status_.ok() ? iter_->status() : status_; return status_.ok() ? iter_->status() : status_;
} }
private: private:
mutable Status status_; mutable Status status_;
Iterator* iter_; Iterator* iter_;
// No copying allowed
KeyConvertingIterator(const KeyConvertingIterator&);
void operator=(const KeyConvertingIterator&);
}; };
class MemTableConstructor : public Constructor { class MemTableConstructor : public Constructor {
public: public:
explicit MemTableConstructor(const Comparator* cmp) explicit MemTableConstructor(const Comparator* cmp)
: Constructor(cmp), : Constructor(cmp), internal_comparator_(cmp) {
internal_comparator_(cmp) {
memtable_ = new MemTable(internal_comparator_); memtable_ = new MemTable(internal_comparator_);
memtable_->Ref(); memtable_->Ref();
} }
~MemTableConstructor() { ~MemTableConstructor() override { memtable_->Unref(); }
memtable_->Unref(); Status FinishImpl(const Options& options, const KVMap& data) override {
}
virtual Status FinishImpl(const Options& options, const KVMap& data) {
memtable_->Unref(); memtable_->Unref();
memtable_ = new MemTable(internal_comparator_); memtable_ = new MemTable(internal_comparator_);
memtable_->Ref(); memtable_->Ref();
int seq = 1; int seq = 1;
for (KVMap::const_iterator it = data.begin(); for (const auto& kvp : data) {
it != data.end(); memtable_->Add(seq, kTypeValue, kvp.first, kvp.second);
++it) {
memtable_->Add(seq, kTypeValue, it->first, it->second);
seq++; seq++;
} }
return Status::OK(); return Status::OK();
} }
virtual Iterator* NewIterator() const { Iterator* NewIterator() const override {
return new KeyConvertingIterator(memtable_->NewIterator()); return new KeyConvertingIterator(memtable_->NewIterator());
} }
private: private:
InternalKeyComparator internal_comparator_; const InternalKeyComparator internal_comparator_;
MemTable* memtable_; MemTable* memtable_;
}; };
class DBConstructor : public Constructor { class DBConstructor : public Constructor {
public: public:
explicit DBConstructor(const Comparator* cmp) explicit DBConstructor(const Comparator* cmp)
: Constructor(cmp), : Constructor(cmp), comparator_(cmp) {
comparator_(cmp) {
db_ = nullptr; db_ = nullptr;
NewDB(); NewDB();
} }
~DBConstructor() { ~DBConstructor() override { delete db_; }
delete db_; Status FinishImpl(const Options& options, const KVMap& data) override {
}
virtual Status FinishImpl(const Options& options, const KVMap& data) {
delete db_; delete db_;
db_ = nullptr; db_ = nullptr;
NewDB(); NewDB();
for (KVMap::const_iterator it = data.begin(); for (const auto& kvp : data) {
it != data.end();
++it) {
WriteBatch batch; WriteBatch batch;
batch.Put(it->first, it->second); batch.Put(kvp.first, kvp.second);
ASSERT_TRUE(db_->Write(WriteOptions(), &batch).ok()); EXPECT_TRUE(db_->Write(WriteOptions(), &batch).ok());
} }
return Status::OK(); return Status::OK();
} }
virtual Iterator* NewIterator() const { Iterator* NewIterator() const override {
return db_->NewIterator(ReadOptions()); return db_->NewIterator(ReadOptions());
} }
virtual DB* db() const { return db_; } DB* db() const override { return db_; }
private: private:
void NewDB() { void NewDB() {
std::string name = test::TmpDir() + "/table_testdb"; std::string name = testing::TempDir() + "table_testdb";
Options options; Options options;
options.comparator = comparator_; options.comparator = comparator_;
@ -392,16 +366,11 @@ class DBConstructor: public Constructor {
ASSERT_TRUE(status.ok()) << status.ToString(); ASSERT_TRUE(status.ok()) << status.ToString();
} }
const Comparator* comparator_; const Comparator* const comparator_;
DB* db_; DB* db_;
}; };
enum TestType { enum TestType { TABLE_TEST, BLOCK_TEST, MEMTABLE_TEST, DB_TEST };
TABLE_TEST,
BLOCK_TEST,
MEMTABLE_TEST,
DB_TEST
};
struct TestArgs { struct TestArgs {
TestType type; TestType type;
@ -434,7 +403,7 @@ static const TestArgs kTestArgList[] = {
}; };
static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]); static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]);
class Harness { class Harness : public testing::Test {
public: public:
Harness() : constructor_(nullptr) {} Harness() : constructor_(nullptr) {}
@ -466,9 +435,7 @@ class Harness {
} }
} }
~Harness() { ~Harness() { delete constructor_; }
delete constructor_;
}
void Add(const std::string& key, const std::string& value) { void Add(const std::string& key, const std::string& value) {
constructor_->Add(key, value); constructor_->Add(key, value);
@ -490,8 +457,7 @@ class Harness {
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
iter->SeekToFirst(); iter->SeekToFirst();
for (KVMap::const_iterator model_iter = data.begin(); for (KVMap::const_iterator model_iter = data.begin();
model_iter != data.end(); model_iter != data.end(); ++model_iter) {
++model_iter) {
ASSERT_EQ(ToString(data, model_iter), ToString(iter)); ASSERT_EQ(ToString(data, model_iter), ToString(iter));
iter->Next(); iter->Next();
} }
@ -505,8 +471,7 @@ class Harness {
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
iter->SeekToLast(); iter->SeekToLast();
for (KVMap::const_reverse_iterator model_iter = data.rbegin(); for (KVMap::const_reverse_iterator model_iter = data.rbegin();
model_iter != data.rend(); model_iter != data.rend(); ++model_iter) {
++model_iter) {
ASSERT_EQ(ToString(data, model_iter), ToString(iter)); ASSERT_EQ(ToString(data, model_iter), ToString(iter));
iter->Prev(); iter->Prev();
} }
@ -514,8 +479,7 @@ class Harness {
delete iter; delete iter;
} }
void TestRandomAccess(Random* rnd, void TestRandomAccess(Random* rnd, const std::vector<std::string>& keys,
const std::vector<std::string>& keys,
const KVMap& data) { const KVMap& data) {
static const bool kVerbose = false; static const bool kVerbose = false;
Iterator* iter = constructor_->NewIterator(); Iterator* iter = constructor_->NewIterator();
@ -546,8 +510,8 @@ class Harness {
case 2: { case 2: {
std::string key = PickRandomKey(rnd, keys); std::string key = PickRandomKey(rnd, keys);
model_iter = data.lower_bound(key); model_iter = data.lower_bound(key);
if (kVerbose) fprintf(stderr, "Seek '%s'\n", if (kVerbose)
EscapeString(key).c_str()); fprintf(stderr, "Seek '%s'\n", EscapeString(key).c_str());
iter->Seek(Slice(key)); iter->Seek(Slice(key));
ASSERT_EQ(ToString(data, model_iter), ToString(iter)); ASSERT_EQ(ToString(data, model_iter), ToString(iter));
break; break;
@ -621,7 +585,7 @@ class Harness {
break; break;
case 1: { case 1: {
// Attempt to return something smaller than an existing key // Attempt to return something smaller than an existing key
if (result.size() > 0 && result[result.size()-1] > '\0') { if (!result.empty() && result[result.size() - 1] > '\0') {
result[result.size() - 1]--; result[result.size() - 1]--;
} }
break; break;
@ -645,7 +609,7 @@ class Harness {
}; };
// Test empty table/block. // Test empty table/block.
TEST(Harness, Empty) { TEST_F(Harness, Empty) {
for (int i = 0; i < kNumTestArgs; i++) { for (int i = 0; i < kNumTestArgs; i++) {
Init(kTestArgList[i]); Init(kTestArgList[i]);
Random rnd(test::RandomSeed() + 1); Random rnd(test::RandomSeed() + 1);
@ -656,7 +620,7 @@ TEST(Harness, Empty) {
// Special test for a block with no restart entries. The C++ leveldb // Special test for a block with no restart entries. The C++ leveldb
// code never generates such blocks, but the Java version of leveldb // code never generates such blocks, but the Java version of leveldb
// seems to. // seems to.
TEST(Harness, ZeroRestartPointsInBlock) { TEST_F(Harness, ZeroRestartPointsInBlock) {
char data[sizeof(uint32_t)]; char data[sizeof(uint32_t)];
memset(data, 0, sizeof(data)); memset(data, 0, sizeof(data));
BlockContents contents; BlockContents contents;
@ -675,7 +639,7 @@ TEST(Harness, ZeroRestartPointsInBlock) {
} }
// Test the empty key // Test the empty key
TEST(Harness, SimpleEmptyKey) { TEST_F(Harness, SimpleEmptyKey) {
for (int i = 0; i < kNumTestArgs; i++) { for (int i = 0; i < kNumTestArgs; i++) {
Init(kTestArgList[i]); Init(kTestArgList[i]);
Random rnd(test::RandomSeed() + 1); Random rnd(test::RandomSeed() + 1);
@ -684,7 +648,7 @@ TEST(Harness, SimpleEmptyKey) {
} }
} }
TEST(Harness, SimpleSingle) { TEST_F(Harness, SimpleSingle) {
for (int i = 0; i < kNumTestArgs; i++) { for (int i = 0; i < kNumTestArgs; i++) {
Init(kTestArgList[i]); Init(kTestArgList[i]);
Random rnd(test::RandomSeed() + 2); Random rnd(test::RandomSeed() + 2);
@ -693,7 +657,7 @@ TEST(Harness, SimpleSingle) {
} }
} }
TEST(Harness, SimpleMulti) { TEST_F(Harness, SimpleMulti) {
for (int i = 0; i < kNumTestArgs; i++) { for (int i = 0; i < kNumTestArgs; i++) {
Init(kTestArgList[i]); Init(kTestArgList[i]);
Random rnd(test::RandomSeed() + 3); Random rnd(test::RandomSeed() + 3);
@ -704,7 +668,7 @@ TEST(Harness, SimpleMulti) {
} }
} }
TEST(Harness, SimpleSpecialKey) { TEST_F(Harness, SimpleSpecialKey) {
for (int i = 0; i < kNumTestArgs; i++) { for (int i = 0; i < kNumTestArgs; i++) {
Init(kTestArgList[i]); Init(kTestArgList[i]);
Random rnd(test::RandomSeed() + 4); Random rnd(test::RandomSeed() + 4);
@ -713,15 +677,15 @@ TEST(Harness, SimpleSpecialKey) {
} }
} }
TEST(Harness, Randomized) { TEST_F(Harness, Randomized) {
for (int i = 0; i < kNumTestArgs; i++) { for (int i = 0; i < kNumTestArgs; i++) {
Init(kTestArgList[i]); Init(kTestArgList[i]);
Random rnd(test::RandomSeed() + 5); Random rnd(test::RandomSeed() + 5);
for (int num_entries = 0; num_entries < 2000; for (int num_entries = 0; num_entries < 2000;
num_entries += (num_entries < 50 ? 1 : 200)) { num_entries += (num_entries < 50 ? 1 : 200)) {
if ((num_entries % 10) == 0) { if ((num_entries % 10) == 0) {
fprintf(stderr, "case %d of %d: num_entries = %d\n", fprintf(stderr, "case %d of %d: num_entries = %d\n", (i + 1),
(i + 1), int(kNumTestArgs), num_entries); int(kNumTestArgs), num_entries);
} }
for (int e = 0; e < num_entries; e++) { for (int e = 0; e < num_entries; e++) {
std::string v; std::string v;
@ -733,7 +697,7 @@ TEST(Harness, Randomized) {
} }
} }
TEST(Harness, RandomizedLongDB) { TEST_F(Harness, RandomizedLongDB) {
Random rnd(test::RandomSeed()); Random rnd(test::RandomSeed());
TestArgs args = {DB_TEST, false, 16}; TestArgs args = {DB_TEST, false, 16};
Init(args); Init(args);
@ -757,8 +721,6 @@ TEST(Harness, RandomizedLongDB) {
ASSERT_GT(files, 0); ASSERT_GT(files, 0);
} }
class MemTableTest { };
TEST(MemTableTest, Simple) { TEST(MemTableTest, Simple) {
InternalKeyComparator cmp(BytewiseComparator()); InternalKeyComparator cmp(BytewiseComparator());
MemTable* memtable = new MemTable(cmp); MemTable* memtable = new MemTable(cmp);
@ -774,8 +736,7 @@ TEST(MemTableTest, Simple) {
Iterator* iter = memtable->NewIterator(); Iterator* iter = memtable->NewIterator();
iter->SeekToFirst(); iter->SeekToFirst();
while (iter->Valid()) { while (iter->Valid()) {
fprintf(stderr, "key: '%s' -> '%s'\n", fprintf(stderr, "key: '%s' -> '%s'\n", iter->key().ToString().c_str(),
iter->key().ToString().c_str(),
iter->value().ToString().c_str()); iter->value().ToString().c_str());
iter->Next(); iter->Next();
} }
@ -788,15 +749,12 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
bool result = (val >= low) && (val <= high); bool result = (val >= low) && (val <= high);
if (!result) { if (!result) {
fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
(unsigned long long)(val), (unsigned long long)(val), (unsigned long long)(low),
(unsigned long long)(low),
(unsigned long long)(high)); (unsigned long long)(high));
} }
return result; return result;
} }
class TableTest { };
TEST(TableTest, ApproximateOffsetOfPlain) { TEST(TableTest, ApproximateOffsetOfPlain) {
TableConstructor c(BytewiseComparator()); TableConstructor c(BytewiseComparator());
c.Add("k01", "hello"); c.Add("k01", "hello");
@ -824,7 +782,6 @@ TEST(TableTest, ApproximateOffsetOfPlain) {
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000)); ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000));
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000)); ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000));
ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000)); ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000));
} }
static bool SnappyCompressionSupported() { static bool SnappyCompressionSupported() {
@ -872,5 +829,6 @@ TEST(TableTest, ApproximateOffsetOfCompressed) {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
return leveldb::test::RunAllTests(); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
} }

Some files were not shown because too many files have changed in this diff Show More