From e1f98fbfc015b5abd39427cdd5952b53ed4500c4 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 27 Nov 2016 16:19:26 +0100 Subject: [PATCH 1/5] :construction: splitting tests into several binaries --- .gitignore | 2 ++ .travis.yml | 60 ++++----------------------------------------------- Makefile | 14 ++++++------ test/Makefile | 30 +++++++++++++++++++++++--- 4 files changed, 41 insertions(+), 65 deletions(-) diff --git a/.gitignore b/.gitignore index bd8c86992..42a18e82b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ benchmarks/files/numbers/*.json .idea cmake-build-debug + +test/test-* diff --git a/.travis.yml b/.travis.yml index 8a9b10324..430ef75c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: sources: ['ubuntu-toolchain-r-test'] packages: [g++-4.9, valgrind] after_success: - - valgrind --error-exitcode=1 --leak-check=full test/json_unit + - make check make check TEST_PREFIX="valgrind --error-exitcode=1 --leak-check=full " TEST_PATTERN="" # cppcheck @@ -70,8 +70,7 @@ matrix: - gem install coveralls-lcov after_success: - make clean - - CXXFLAGS="--coverage -g -O0" CPPFLAGS="-DNDEBUG" make - - test/json_unit "*" + - CXXFLAGS="--coverage -g -O0" CPPFLAGS="-DNDEBUG" make check - coveralls --build-root test --exclude src/catch.hpp --exclude src/unit-algorithms.cpp --exclude src/unit-allocator.cpp --exclude src/unit-capacity.cpp --exclude src/unit-class_const_iterator.cpp --exclude src/unit-class_iterator.cpp --exclude src/unit-class_lexer.cpp --exclude src/unit-class_parser.cpp --exclude src/unit-comparison.cpp --exclude src/unit-concepts.cpp --exclude src/unit-constructor1.cpp --exclude src/unit-constructor2.cpp --exclude src/unit-convenience.cpp --exclude src/unit-conversions.cpp --exclude src/unit-deserialization.cpp --exclude src/unit-element_access1.cpp --exclude src/unit-element_access2.cpp --exclude src/unit-inspection.cpp --exclude src/unit-iterator_wrapper.cpp --exclude src/unit-iterators1.cpp --exclude src/unit-iterators2.cpp --exclude src/unit-json_patch.cpp --exclude src/unit-json_pointer.cpp --exclude src/unit-modifiers.cpp --exclude src/unit-pointer_access.cpp --exclude src/unit-readme.cpp --exclude src/unit-reference_access.cpp --exclude src/unit-regression.cpp --exclude src/unit-serialization.cpp --exclude src/unit-testsuites.cpp --exclude src/unit-unicode.cpp --include ../src/json.hpp --gcov-options '\-lp' --gcov 'gcov-4.9' - lcov --directory src --directory test/src --capture --output-file coverage.info --rc lcov_branch_coverage=1 --no-external - lcov --remove coverage.info 'test/src/*' --output-file coverage.info --rc lcov_branch_coverage=1 @@ -248,11 +247,8 @@ script: - uname -a - $CXX --version - # compile - - make - - # execute unit tests - - test/json_unit "*" + # compile and execute unit tests + - make check # check if homebrew works (only checks develop branch) - if [ `which brew` ]; then @@ -261,51 +257,3 @@ script: brew install nlohmann_json --HEAD ; brew test nlohmann_json ; fi - -#language: cpp -# -#dist: trusty -#sudo: required -# -#env: -# global: -# # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created -# # via the "travis encrypt" command using the project repo's public key -# - secure: "m89SSgE+ASLO38rSKx7MTXK3n5NkP9bIx95jwY71YEiuFzib30PDJ/DifKnXxBjvy/AkCGztErQRk/8ZCvq+4HXozU2knEGnL/RUitvlwbhzfh2D4lmS3BvWBGS3N3NewoPBrRmdcvnT0xjOGXxtZaJ3P74TkB9GBnlz/HmKORA=" -# -## from http://stackoverflow.com/a/32127147/266378 -#matrix: -# include: -# - os: linux -# compiler: gcc -# addons: -# apt: -# sources: ['ubuntu-toolchain-r-test'] -# packages: ['g++-4.9', 'valgrind', 'python-pip', 'python-yaml'] -# before_script: -# - pip install --user git+git://github.com/eddyxu/cpp-coveralls.git -# after_success: -# - make clean -# - touch src/json.hpp -# - make json_unit CXXFLAGS="-fprofile-arcs -ftest-coverage -std=c++11 -lstdc++" CXX=$COMPILER -# - test/json_unit "*" -# - coveralls --build-root test --exclude src/catch.hpp --exclude src/unit-algorithms.cpp --exclude src/unit-allocator.cpp --exclude src/unit-capacity.cpp --exclude src/unit-class_const_iterator.cpp --exclude src/unit-class_iterator.cpp --exclude src/unit-class_lexer.cpp --exclude src/unit-class_parser.cpp --exclude src/unit-comparison.cpp --exclude src/unit-concepts.cpp --exclude src/unit-constructor1.cpp --exclude src/unit-constructor2.cpp --exclude src/unit-convenience.cpp --exclude src/unit-conversions.cpp --exclude src/unit-deserialization.cpp --exclude src/unit-element_access1.cpp --exclude src/unit-element_access2.cpp --exclude src/unit-inspection.cpp --exclude src/unit-iterator_wrapper.cpp --exclude src/unit-iterators1.cpp --exclude src/unit-iterators2.cpp --exclude src/unit-json_patch.cpp --exclude src/unit-json_pointer.cpp --exclude src/unit-modifiers.cpp --exclude src/unit-pointer_access.cpp --exclude src/unit-readme.cpp --exclude src/unit-reference_access.cpp --exclude src/unit-regression.cpp --exclude src/unit-serialization.cpp --exclude src/unit-testsuites.cpp --exclude src/unit-unicode.cpp --include ../src/json.hpp --gcov-options '\-lp' --gcov 'gcov-4.9' -# env: COMPILER=g++-4.9 -# -# - os: linux -# compiler: gcc -# before_install: echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-certificates.crt -# addons: -# apt: -# sources: ['ubuntu-toolchain-r-test'] -# packages: ['g++-5', 'valgrind'] -# coverity_scan: -# project: -# name: "nlohmann/json" -# description: "Build submitted via Travis CI" -# notification_email: niels.lohmann@gmail.com -# build_command_prepend: "make clean ; sudo cp $(which g++-5) $(which g++)" -# build_command: "make" -# branch_pattern: coverity_scan -# env: COMPILER=g++-5 -# diff --git a/Makefile b/Makefile index 3fd6d54b1..fcce453e2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,8 @@ RE2C = re2c SED = sed # main target -all: json_unit +all: + $(MAKE) -C test # clean up clean: @@ -21,14 +22,14 @@ clean: # build unit tests json_unit: - @$(MAKE) -C test + @$(MAKE) json_unit -C test # run unit tests -check: json_unit - test/json_unit "*" +check: + $(MAKE) check -C test -check-fast: json_unit - test/json_unit +check-fast: + $(MAKE) check -C test TEST_PATTERN="" ########################################################################## @@ -69,6 +70,7 @@ cppcheck: clang_sanitize: clean CXX=clang++ CXXFLAGS="-g -O2 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer" $(MAKE) + ########################################################################## # maintainer targets ########################################################################## diff --git a/test/Makefile b/test/Makefile index ead1f0738..ba146bff4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -40,7 +40,20 @@ SOURCES = src/unit.cpp \ OBJECTS = $(SOURCES:.cpp=.o) -all: json_unit +TESTCASES = $(patsubst src/unit-%.cpp,test-%,$(wildcard src/unit-*.cpp)) + +############################################################################## +# main rules +############################################################################## + +all: $(TESTCASES) + +clean: + rm -fr json_unit $(OBJECTS) $(SOURCES:.cpp=.gcno) $(SOURCES:.cpp=.gcda) $(TESTCASES) + +############################################################################## +# single test file +############################################################################## json_unit: $(OBJECTS) ../src/json.hpp src/catch.hpp @echo "[CXXLD] $@" @@ -50,5 +63,16 @@ json_unit: $(OBJECTS) ../src/json.hpp src/catch.hpp @echo "[CXX] $@" @$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $< -o $@ -clean: - rm -fr json_unit $(OBJECTS) $(SOURCES:.cpp=.gcno) $(SOURCES:.cpp=.gcda) + +############################################################################## +# individual test cases +############################################################################## + +test-%: src/unit-%.cpp + @echo "[CXXLD] $@" + @$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) -DCATCH_CONFIG_MAIN $< -o $@ + +TEST_PATTERN = "*" +TEST_PREFIX = "" +check: $(TESTCASES) + @cd .. ; for testcase in $(TESTCASES); do echo "Executing $$testcase..."; $(TEST_PREFIX)test/$$testcase $(TEST_PATTERN) -r compact; done From 36c79113fb1b6232d099e62a2fac2f3fd492bade Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 27 Nov 2016 16:45:24 +0100 Subject: [PATCH 2/5] :construction: fixing coveralls and valgrind targets --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 430ef75c2..48bc2ad56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: sources: ['ubuntu-toolchain-r-test'] packages: [g++-4.9, valgrind] after_success: - - make check make check TEST_PREFIX="valgrind --error-exitcode=1 --leak-check=full " TEST_PATTERN="" + - make check TEST_PREFIX="valgrind --error-exitcode=1 --leak-check=full " TEST_PATTERN="" # cppcheck @@ -68,7 +68,8 @@ matrix: - tar xf lcov_1.11.orig.tar.gz - sudo make -C lcov-1.11/ install - gem install coveralls-lcov - after_success: + - pip install --user cpp-coveralls + after_success: - make clean - CXXFLAGS="--coverage -g -O0" CPPFLAGS="-DNDEBUG" make check - coveralls --build-root test --exclude src/catch.hpp --exclude src/unit-algorithms.cpp --exclude src/unit-allocator.cpp --exclude src/unit-capacity.cpp --exclude src/unit-class_const_iterator.cpp --exclude src/unit-class_iterator.cpp --exclude src/unit-class_lexer.cpp --exclude src/unit-class_parser.cpp --exclude src/unit-comparison.cpp --exclude src/unit-concepts.cpp --exclude src/unit-constructor1.cpp --exclude src/unit-constructor2.cpp --exclude src/unit-convenience.cpp --exclude src/unit-conversions.cpp --exclude src/unit-deserialization.cpp --exclude src/unit-element_access1.cpp --exclude src/unit-element_access2.cpp --exclude src/unit-inspection.cpp --exclude src/unit-iterator_wrapper.cpp --exclude src/unit-iterators1.cpp --exclude src/unit-iterators2.cpp --exclude src/unit-json_patch.cpp --exclude src/unit-json_pointer.cpp --exclude src/unit-modifiers.cpp --exclude src/unit-pointer_access.cpp --exclude src/unit-readme.cpp --exclude src/unit-reference_access.cpp --exclude src/unit-regression.cpp --exclude src/unit-serialization.cpp --exclude src/unit-testsuites.cpp --exclude src/unit-unicode.cpp --include ../src/json.hpp --gcov-options '\-lp' --gcov 'gcov-4.9' @@ -186,6 +187,10 @@ matrix: env: LLVM_VERSION=3.8.1 compiler: clang + - os: linux + env: LLVM_VERSION=3.9.0 + compiler: clang + ##################### # installation step # ##################### @@ -200,6 +205,7 @@ cache: - ${TRAVIS_BUILD_DIR}/deps/llvm-3.7.1 - ${TRAVIS_BUILD_DIR}/deps/llvm-3.8.0 - ${TRAVIS_BUILD_DIR}/deps/llvm-3.8.1 + - ${TRAVIS_BUILD_DIR}/deps/llvm-3.9.0 install: From 71c1232e32f85967dc23030f3f26267d6aeb559e Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 27 Nov 2016 16:46:33 +0100 Subject: [PATCH 3/5] :bug: fixed YAML --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 48bc2ad56..49312f7cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,7 @@ matrix: - sudo make -C lcov-1.11/ install - gem install coveralls-lcov - pip install --user cpp-coveralls - after_success: + after_success: - make clean - CXXFLAGS="--coverage -g -O0" CPPFLAGS="-DNDEBUG" make check - coveralls --build-root test --exclude src/catch.hpp --exclude src/unit-algorithms.cpp --exclude src/unit-allocator.cpp --exclude src/unit-capacity.cpp --exclude src/unit-class_const_iterator.cpp --exclude src/unit-class_iterator.cpp --exclude src/unit-class_lexer.cpp --exclude src/unit-class_parser.cpp --exclude src/unit-comparison.cpp --exclude src/unit-concepts.cpp --exclude src/unit-constructor1.cpp --exclude src/unit-constructor2.cpp --exclude src/unit-convenience.cpp --exclude src/unit-conversions.cpp --exclude src/unit-deserialization.cpp --exclude src/unit-element_access1.cpp --exclude src/unit-element_access2.cpp --exclude src/unit-inspection.cpp --exclude src/unit-iterator_wrapper.cpp --exclude src/unit-iterators1.cpp --exclude src/unit-iterators2.cpp --exclude src/unit-json_patch.cpp --exclude src/unit-json_pointer.cpp --exclude src/unit-modifiers.cpp --exclude src/unit-pointer_access.cpp --exclude src/unit-readme.cpp --exclude src/unit-reference_access.cpp --exclude src/unit-regression.cpp --exclude src/unit-serialization.cpp --exclude src/unit-testsuites.cpp --exclude src/unit-unicode.cpp --include ../src/json.hpp --gcov-options '\-lp' --gcov 'gcov-4.9' From f860f323be4d2c4b6bf330d6aecf2803ea77e758 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 27 Nov 2016 17:09:05 +0100 Subject: [PATCH 4/5] :construction: falling back to single unit test for coveralls --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 49312f7cc..3fa3e4f5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,7 +71,8 @@ matrix: - pip install --user cpp-coveralls after_success: - make clean - - CXXFLAGS="--coverage -g -O0" CPPFLAGS="-DNDEBUG" make check + - CXXFLAGS="--coverage -g -O0" CPPFLAGS="-DNDEBUG" make json_unit + - test/json_unit "*" - coveralls --build-root test --exclude src/catch.hpp --exclude src/unit-algorithms.cpp --exclude src/unit-allocator.cpp --exclude src/unit-capacity.cpp --exclude src/unit-class_const_iterator.cpp --exclude src/unit-class_iterator.cpp --exclude src/unit-class_lexer.cpp --exclude src/unit-class_parser.cpp --exclude src/unit-comparison.cpp --exclude src/unit-concepts.cpp --exclude src/unit-constructor1.cpp --exclude src/unit-constructor2.cpp --exclude src/unit-convenience.cpp --exclude src/unit-conversions.cpp --exclude src/unit-deserialization.cpp --exclude src/unit-element_access1.cpp --exclude src/unit-element_access2.cpp --exclude src/unit-inspection.cpp --exclude src/unit-iterator_wrapper.cpp --exclude src/unit-iterators1.cpp --exclude src/unit-iterators2.cpp --exclude src/unit-json_patch.cpp --exclude src/unit-json_pointer.cpp --exclude src/unit-modifiers.cpp --exclude src/unit-pointer_access.cpp --exclude src/unit-readme.cpp --exclude src/unit-reference_access.cpp --exclude src/unit-regression.cpp --exclude src/unit-serialization.cpp --exclude src/unit-testsuites.cpp --exclude src/unit-unicode.cpp --include ../src/json.hpp --gcov-options '\-lp' --gcov 'gcov-4.9' - lcov --directory src --directory test/src --capture --output-file coverage.info --rc lcov_branch_coverage=1 --no-external - lcov --remove coverage.info 'test/src/*' --output-file coverage.info --rc lcov_branch_coverage=1 From 2477b09cb9ec0e476a303cf0c5f88721921579ed Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 27 Nov 2016 17:29:19 +0100 Subject: [PATCH 5/5] :bug: Clang 3.9.0 seems to not work right now --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3fa3e4f5e..945623574 100644 --- a/.travis.yml +++ b/.travis.yml @@ -188,10 +188,6 @@ matrix: env: LLVM_VERSION=3.8.1 compiler: clang - - os: linux - env: LLVM_VERSION=3.9.0 - compiler: clang - ##################### # installation step # ##################### @@ -206,7 +202,6 @@ cache: - ${TRAVIS_BUILD_DIR}/deps/llvm-3.7.1 - ${TRAVIS_BUILD_DIR}/deps/llvm-3.8.0 - ${TRAVIS_BUILD_DIR}/deps/llvm-3.8.1 - - ${TRAVIS_BUILD_DIR}/deps/llvm-3.9.0 install: