From 95aa0bf79b968d7ea04a51561a176ae1f5a460bd Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 9 Dec 2017 23:04:49 +0000 Subject: [PATCH 01/12] add load_from_binary/load_from_json fuzzers --- contrib/fuzz_testing/fuzz.sh | 6 +- tests/data/fuzz/load-from-binary/BINARY1 | Bin 0 -> 1 bytes tests/data/fuzz/load-from-json/JSON1 | 1 + tests/fuzz/CMakeLists.txt | 24 +++++++ tests/fuzz/load_from_binary.cpp | 76 +++++++++++++++++++++++ tests/fuzz/load_from_json.cpp | 76 +++++++++++++++++++++++ 6 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 tests/data/fuzz/load-from-binary/BINARY1 create mode 100644 tests/data/fuzz/load-from-json/JSON1 create mode 100644 tests/fuzz/load_from_binary.cpp create mode 100644 tests/fuzz/load_from_json.cpp diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index 35b74f7e4..ad321210a 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -10,12 +10,12 @@ fi type="$1" if test -z "$type" then - echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction" + echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json" exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json"; exit 1 ;; esac afl-fuzz -i tests/data/fuzz/$type -m 150 -t 250 -o fuzz-out/$type build/fuzz/tests/fuzz/${type}_fuzz_tests diff --git a/tests/data/fuzz/load-from-binary/BINARY1 b/tests/data/fuzz/load-from-binary/BINARY1 new file mode 100644 index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d GIT binary patch literal 1 IcmZPo000310RR91 literal 0 HcmV?d00001 diff --git a/tests/data/fuzz/load-from-json/JSON1 b/tests/data/fuzz/load-from-json/JSON1 new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tests/data/fuzz/load-from-json/JSON1 @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index 853d46a12..01c17359c 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -89,3 +89,27 @@ set_property(TARGET cold-transaction_fuzz_tests PROPERTY FOLDER "tests") +add_executable(load-from-binary_fuzz_tests load_from_binary.cpp fuzzer.cpp) +target_link_libraries(load-from-binary_fuzz_tests + PRIVATE + common + epee + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) +set_property(TARGET load-from-binary_fuzz_tests + PROPERTY + FOLDER "tests") + +add_executable(load-from-json_fuzz_tests load_from_json.cpp fuzzer.cpp) +target_link_libraries(load-from-json_fuzz_tests + PRIVATE + common + epee + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) +set_property(TARGET load-from-json_fuzz_tests + PROPERTY + FOLDER "tests") + diff --git a/tests/fuzz/load_from_binary.cpp b/tests/fuzz/load_from_binary.cpp new file mode 100644 index 000000000..3c8dd177b --- /dev/null +++ b/tests/fuzz/load_from_binary.cpp @@ -0,0 +1,76 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage_template_helper.h" +#include "storages/portable_storage_base.h" +#include "fuzzer.h" + +class PortableStorageFuzzer: public Fuzzer +{ +public: + PortableStorageFuzzer() {} + virtual int init(); + virtual int run(const std::string &filename); +}; + +int PortableStorageFuzzer::init() +{ + return 0; +} + +int PortableStorageFuzzer::run(const std::string &filename) +{ + std::string s; + + if (!epee::file_io_utils::load_file_to_string(filename, s)) + { + std::cout << "Error: failed to load file " << filename << std::endl; + return 1; + } + try + { + epee::serialization::portable_storage ps; + ps.load_from_binary(s); + } + catch (const std::exception &e) + { + std::cerr << "Failed to load from binary: " << e.what() << std::endl; + return 1; + } + return 0; +} + +int main(int argc, const char **argv) +{ + PortableStorageFuzzer fuzzer; + return run_fuzzer(argc, argv, fuzzer); +} + diff --git a/tests/fuzz/load_from_json.cpp b/tests/fuzz/load_from_json.cpp new file mode 100644 index 000000000..5d39c89a6 --- /dev/null +++ b/tests/fuzz/load_from_json.cpp @@ -0,0 +1,76 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage_template_helper.h" +#include "storages/portable_storage_base.h" +#include "fuzzer.h" + +class PortableStorageFuzzer: public Fuzzer +{ +public: + PortableStorageFuzzer() {} + virtual int init(); + virtual int run(const std::string &filename); +}; + +int PortableStorageFuzzer::init() +{ + return 0; +} + +int PortableStorageFuzzer::run(const std::string &filename) +{ + std::string s; + + if (!epee::file_io_utils::load_file_to_string(filename, s)) + { + std::cout << "Error: failed to load file " << filename << std::endl; + return 1; + } + try + { + epee::serialization::portable_storage ps; + ps.load_from_json(s); + } + catch (const std::exception &e) + { + std::cerr << "Failed to load from binary: " << e.what() << std::endl; + return 1; + } + return 0; +} + +int main(int argc, const char **argv) +{ + PortableStorageFuzzer fuzzer; + return run_fuzzer(argc, argv, fuzzer); +} + From c27d9092394ca8c37fba6066c497b5f8445702ef Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 09:58:08 +0000 Subject: [PATCH 02/12] Makefile: build fuzz tests statically, starts faster --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4f6e6465f..6e651a431 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ release-static-win32: fuzz: mkdir -p build/fuzz - cd build/fuzz && cmake -D BUILD_TESTS=ON -D USE_LTO=OFF -D CMAKE_C_COMPILER=afl-gcc -D CMAKE_CXX_COMPILER=afl-g++ -D ARCH="x86-64" -D CMAKE_BUILD_TYPE=fuzz -D BUILD_TAG="linux-x64" ../.. && $(MAKE) + cd build/fuzz && cmake -D STATIC=ON -D BUILD_TESTS=ON -D USE_LTO=OFF -D CMAKE_C_COMPILER=afl-gcc -D CMAKE_CXX_COMPILER=afl-g++ -D ARCH="x86-64" -D CMAKE_BUILD_TYPE=fuzz -D BUILD_TAG="linux-x64" ../.. && $(MAKE) clean: @echo "WARNING: Back-up your wallet if it exists within ./build!" ; \ From 4cd4a4313dea6a270a941ae301b4cc20393e7967 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 11:24:26 +0000 Subject: [PATCH 03/12] fuzz_testing: create out directory if needed, and fix filename passing --- contrib/fuzz_testing/fuzz.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index ad321210a..c339ceeed 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -18,4 +18,5 @@ case "$type" in *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json"; exit 1 ;; esac -afl-fuzz -i tests/data/fuzz/$type -m 150 -t 250 -o fuzz-out/$type build/fuzz/tests/fuzz/${type}_fuzz_tests +mkdir -p fuzz-out +afl-fuzz -i tests/data/fuzz/$type -m none -t 250 -o fuzz-out/$type build/fuzz/tests/fuzz/${type}_fuzz_tests @@ From 1a379ef656622ac535774e7111e85e02b6aa1d8a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 11:25:12 +0000 Subject: [PATCH 04/12] fuzz_testing: build with ASAN (assumed to be available) --- CMakeLists.txt | 10 ++++++++++ Makefile | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b11d6ba6f..11c549d7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -238,6 +238,16 @@ if(STATIC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DZMQ_STATIC") endif() +if(SANITIZE) + if (MSVC) + message(FATAL_ERROR "Cannot sanitize with MSVC") + else() + message(STATUS "Using ASAN") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + endif() +endif() + # Set default blockchain storage location: # memory was the default in Cryptonote before Monero implimented LMDB, it still works but is unneccessary. # set(DATABASE memory) diff --git a/Makefile b/Makefile index 6e651a431..5a5e4b703 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ release-static-win32: fuzz: mkdir -p build/fuzz - cd build/fuzz && cmake -D STATIC=ON -D BUILD_TESTS=ON -D USE_LTO=OFF -D CMAKE_C_COMPILER=afl-gcc -D CMAKE_CXX_COMPILER=afl-g++ -D ARCH="x86-64" -D CMAKE_BUILD_TYPE=fuzz -D BUILD_TAG="linux-x64" ../.. && $(MAKE) + cd build/fuzz && cmake -D STATIC=ON -D SANITIZE=ON -D BUILD_TESTS=ON -D USE_LTO=OFF -D CMAKE_C_COMPILER=afl-gcc -D CMAKE_CXX_COMPILER=afl-g++ -D ARCH="x86-64" -D CMAKE_BUILD_TYPE=fuzz -D BUILD_TAG="linux-x64" ../.. && $(MAKE) clean: @echo "WARNING: Back-up your wallet if it exists within ./build!" ; \ From 53b83a83fcbfac7b6d3a389e47dcbc299e310877 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 11:34:52 +0000 Subject: [PATCH 05/12] tests: better load-from-binary fuzz test data file Looks like there's some kind of header/signature --- tests/data/fuzz/load-from-binary/BINARY1 | Bin 1 -> 10 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/data/fuzz/load-from-binary/BINARY1 b/tests/data/fuzz/load-from-binary/BINARY1 index f76dd238ade08917e6712764a16a22005a50573d..c99d7e7bcf358e6b29e40d4c9090d36dad3be581 100644 GIT binary patch literal 10 PcmZP+WCQ{xMn(nz0Mq~* literal 1 IcmZPo000310RR91 From c80bb0eb6fdcb0a6157c0adf15c15619f3c8445f Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 13:20:06 +0000 Subject: [PATCH 06/12] tests: don't init stuff we don't need in fuzz tests - faster --- tests/fuzz/fuzzer.cpp | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/tests/fuzz/fuzzer.cpp b/tests/fuzz/fuzzer.cpp index ede3fcc40..686a5e5f0 100644 --- a/tests/fuzz/fuzzer.cpp +++ b/tests/fuzz/fuzzer.cpp @@ -44,41 +44,11 @@ static int __AFL_LOOP(int) } #endif -using namespace epee; -using namespace boost::program_options; - int run_fuzzer(int argc, const char **argv, Fuzzer &fuzzer) { - TRY_ENTRY(); - tools::on_startup(); - string_tools::set_module_name_and_folder(argv[0]); - - //set up logging options - mlog_configure(mlog_get_default_log_path("fuzztests.log"), true); - mlog_set_log("*:FATAL,logging:none"); - - options_description desc_options("Allowed options"); - command_line::add_arg(desc_options, command_line::arg_help); - - variables_map vm; - bool r = command_line::handle_error_helper(desc_options, [&]() - { - store(parse_command_line(argc, argv, desc_options), vm); - notify(vm); - return true; - }); - if (!r) - return 1; - - if (command_line::get_arg(vm, command_line::arg_help)) - { - std::cout << desc_options << std::endl; - return 0; - } - if (argc < 2) { - std::cout << desc_options << std::endl; + std::cout << "usage: " << argv[0] << " " << "" << std::endl; return 1; } @@ -94,6 +64,5 @@ int run_fuzzer(int argc, const char **argv, Fuzzer &fuzzer) return ret; } - CATCH_ENTRY_L0("fuzzer_main", 1); return 0; } From fd05208762c04a8a85cfe268361a09dae3783098 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 13:20:32 +0000 Subject: [PATCH 07/12] fuzz_testing: allow automatically resuming an interrupted job --- contrib/fuzz_testing/fuzz.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index c339ceeed..dea09e64c 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -18,5 +18,12 @@ case "$type" in *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json"; exit 1 ;; esac +if test -d "fuzz-out/$type" +then + dir="-" +else + dir="tests/data/fuzz/$type" +fi + mkdir -p fuzz-out -afl-fuzz -i tests/data/fuzz/$type -m none -t 250 -o fuzz-out/$type build/fuzz/tests/fuzz/${type}_fuzz_tests @@ +afl-fuzz -i "$dir" -m none -t 250 -o fuzz-out/$type build/fuzz/tests/fuzz/${type}_fuzz_tests @@ From 261b0dd0e041e9e1fd5370cf5fca44ae2d84123e Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 15:52:24 +0000 Subject: [PATCH 08/12] tests: add base58 fuzz test --- contrib/fuzz_testing/fuzz.sh | 6 +-- tests/data/fuzz/base58/ENC1 | 0 tests/data/fuzz/base58/ENC2 | 1 + tests/fuzz/CMakeLists.txt | 12 ++++++ tests/fuzz/base58.cpp | 74 ++++++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 tests/data/fuzz/base58/ENC1 create mode 100644 tests/data/fuzz/base58/ENC2 create mode 100644 tests/fuzz/base58.cpp diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index dea09e64c..4e63e6bcd 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -10,12 +10,12 @@ fi type="$1" if test -z "$type" then - echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json" + echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58" exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58"; exit 1 ;; esac if test -d "fuzz-out/$type" diff --git a/tests/data/fuzz/base58/ENC1 b/tests/data/fuzz/base58/ENC1 new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/fuzz/base58/ENC2 b/tests/data/fuzz/base58/ENC2 new file mode 100644 index 000000000..73da627e4 --- /dev/null +++ b/tests/data/fuzz/base58/ENC2 @@ -0,0 +1 @@ +9zZBkWRgMNPEnVofRFqWK9MKBwgXNyKELBJSttxb1t2UhDM114URntt5iYcXzXusoHZygfSojsbviXZhnP9pJ4p2SDcv81L \ No newline at end of file diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index 01c17359c..5b1cef182 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -113,3 +113,15 @@ set_property(TARGET load-from-json_fuzz_tests PROPERTY FOLDER "tests") +add_executable(base58_fuzz_tests base58.cpp fuzzer.cpp) +target_link_libraries(base58_fuzz_tests + PRIVATE + common + epee + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) +set_property(TARGET base58_fuzz_tests + PROPERTY + FOLDER "tests") + diff --git a/tests/fuzz/base58.cpp b/tests/fuzz/base58.cpp new file mode 100644 index 000000000..aea62f721 --- /dev/null +++ b/tests/fuzz/base58.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "common/base58.h" +#include "fuzzer.h" + +class Base58Fuzzer: public Fuzzer +{ +public: + Base58Fuzzer() {} + virtual int init(); + virtual int run(const std::string &filename); +}; + +int Base58Fuzzer::init() +{ + return 0; +} + +int Base58Fuzzer::run(const std::string &filename) +{ + std::string s; + + if (!epee::file_io_utils::load_file_to_string(filename, s)) + { + std::cout << "Error: failed to load file " << filename << std::endl; + return 1; + } + try + { + std::string data; + tools::base58::decode(s, data); + } + catch (const std::exception &e) + { + std::cerr << "Failed to load from binary: " << e.what() << std::endl; + return 1; + } + return 0; +} + +int main(int argc, const char **argv) +{ + Base58Fuzzer fuzzer; + return run_fuzzer(argc, argv, fuzzer); +} + From 0272df9e614d472973fbcbdb50591d2dcd8eb10c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 10 Dec 2017 20:17:16 +0000 Subject: [PATCH 09/12] add parse_url fuzz test --- contrib/fuzz_testing/fuzz.sh | 6 +-- tests/data/fuzz/parse-url/URL1 | 1 + tests/data/fuzz/parse-url/URL2 | 1 + tests/fuzz/CMakeLists.txt | 13 ++++++ tests/fuzz/parse_url.cpp | 74 ++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/data/fuzz/parse-url/URL1 create mode 100644 tests/data/fuzz/parse-url/URL2 create mode 100644 tests/fuzz/parse_url.cpp diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index 4e63e6bcd..5be16a7d6 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -10,12 +10,12 @@ fi type="$1" if test -z "$type" then - echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58" + echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url" exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url"; exit 1 ;; esac if test -d "fuzz-out/$type" diff --git a/tests/data/fuzz/parse-url/URL1 b/tests/data/fuzz/parse-url/URL1 new file mode 100644 index 000000000..e56ea71e3 --- /dev/null +++ b/tests/data/fuzz/parse-url/URL1 @@ -0,0 +1 @@ +127.0.0.1 \ No newline at end of file diff --git a/tests/data/fuzz/parse-url/URL2 b/tests/data/fuzz/parse-url/URL2 new file mode 100644 index 000000000..b66e7de9a --- /dev/null +++ b/tests/data/fuzz/parse-url/URL2 @@ -0,0 +1 @@ +iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= \ No newline at end of file diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index 5b1cef182..8090b465a 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -125,3 +125,16 @@ set_property(TARGET base58_fuzz_tests PROPERTY FOLDER "tests") +add_executable(parse-url_fuzz_tests parse_url.cpp fuzzer.cpp) +target_link_libraries(parse-url_fuzz_tests + PRIVATE + epee + ${Boost_REGEX_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) +set_property(TARGET parse-url_fuzz_tests + PROPERTY + FOLDER "tests") + diff --git a/tests/fuzz/parse_url.cpp b/tests/fuzz/parse_url.cpp new file mode 100644 index 000000000..bf3a3bdd4 --- /dev/null +++ b/tests/fuzz/parse_url.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "net/net_parse_helpers.h" +#include "fuzzer.h" + +class ParseURLFuzzer: public Fuzzer +{ +public: + ParseURLFuzzer() {} + virtual int init(); + virtual int run(const std::string &filename); +}; + +int ParseURLFuzzer::init() +{ + return 0; +} + +int ParseURLFuzzer::run(const std::string &filename) +{ + std::string s; + + if (!epee::file_io_utils::load_file_to_string(filename, s)) + { + std::cout << "Error: failed to load file " << filename << std::endl; + return 1; + } + try + { + epee::net_utils::http::url_content url; + epee::net_utils::parse_url(s, url); + } + catch (const std::exception &e) + { + std::cerr << "Failed to load from binary: " << e.what() << std::endl; + return 1; + } + return 0; +} + +int main(int argc, const char **argv) +{ + ParseURLFuzzer fuzzer; + return run_fuzzer(argc, argv, fuzzer); +} + From f1bdc9a42a7d75f90cc0c1e4191f4ef1a016a11a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 11 Dec 2017 09:33:01 +0000 Subject: [PATCH 10/12] tests: add http client fuzz test --- contrib/epee/include/net/http_client.h | 17 ++++- contrib/fuzz_testing/fuzz.sh | 6 +- tests/data/fuzz/http-client/RESP1 | 8 +++ tests/fuzz/CMakeLists.txt | 14 ++++ tests/fuzz/http-client.cpp | 98 ++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 tests/data/fuzz/http-client/RESP1 create mode 100644 tests/fuzz/http-client.cpp diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 1a9d5d064..5c448fcb3 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -236,7 +236,8 @@ namespace net_utils namespace http { - class http_simple_client: public i_target_handler + template + class http_simple_client_template: public i_target_handler { private: enum reciev_machine_state @@ -259,7 +260,7 @@ namespace net_utils }; - blocked_mode_client m_net_client; + net_client_type m_net_client; std::string m_host_buff; std::string m_port; http_client_auth m_auth; @@ -276,7 +277,7 @@ namespace net_utils bool m_ssl; public: - explicit http_simple_client() + explicit http_simple_client_template() : i_target_handler() , m_net_client() , m_host_buff() @@ -428,6 +429,15 @@ namespace net_utils CRITICAL_REGION_LOCAL(m_lock); return invoke(uri, "POST", body, timeout, ppresponse_info, additional_params); } + //--------------------------------------------------------------------------- + bool test(const std::string &s, std::chrono::milliseconds timeout) // TEST FUNC ONLY + { + CRITICAL_REGION_LOCAL(m_lock); + m_net_client.set_test_data(s); + m_state = reciev_machine_state_header; + return handle_reciev(timeout); + } + //--------------------------------------------------------------------------- private: //--------------------------------------------------------------------------- inline bool handle_reciev(std::chrono::milliseconds timeout) @@ -957,6 +967,7 @@ namespace net_utils return true; } }; + typedef http_simple_client_template http_simple_client; } } } diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index 5be16a7d6..a97713510 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -10,12 +10,12 @@ fi type="$1" if test -z "$type" then - echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url" + echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client" exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client"; exit 1 ;; esac if test -d "fuzz-out/$type" diff --git a/tests/data/fuzz/http-client/RESP1 b/tests/data/fuzz/http-client/RESP1 new file mode 100644 index 000000000..3046dc886 --- /dev/null +++ b/tests/data/fuzz/http-client/RESP1 @@ -0,0 +1,8 @@ +HTTP/1.1 200 Ok +Server: Epee-based +Content-Length: 5 +Content-Type: text/plain +Last-Modified: Mon, 11 Dec 2017 09:03:03 GMT +Accept-Ranges: bytes + +foo diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index 8090b465a..2184ae48a 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -138,3 +138,17 @@ set_property(TARGET parse-url_fuzz_tests PROPERTY FOLDER "tests") +add_executable(http-client_fuzz_tests http-client.cpp fuzzer.cpp) +target_link_libraries(http-client_fuzz_tests + PRIVATE + epee + ${Boost_THREAD_LIBRARY} + ${Boost_REGEX_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) +set_property(TARGET http-client_fuzz_tests + PROPERTY + FOLDER "tests") + diff --git a/tests/fuzz/http-client.cpp b/tests/fuzz/http-client.cpp new file mode 100644 index 000000000..21560df21 --- /dev/null +++ b/tests/fuzz/http-client.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "net/http_client.h" +#include "fuzzer.h" + +class dummy_client +{ +public: + bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout, bool ssl = false, const std::string& bind_ip = "0.0.0.0") { return true; } + bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, bool ssl = false, const std::string& bind_ip = "0.0.0.0") { return true; } + bool disconnect() { return true; } + bool send(const std::string& buff, std::chrono::milliseconds timeout) { return true; } + bool send(const void* data, size_t sz) { return true; } + bool is_connected() { return true; } + bool recv(std::string& buff, std::chrono::milliseconds timeout) + { + buff = data; + data.clear(); + return true; + } + + void set_test_data(const std::string &s) { data = s; } + +private: + std::string data; +}; + +class HTTPClientFuzzer: public Fuzzer +{ +public: + HTTPClientFuzzer() {} + virtual int init(); + virtual int run(const std::string &filename); + +private: + epee::net_utils::http::http_simple_client_template client; +}; + +int HTTPClientFuzzer::init() +{ + return 0; +} + +int HTTPClientFuzzer::run(const std::string &filename) +{ + std::string s; + + if (!epee::file_io_utils::load_file_to_string(filename, s)) + { + std::cout << "Error: failed to load file " << filename << std::endl; + return 1; + } + try + { + client.test(s, std::chrono::milliseconds(1000)); + } + catch (const std::exception &e) + { + std::cerr << "Failed to test http client: " << e.what() << std::endl; + return 1; + } + return 0; +} + +int main(int argc, const char **argv) +{ + HTTPClientFuzzer fuzzer; + return run_fuzzer(argc, argv, fuzzer); +} + From ec724eb64ab4fc5dcc518efc606d630ebc89f119 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 11 Dec 2017 18:32:40 +0000 Subject: [PATCH 11/12] tests: add levin fuzz test --- contrib/fuzz_testing/fuzz.sh | 6 +- tests/data/fuzz/levin/LEVIN1 | Bin 0 -> 33 bytes tests/fuzz/CMakeLists.txt | 14 ++ tests/fuzz/levin.cpp | 345 +++++++++++++++++++++++++++++++++++ 4 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 tests/data/fuzz/levin/LEVIN1 create mode 100644 tests/fuzz/levin.cpp diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index a97713510..f1c4ff202 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -10,12 +10,12 @@ fi type="$1" if test -z "$type" then - echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client" + echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin" exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin"; exit 1 ;; esac if test -d "fuzz-out/$type" diff --git a/tests/data/fuzz/levin/LEVIN1 b/tests/data/fuzz/levin/LEVIN1 new file mode 100644 index 0000000000000000000000000000000000000000..51a640c61e78465080cae06e3b22fcb8967114ba GIT binary patch literal 33 YcmZQnWCQ~SC}6z64rV1X0x1v%02TZJWB>pF literal 0 HcmV?d00001 diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index 2184ae48a..5d58f9a3c 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -152,3 +152,17 @@ set_property(TARGET http-client_fuzz_tests PROPERTY FOLDER "tests") +add_executable(levin_fuzz_tests levin.cpp fuzzer.cpp) +target_link_libraries(levin_fuzz_tests + PRIVATE + common + epee + ${Boost_THREAD_LIBRARY} + ${Boost_REGEX_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) +set_property(TARGET levin_fuzz_tests + PROPERTY + FOLDER "tests") + diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp new file mode 100644 index 000000000..2fd60ae50 --- /dev/null +++ b/tests/fuzz/levin.cpp @@ -0,0 +1,345 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "net/net_utils_base.h" +#include "net/abstract_tcp_server2.h" +#include "storages/levin_abstract_invoke2.h" +#include "net/levin_protocol_handler_async.h" +#include "fuzzer.h" + +namespace +{ + class call_counter + { + public: + call_counter() : m_counter(0) { } + + // memory_order_relaxed is enough for call counter + void inc() volatile { m_counter.fetch_add(1, std::memory_order_relaxed); } + size_t get() volatile const { return m_counter.load(std::memory_order_relaxed); } + void reset() volatile { m_counter.store(0, std::memory_order_relaxed); } + + private: + std::atomic m_counter; + }; + + struct test_levin_connection_context : public epee::net_utils::connection_context_base + { + }; + + typedef epee::levin::async_protocol_handler_config test_levin_protocol_handler_config; + typedef epee::levin::async_protocol_handler test_levin_protocol_handler; + + struct test_levin_commands_handler : public epee::levin::levin_commands_handler + { + test_levin_commands_handler() + : m_return_code(LEVIN_OK) + , m_last_command(-1) + { + } + + virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, test_levin_connection_context& context) + { + m_invoke_counter.inc(); + boost::unique_lock lock(m_mutex); + m_last_command = command; + m_last_in_buf = in_buff; + buff_out = m_invoke_out_buf; + return m_return_code; + } + + virtual int notify(int command, const std::string& in_buff, test_levin_connection_context& context) + { + m_notify_counter.inc(); + boost::unique_lock lock(m_mutex); + m_last_command = command; + m_last_in_buf = in_buff; + return m_return_code; + } + + virtual void callback(test_levin_connection_context& context) + { + m_callback_counter.inc(); + //std::cout << "test_levin_commands_handler::callback()" << std::endl; + } + + virtual void on_connection_new(test_levin_connection_context& context) + { + m_new_connection_counter.inc(); + //std::cout << "test_levin_commands_handler::on_connection_new()" << std::endl; + } + + virtual void on_connection_close(test_levin_connection_context& context) + { + m_close_connection_counter.inc(); + //std::cout << "test_levin_commands_handler::on_connection_close()" << std::endl; + } + + size_t invoke_counter() const { return m_invoke_counter.get(); } + size_t notify_counter() const { return m_notify_counter.get(); } + size_t callback_counter() const { return m_callback_counter.get(); } + size_t new_connection_counter() const { return m_new_connection_counter.get(); } + size_t close_connection_counter() const { return m_close_connection_counter.get(); } + + int return_code() const { return m_return_code; } + void return_code(int v) { m_return_code = v; } + + const std::string& invoke_out_buf() const { return m_invoke_out_buf; } + void invoke_out_buf(const std::string& v) { m_invoke_out_buf = v; } + + int last_command() const { return m_last_command; } + const std::string& last_in_buf() const { return m_last_in_buf; } + + private: + call_counter m_invoke_counter; + call_counter m_notify_counter; + call_counter m_callback_counter; + call_counter m_new_connection_counter; + call_counter m_close_connection_counter; + + boost::mutex m_mutex; + + int m_return_code; + std::string m_invoke_out_buf; + + int m_last_command; + std::string m_last_in_buf; + }; + + class test_connection : public epee::net_utils::i_service_endpoint + { + public: + test_connection(boost::asio::io_service& io_service, test_levin_protocol_handler_config& protocol_config) + : m_io_service(io_service) + , m_protocol_handler(this, protocol_config, m_context) + , m_send_return(true) + { + } + + void start() + { + m_protocol_handler.after_init_connection(); + } + + // Implement epee::net_utils::i_service_endpoint interface + virtual bool do_send(const void* ptr, size_t cb) + { + m_send_counter.inc(); + boost::unique_lock lock(m_mutex); + m_last_send_data.append(reinterpret_cast(ptr), cb); + return m_send_return; + } + + virtual bool close() { return true; } + virtual bool call_run_once_service_io() { return true; } + virtual bool request_callback() { return true; } + virtual boost::asio::io_service& get_io_service() { return m_io_service; } + virtual bool add_ref() { return true; } + virtual bool release() { return true; } + + size_t send_counter() const { return m_send_counter.get(); } + + const std::string& last_send_data() const { return m_last_send_data; } + void reset_last_send_data() { boost::unique_lock lock(m_mutex); m_last_send_data.clear(); } + + bool send_return() const { return m_send_return; } + void send_return(bool v) { m_send_return = v; } + + public: + test_levin_connection_context m_context; + test_levin_protocol_handler m_protocol_handler; + + private: + boost::asio::io_service& m_io_service; + + call_counter m_send_counter; + boost::mutex m_mutex; + + std::string m_last_send_data; + + bool m_send_return; + }; + +#if 0 + class async_protocol_handler_test : public ::testing::Test + { + public: + const static uint64_t invoke_timeout = 5 * 1000; + const static size_t max_packet_size = 10 * 1024 * 1024; + + typedef std::unique_ptr test_connection_ptr; + + async_protocol_handler_test(): + m_pcommands_handler(new test_levin_commands_handler()), + m_commands_handler(*m_pcommands_handler) + { + m_handler_config.set_handler(m_pcommands_handler, [](epee::levin::levin_commands_handler *handler) { delete handler; }); + m_handler_config.m_invoke_timeout = invoke_timeout; + m_handler_config.m_max_packet_size = max_packet_size; + } + + virtual void SetUp() + { + } + + protected: + test_connection_ptr create_connection(bool start = true) + { + test_connection_ptr conn(new test_connection(m_io_service, m_handler_config)); + if (start) + { + conn->start(); + } + return conn; + } + + protected: + boost::asio::io_service m_io_service; + test_levin_protocol_handler_config m_handler_config; + test_levin_commands_handler *m_pcommands_handler, &m_commands_handler; + }; + + class positive_test_connection_to_levin_protocol_handler_calls : public async_protocol_handler_test + { + }; + + class test_levin_protocol_handler__hanle_recv_with_invalid_data : public async_protocol_handler_test + { + public: + static const int expected_command = 5615871; + static const int expected_return_code = 782546; + + test_levin_protocol_handler__hanle_recv_with_invalid_data() + : m_expected_invoke_out_buf(512, 'y') + { + } + + virtual void SetUp() + { + async_protocol_handler_test::SetUp(); + + m_conn = create_connection(); + + m_in_data.assign(256, 't'); + + m_req_head.m_signature = LEVIN_SIGNATURE; + m_req_head.m_cb = m_in_data.size(); + m_req_head.m_have_to_return_data = true; + m_req_head.m_command = expected_command; + m_req_head.m_return_code = LEVIN_OK; + m_req_head.m_flags = LEVIN_PACKET_REQUEST; + m_req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + + m_commands_handler.return_code(expected_return_code); + m_commands_handler.invoke_out_buf(m_expected_invoke_out_buf); + } + + protected: + void prepare_buf() + { + m_buf.assign(reinterpret_cast(&m_req_head), sizeof(m_req_head)); + m_buf += m_in_data; + } + + protected: + test_connection_ptr m_conn; + epee::levin::bucket_head2 m_req_head; + std::string m_in_data; + std::string m_buf; + std::string m_expected_invoke_out_buf; + }; +#endif +} + +class LevinFuzzer: public Fuzzer +{ +public: + LevinFuzzer() {} //: handler(endpoint, config, context) {} + virtual int init(); + virtual int run(const std::string &filename); + +private: + //epee::net_utils::connection_context_base context; + //epee::levin::async_protocol_handler<> handler; +}; + +int LevinFuzzer::init() +{ + return 0; +} + +int LevinFuzzer::run(const std::string &filename) +{ + std::string s; + +// + epee::levin::bucket_head2 req_head; + req_head.m_signature = LEVIN_SIGNATURE; + req_head.m_cb = 0; + req_head.m_have_to_return_data = true; + req_head.m_command = 2000; + req_head.m_flags = LEVIN_PACKET_REQUEST; + req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + FILE *f=fopen("/tmp/out.levin", "w"); + fwrite(&req_head,sizeof(req_head),1, f); + fclose(f); +// + if (!epee::file_io_utils::load_file_to_string(filename, s)) + { + std::cout << "Error: failed to load file " << filename << std::endl; + return 1; + } + try + { + //std::unique_ptr conn = new test(); + boost::asio::io_service io_service; + test_levin_protocol_handler_config m_handler_config; + test_levin_commands_handler *m_pcommands_handler = new test_levin_commands_handler(); + m_handler_config.set_handler(m_pcommands_handler, [](epee::levin::levin_commands_handler *handler) { delete handler; }); + std::unique_ptr conn(new test_connection(io_service, m_handler_config)); + conn->start(); + //m_commands_handler.invoke_out_buf(expected_out_data); + //m_commands_handler.return_code(expected_return_code); + conn->m_protocol_handler.handle_recv(s.data(), s.size()); + } + catch (const std::exception &e) + { + std::cerr << "Failed to test http client: " << e.what() << std::endl; + return 1; + } + return 0; +} + +int main(int argc, const char **argv) +{ + LevinFuzzer fuzzer; + return run_fuzzer(argc, argv, fuzzer); +} + From bd1f6029a367e411b66d4ff49f2537947e9d300f Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 12 Dec 2017 13:44:11 +0000 Subject: [PATCH 12/12] http_client: rewrite header parsing manually for speed boost::regex is stupendously atrocious at parsing malformed data --- contrib/epee/include/net/http_client.h | 169 ++++++++++++++----------- 1 file changed, 95 insertions(+), 74 deletions(-) diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 5c448fcb3..1edf65928 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -27,6 +27,7 @@ #pragma once +#include #include #include #include @@ -752,87 +753,107 @@ namespace net_utils return true; } //--------------------------------------------------------------------------- - inline - bool parse_header(http_header_info& body_info, const std::string& m_cache_to_process) - { + inline bool parse_header(http_header_info& body_info, const std::string& m_cache_to_process) + { MTRACE("http_stream_filter::parse_cached_header(*)"); - - STATIC_REGEXP_EXPR_1(rexp_mach_field, - "\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)|(User-Agent)|(Origin)" - // 12 3 4 5 6 7 8 9 10 11 - "|([\\w-]+?)) ?: ?((.*?)(\r?\n))[^\t ]", - //12 13 14 15 - boost::regex::icase | boost::regex::normal); - - boost::smatch result; - std::string::const_iterator it_current_bound = m_cache_to_process.begin(); - std::string::const_iterator it_end_bound = m_cache_to_process.end(); - - - //lookup all fields and fill well-known fields - while( boost::regex_search( it_current_bound, it_end_bound, result, rexp_mach_field, boost::match_default) && result[0].matched) + const char *ptr = m_cache_to_process.c_str(); + while (ptr[0] != '\r' || ptr[1] != '\n') { - const size_t field_val = 14; - //const size_t field_etc_name = 11; - - int i = 2; //start position = 2 - if(result[i++].matched)//"Connection" - body_info.m_connection = result[field_val]; - else if(result[i++].matched)//"Referrer" - body_info.m_referer = result[field_val]; - else if(result[i++].matched)//"Content-Length" - body_info.m_content_length = result[field_val]; - else if(result[i++].matched)//"Content-Type" - body_info.m_content_type = result[field_val]; - else if(result[i++].matched)//"Transfer-Encoding" - body_info.m_transfer_encoding = result[field_val]; - else if(result[i++].matched)//"Content-Encoding" - body_info.m_content_encoding = result[field_val]; - else if(result[i++].matched)//"Host" - { body_info.m_host = result[field_val]; - string_tools::trim(body_info.m_host); + // optional \n + if (*ptr == '\n') + ++ptr; + // an identifier composed of letters or - + const char *key_pos = ptr; + while (isalnum(*ptr) || *ptr == '_' || *ptr == '-') + ++ptr; + const char *key_end = ptr; + // optional space (not in RFC, but in previous code) + if (*ptr == ' ') + ++ptr; + CHECK_AND_ASSERT_MES(*ptr == ':', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); + ++ptr; + // optional whitespace, but not newlines - line folding is obsolete, let's ignore it + while (isblank(*ptr)) + ++ptr; + const char *value_pos = ptr; + while (*ptr != '\r' && *ptr != '\n') + ++ptr; + const char *value_end = ptr; + // optional trailing whitespace + while (value_end > value_pos && isblank(*(value_end-1))) + --value_end; + if (*ptr == '\r') + ++ptr; + CHECK_AND_ASSERT_MES(*ptr == '\n', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); + ++ptr; + + const std::string key = std::string(key_pos, key_end - key_pos); + const std::string value = std::string(value_pos, value_end - value_pos); + if (!key.empty()) + { + if (!string_tools::compare_no_case(key, "Connection")) + body_info.m_connection = value; + else if(!string_tools::compare_no_case(key, "Referrer")) + body_info.m_referer = value; + else if(!string_tools::compare_no_case(key, "Content-Length")) + body_info.m_content_length = value; + else if(!string_tools::compare_no_case(key, "Content-Type")) + body_info.m_content_type = value; + else if(!string_tools::compare_no_case(key, "Transfer-Encoding")) + body_info.m_transfer_encoding = value; + else if(!string_tools::compare_no_case(key, "Content-Encoding")) + body_info.m_content_encoding = value; + else if(!string_tools::compare_no_case(key, "Host")) + body_info.m_host = value; + else if(!string_tools::compare_no_case(key, "Cookie")) + body_info.m_cookie = value; + else if(!string_tools::compare_no_case(key, "User-Agent")) + body_info.m_user_agent = value; + else if(!string_tools::compare_no_case(key, "Origin")) + body_info.m_origin = value; + else + body_info.m_etc_fields.emplace_back(key, value); } - else if(result[i++].matched)//"Cookie" - body_info.m_cookie = result[field_val]; - else if(result[i++].matched)//"User-Agent" - body_info.m_user_agent = result[field_val]; - else if(result[i++].matched)//"Origin" - body_info.m_origin = result[field_val]; - else if(result[i++].matched)//e.t.c (HAVE TO BE MATCHED!) - body_info.m_etc_fields.emplace_back(result[12], result[field_val]); - else - {CHECK_AND_ASSERT_MES(false, false, "http_stream_filter::parse_cached_header() not matched last entry in:"<(result[1]); - m_response_info.m_http_ver_lo = boost::lexical_cast(result[2]); - m_response_info.m_response_code = boost::lexical_cast(result[3]); - - m_header_cache.erase(to_nonsonst_iterator(m_header_cache, result[0].first), to_nonsonst_iterator(m_header_cache, result[0].second)); - return true; - }else - { - LOG_ERROR("http_stream_filter::handle_invoke_reply_line(): Failed to match first response line:" << m_header_cache); - return false; - } - + //First line response, look like this: "HTTP/1.1 200 OK" + const char *ptr = m_header_cache.c_str(); + CHECK_AND_ASSERT_MES(!memcmp(ptr, "HTTP/", 5), false, "Invalid first response line: " + m_header_cache); + ptr += 5; + CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + unsigned long ul; + char *end; + ul = strtoul(ptr, &end, 10); + CHECK_AND_ASSERT_MES(ul <= INT_MAX && *end =='.', false, "Invalid first response line: " + m_header_cache); + m_response_info.m_http_ver_hi = ul; + ptr = end + 1; + CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); + ul = strtoul(ptr, &end, 10); + CHECK_AND_ASSERT_MES(ul <= INT_MAX && isblank(*end), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); + m_response_info.m_http_ver_lo = ul; + ptr = end + 1; + while (isblank(*ptr)) + ++ptr; + CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + ul = strtoul(ptr, &end, 10); + CHECK_AND_ASSERT_MES(ul >= 100 && ul <= 999 && isspace(*end), false, "Invalid first response line: " + m_header_cache); + m_response_info.m_response_code = ul; + ptr = end; + // ignore the optional text, till the end + while (*ptr != '\r' && *ptr != '\n') + ++ptr; + if (*ptr == '\r') + ++ptr; + CHECK_AND_ASSERT_MES(*ptr == '\n', false, "Invalid first response line: " << m_header_cache); + ++ptr; + + m_header_cache.erase(0, ptr - m_header_cache.c_str()); + return true; } inline bool set_reply_content_encoder()