Merge branch 'develop' into feature/emplace

.gitignore

@@ -14,3 +14,7 @@ doc/html
 me.nlohmann.json.docset
 
 benchmarks/files/numbers/*.json
+
+.idea
+cmake-build-debug
+

.travis.yml

@@ -84,22 +84,22 @@ matrix:
   # Coverity (only for branch coverity_scan)
 
   - os: linux
-    compiler: gcc
+    compiler: clang
     before_install: echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-certificates.crt
     addons:
       apt:
         sources: ['ubuntu-toolchain-r-test']
-        packages: ['g++-5', 'valgrind']
+        packages: ['valgrind']
       coverity_scan:
         project:
           name: "nlohmann/json"
           description: "Build submitted via Travis CI"
         notification_email: niels.lohmann@gmail.com
-        build_command_prepend: "make clean ; sudo cp $(which g++-5) $(which g++)"
+        build_command_prepend: "make clean"
         build_command: "make"
         branch_pattern: coverity_scan
     env:
-      - COMPILER=g++-5
+      - LLVM_VERSION=3.6.0
       - SPECIAL=coverity
 
   # OSX / Clang

Makefile

@@ -95,7 +95,7 @@ pretty:
 # benchmarks
 json_benchmarks: benchmarks/benchmarks.cpp benchmarks/benchpress.hpp benchmarks/cxxopts.hpp src/json.hpp
 	cd benchmarks/files/numbers ; python generate.py
-	$(CXX) -std=c++11 $(CXXFLAGS) -DNDEBUG -O3 -flto -I src -I benchmarks $< $(LDFLAGS) -o $@
+	$(CXX) -std=c++11 -pthread $(CXXFLAGS) -DNDEBUG -O3 -flto -I src -I benchmarks $< $(LDFLAGS) -o $@
 	./json_benchmarks
 
 

README.md

@@ -3,6 +3,7 @@
 [![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json)
 [![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk/branch/develop?svg=true)](https://ci.appveyor.com/project/nlohmann/json)
 [![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json)
+[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json)
 [![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/fsf5FqYe6GoX68W6)
 [![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json)
 [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT)
@@ -499,6 +500,9 @@ I deeply appreciate the help of the following people.
 - [ChristophJud](https://github.com/ChristophJud) overworked the CMake files to ease project inclusion.
 - [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable.
 - [Denis Andrejew](https://github.com/seeekr) fixed a grammar issue in the README file.
+- [Pierre-Antoine Lacaze](https://github.com/palacaze) found a subtle bug in the `dump()` function.
+- [TurpentineDistillery](https://github.com/TurpentineDistillery) pointed to [`std::locale::classic()`](http://en.cppreference.com/w/cpp/locale/locale/classic) to avoid too much locale joggling.
+- [cgzones](https://github.com/cgzones) had an idea how to fix the Coverity scan.
 
 Thanks a lot for helping out!
 

benchmarks/benchmarks.cpp

@@ -1,107 +1,118 @@
 #define BENCHPRESS_CONFIG_MAIN
 
 #include <fstream>
+#include <sstream>
 #include <benchpress.hpp>
 #include <json.hpp>
+#include <pthread.h>
+#include <thread>
 
-BENCHMARK("parse jeopardy.json", [](benchpress::context* ctx)
+using json = nlohmann::json;
+
+struct StartUp
 {
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    StartUp()
     {
-        std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
+#ifndef __llvm__
-        nlohmann::json j;
+        // pin thread to a single CPU
-        j << input_file;
+        cpu_set_t cpuset;
+        pthread_t thread;
+        thread = pthread_self();
+        CPU_ZERO(&cpuset);
+        CPU_SET(std::thread::hardware_concurrency() - 1, &cpuset);
+        pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
+#endif
     }
-})
+};
+StartUp startup;
 
-BENCHMARK("parse canada.json", [](benchpress::context* ctx)
+enum class EMode { input, output_no_indent, output_with_indent };
+
+static void bench(benchpress::context& ctx,
+                  const std::string& in_path,
+                  const EMode mode)
 {
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    // using string streams for benchmarking to factor-out cold-cache disk
+    // access.
+    std::stringstream istr;
     {
-        std::ifstream input_file("benchmarks/files/nativejson-benchmark/canada.json");
+        // read file into string stream
-        nlohmann::json j;
+        std::ifstream input_file(in_path);
-        j << input_file;
+        istr << input_file.rdbuf();
-    }
+        input_file.close();
-})
 
-BENCHMARK("parse citm_catalog.json", [](benchpress::context* ctx)
+        // read the stream once
-{
+        json j;
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+        j << istr;
-    {
+        // clear flags and rewind
-        std::ifstream input_file("benchmarks/files/nativejson-benchmark/citm_catalog.json");
+        istr.clear();
-        nlohmann::json j;
+        istr.seekg(0);
-        j << input_file;
-    }
-})
-
-BENCHMARK("parse twitter.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        std::ifstream input_file("benchmarks/files/nativejson-benchmark/twitter.json");
-        nlohmann::json j;
-        j << input_file;
-    }
-})
-
-BENCHMARK("parse numbers/floats.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        std::ifstream input_file("benchmarks/files/numbers/floats.json");
-        nlohmann::json j;
-        j << input_file;
-    }
-})
-
-BENCHMARK("parse numbers/signed_ints.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        std::ifstream input_file("benchmarks/files/numbers/signed_ints.json");
-        nlohmann::json j;
-        j << input_file;
-    }
-})
-
-BENCHMARK("parse numbers/unsigned_ints.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        std::ifstream input_file("benchmarks/files/numbers/unsigned_ints.json");
-        nlohmann::json j;
-        j << input_file;
-    }
-})
-
-BENCHMARK("dump jeopardy.json", [](benchpress::context* ctx)
-{
-    std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
-    nlohmann::json j;
-    j << input_file;
-    std::ofstream output_file("jeopardy.dump.json");
-
-    ctx->reset_timer();
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        output_file << j;
     }
 
-    std::remove("jeopardy.dump.json");
+    switch (mode)
-})
-
-BENCHMARK("dump jeopardy.json with indent", [](benchpress::context* ctx)
-{
-    std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
-    nlohmann::json j;
-    j << input_file;
-    std::ofstream output_file("jeopardy.dump.json");
-
-    ctx->reset_timer();
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
-        output_file << std::setw(4) << j;
+        // benchmarking input
-    }
+        case EMode::input:
+        {
+            ctx.reset_timer();
 
-    std::remove("jeopardy.dump.json");
+            for (size_t i = 0; i < ctx.num_iterations(); ++i)
-})
+            {
+                // clear flags and rewind
+                istr.clear();
+                istr.seekg(0);
+                json j;
+                j << istr;
+            }
+
+            break;
+        }
+
+        // benchmarking output
+        case EMode::output_no_indent:
+        case EMode::output_with_indent:
+        {
+            // create JSON value from input
+            json j;
+            j << istr;
+            std::stringstream ostr;
+
+            ctx.reset_timer();
+            for (size_t i = 0; i < ctx.num_iterations(); ++i)
+            {
+                if (mode == EMode::output_no_indent)
+                {
+                    ostr << j;
+                }
+                else
+                {
+                    ostr << std::setw(4) << j;
+                }
+
+                // reset data
+                ostr.str(std::string());
+            }
+
+            break;
+        }
+    }
+}
+
+#define BENCHMARK_I(mode, title, in_path)           \
+    BENCHMARK((title), [](benchpress::context* ctx) \
+    {                                               \
+        bench(*ctx, (in_path), (mode));             \
+    })
+
+BENCHMARK_I(EMode::input, "parse jeopardy.json",              "benchmarks/files/jeopardy/jeopardy.json");
+BENCHMARK_I(EMode::input, "parse canada.json",                "benchmarks/files/nativejson-benchmark/canada.json");
+BENCHMARK_I(EMode::input, "parse citm_catalog.json",          "benchmarks/files/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_I(EMode::input, "parse twitter.json",               "benchmarks/files/nativejson-benchmark/twitter.json");
+BENCHMARK_I(EMode::input, "parse numbers/floats.json",        "benchmarks/files/numbers/floats.json");
+BENCHMARK_I(EMode::input, "parse numbers/signed_ints.json",   "benchmarks/files/numbers/signed_ints.json");
+BENCHMARK_I(EMode::input, "parse numbers/unsigned_ints.json", "benchmarks/files/numbers/unsigned_ints.json");
+
+BENCHMARK_I(EMode::output_no_indent,   "dump jeopardy.json",             "benchmarks/files/jeopardy/jeopardy.json");
+BENCHMARK_I(EMode::output_with_indent, "dump jeopardy.json with indent", "benchmarks/files/jeopardy/jeopardy.json");
+BENCHMARK_I(EMode::output_no_indent,   "dump numbers/floats.json",       "benchmarks/files/numbers/floats.json");
+BENCHMARK_I(EMode::output_no_indent,   "dump numbers/signed_ints.json",  "benchmarks/files/numbers/signed_ints.json");

src/json.hpp

@@ -45,7 +45,7 @@ SOFTWARE.
 #include <iostream> // istream, ostream
 #include <iterator> // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
 #include <limits> // numeric_limits
-#include <locale> // locale, numpunct
+#include <locale> // locale
 #include <map> // map
 #include <memory> // addressof, allocator, allocator_traits, unique_ptr
 #include <numeric> // accumulate
@@ -122,26 +122,6 @@ struct has_mapped_type
         std::is_integral<decltype(detect(std::declval<T>()))>::value;
 };
 
-/*!
-@brief helper class to create locales with decimal point
-
-This struct is used a default locale during the JSON serialization. JSON
-requires the decimal point to be `.`, so this function overloads the
-`do_decimal_point()` function to return `.`. This function is called by
-float-to-string conversions to retrieve the decimal separator between integer
-and fractional parts.
-
-@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315
-@since version 2.0.0
-*/
-struct DecimalSeparator : std::numpunct<char>
-{
-    char do_decimal_point() const
-    {
-        return '.';
-    }
-};
-
 }
 
 /*!
@@ -2201,8 +2181,7 @@ class basic_json
     {
         std::stringstream ss;
         // fix locale problems
-        const static std::locale loc(std::locale(), new DecimalSeparator);
+        ss.imbue(std::locale::classic());
-        ss.imbue(loc);
 
         // 6, 15 or 16 digits of precision allows round-trip IEEE 754
         // string->float->string, string->double->string or string->long
@@ -5913,7 +5892,7 @@ class basic_json
         o.width(0);
 
         // fix locale problems
-        const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator));
+        const auto old_locale = o.imbue(std::locale::classic());
         // set precision
 
         // 6, 15 or 16 digits of precision allows round-trip IEEE 754
@@ -7702,6 +7681,12 @@ class basic_json
         explicit lexer(std::istream& s)
             : m_stream(&s), m_line_buffer()
         {
+            // immediately abort if stream is erroneous
+            if (s.fail())
+            {
+                throw std::invalid_argument("stream error: " +  std::string(strerror(errno)));
+            }
+
             // fill buffer
             fill_line_buffer();
 
@@ -8841,6 +8826,7 @@ basic_json_parser_66:
                     m_line_buffer.clear();
                     for (m_cursor = m_start; m_cursor != m_limit; ++m_cursor)
                     {
+                        assert(m_cursor != nullptr);
                         m_line_buffer.append(1, static_cast<const char>(*m_cursor));
                     }
                 }
@@ -8848,7 +8834,10 @@ basic_json_parser_66:
                 // append n characters to make sure that there is sufficient
                 // space between m_cursor and m_limit
                 m_line_buffer.append(1, '\x00');
-                m_line_buffer.append(n - 1, '\x01');
+                if (n > 0)
+                {
+                    m_line_buffer.append(n - 1, '\x01');
+                }
             }
             else
             {

src/json.hpp.re2c

@@ -45,7 +45,7 @@ SOFTWARE.
 #include <iostream> // istream, ostream
 #include <iterator> // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
 #include <limits> // numeric_limits
-#include <locale> // locale, numpunct
+#include <locale> // locale
 #include <map> // map
 #include <memory> // addressof, allocator, allocator_traits, unique_ptr
 #include <numeric> // accumulate
@@ -122,26 +122,6 @@ struct has_mapped_type
         std::is_integral<decltype(detect(std::declval<T>()))>::value;
 };
 
-/*!
-@brief helper class to create locales with decimal point
-
-This struct is used a default locale during the JSON serialization. JSON
-requires the decimal point to be `.`, so this function overloads the
-`do_decimal_point()` function to return `.`. This function is called by
-float-to-string conversions to retrieve the decimal separator between integer
-and fractional parts.
-
-@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315
-@since version 2.0.0
-*/
-struct DecimalSeparator : std::numpunct<char>
-{
-    char do_decimal_point() const
-    {
-        return '.';
-    }
-};
-
 }
 
 /*!
@@ -2201,8 +2181,7 @@ class basic_json
     {
         std::stringstream ss;
         // fix locale problems
-        const static std::locale loc(std::locale(), new DecimalSeparator);
+        ss.imbue(std::locale::classic());
-        ss.imbue(loc);
 
         // 6, 15 or 16 digits of precision allows round-trip IEEE 754
         // string->float->string, string->double->string or string->long
@@ -5913,7 +5892,7 @@ class basic_json
         o.width(0);
 
         // fix locale problems
-        const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator));
+        const auto old_locale = o.imbue(std::locale::classic());
         // set precision
 
         // 6, 15 or 16 digits of precision allows round-trip IEEE 754
@@ -7702,6 +7681,12 @@ class basic_json
         explicit lexer(std::istream& s)
             : m_stream(&s), m_line_buffer()
         {
+            // immediately abort if stream is erroneous
+            if (s.fail())
+            {
+                throw std::invalid_argument("stream error: " +  std::string(strerror(errno)));
+            }
+
             // fill buffer
             fill_line_buffer();
 
@@ -7990,6 +7975,7 @@ class basic_json
                     m_line_buffer.clear();
                     for (m_cursor = m_start; m_cursor != m_limit; ++m_cursor)
                     {
+                        assert(m_cursor != nullptr);
                         m_line_buffer.append(1, static_cast<const char>(*m_cursor));
                     }
                 }
@@ -7997,7 +7983,10 @@ class basic_json
                 // append n characters to make sure that there is sufficient
                 // space between m_cursor and m_limit
                 m_line_buffer.append(1, '\x00');
-                m_line_buffer.append(n - 1, '\x01');
+                if (n > 0)
+                {
+                    m_line_buffer.append(n - 1, '\x01');
+                }
             }
             else
             {

test/src/unit-regression.cpp

@@ -495,4 +495,25 @@ TEST_CASE("regression tests")
         json j = json::parse("22e2222");
         CHECK(j == json());
     }
+
+    SECTION("issue #366 - json::parse on failed stream gets stuck")
+    {
+        std::ifstream f("file_not_found.json");
+        CHECK_THROWS_AS(json::parse(f), std::invalid_argument);
+    }
+
+    SECTION("issue #367 - calling stream at EOF")
+    {
+        std::stringstream ss;
+        json j;
+        ss << "123";
+        CHECK_NOTHROW(j << ss);
+
+        // see https://github.com/nlohmann/json/issues/367#issuecomment-262841893:
+        // ss is not at EOF; this yielded an error before the fix
+        // (threw basic_string::append). No, it should just throw
+        // a parse error because of the EOF.
+        CHECK_THROWS_AS(j << ss, std::invalid_argument);
+        CHECK_THROWS_WITH(j << ss, "parse error - unexpected end of input");
+    }
 }