From 44a8e9bf35c83b760c2ca4bb13df05474dd833cb Mon Sep 17 00:00:00 2001 From: Niels Date: Sun, 3 Apr 2016 16:33:30 +0200 Subject: [PATCH 1/5] locale-independent dump --- src/json.hpp | 10 ++++++++++ src/json.hpp.re2c | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/json.hpp b/src/json.hpp index 0594b38d7..8fb400c70 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -2148,6 +2148,7 @@ class basic_json string_t dump(const int indent = -1) const { std::stringstream ss; + ss.imbue(std::locale("C")); if (indent >= 0) { @@ -5653,11 +5654,20 @@ class basic_json const bool pretty_print = (o.width() > 0); const auto indentation = (pretty_print ? o.width() : 0); + // save locale of o + auto old_locale = o.getloc(); + // set locale of o to "C" + o.imbue(std::locale("C")); + // reset width to 0 for subsequent calls to this stream o.width(0); // do the actual serialization j.dump(o, pretty_print, static_cast(indentation)); + + // reset locale + o.imbue(old_locale); + return o; } diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index f4ddacf71..2422ef8b0 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -2148,6 +2148,7 @@ class basic_json string_t dump(const int indent = -1) const { std::stringstream ss; + ss.imbue(std::locale("C")); if (indent >= 0) { @@ -5653,11 +5654,20 @@ class basic_json const bool pretty_print = (o.width() > 0); const auto indentation = (pretty_print ? o.width() : 0); + // save locale of o + auto old_locale = o.getloc(); + // set locale of o to "C" + o.imbue(std::locale("C")); + // reset width to 0 for subsequent calls to this stream o.width(0); // do the actual serialization j.dump(o, pretty_print, static_cast(indentation)); + + // reset locale + o.imbue(old_locale); + return o; } From 43ee70cef8aedb43956d3ba66ec39ec0dfee9722 Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 5 Apr 2016 20:47:47 +0200 Subject: [PATCH 2/5] revert changes to master --- src/json.hpp | 10 ---------- src/json.hpp.re2c | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 8fb400c70..0594b38d7 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -2148,7 +2148,6 @@ class basic_json string_t dump(const int indent = -1) const { std::stringstream ss; - ss.imbue(std::locale("C")); if (indent >= 0) { @@ -5654,20 +5653,11 @@ class basic_json const bool pretty_print = (o.width() > 0); const auto indentation = (pretty_print ? o.width() : 0); - // save locale of o - auto old_locale = o.getloc(); - // set locale of o to "C" - o.imbue(std::locale("C")); - // reset width to 0 for subsequent calls to this stream o.width(0); // do the actual serialization j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale - o.imbue(old_locale); - return o; } diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 2422ef8b0..f4ddacf71 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -2148,7 +2148,6 @@ class basic_json string_t dump(const int indent = -1) const { std::stringstream ss; - ss.imbue(std::locale("C")); if (indent >= 0) { @@ -5654,20 +5653,11 @@ class basic_json const bool pretty_print = (o.width() > 0); const auto indentation = (pretty_print ? o.width() : 0); - // save locale of o - auto old_locale = o.getloc(); - // set locale of o to "C" - o.imbue(std::locale("C")); - // reset width to 0 for subsequent calls to this stream o.width(0); // do the actual serialization j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale - o.imbue(old_locale); - return o; } From 31bccc83b946f0dd2b8cbfe5644d552b41996121 Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 5 Apr 2016 21:55:51 +0200 Subject: [PATCH 3/5] fixed locale problems --- src/json.hpp | 64 ++++++++++++++++++++++++----------------------- src/json.hpp.re2c | 64 ++++++++++++++++++++++++----------------------- test/unit.cpp | 27 ++++++++++++++++++++ 3 files changed, 93 insertions(+), 62 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 0594b38d7..13a6c49a9 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -6114,24 +6114,26 @@ class basic_json case value_t::number_float: { - // buffer size: precision (2^8-1 = 255) + other ('-.e-xxx' = 7) + null (1) - char buf[263]; - int len; - // check if number was parsed from a string if (m_type.bits.parsed) { // check if parsed number had an exponent given if (m_type.bits.has_exp) { + // buffer size: precision (2^8-1 = 255) + other ('-.e-xxx' = 7) + null (1) + char buf[263]; + int len; + // handle capitalization of the exponent if (m_type.bits.exp_cap) { - len = snprintf(buf, sizeof(buf), "%.*E", m_type.bits.precision, m_value.number_float) + 1; + len = snprintf(buf, sizeof(buf), "%.*E", + m_type.bits.precision, m_value.number_float) + 1; } else { - len = snprintf(buf, sizeof(buf), "%.*e", m_type.bits.precision, m_value.number_float) + 1; + len = snprintf(buf, sizeof(buf), "%.*e", + m_type.bits.precision, m_value.number_float) + 1; } // remove '+' sign from the exponent if necessary @@ -6152,40 +6154,40 @@ class basic_json } } } + + o << buf; } else { // no exponent - output as a decimal - snprintf(buf, sizeof(buf), "%.*f", - m_type.bits.precision, m_value.number_float); + std::stringstream ss; + ss.imbue(std::locale("C")); + ss << std::setprecision(m_type.bits.precision) + << std::fixed << m_value.number_float; + o << ss.str(); } } - else if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - if (std::signbit(m_value.number_float)) - { - o << "-0.0"; - } - else - { - o << "0.0"; - } - return; - } else { - // Otherwise 6, 15 or 16 digits of precision allows - // round-trip IEEE 754 string->float->string, - // string->double->string or string->long double->string; - // to be safe, we read this value from - // std::numeric_limits::digits10 - snprintf(buf, sizeof(buf), "%.*g", - std::numeric_limits::digits10, - m_value.number_float); + if (m_value.number_float == 0) + { + // special case for zero to get "0.0"/"-0.0" + o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); + } + else + { + // Otherwise 6, 15 or 16 digits of precision allows + // round-trip IEEE 754 string->float->string, + // string->double->string or string->long double->string; + // to be safe, we read this value from + // std::numeric_limits::digits10 + std::stringstream ss; + ss.imbue(std::locale("C")); + ss << std::setprecision(std::numeric_limits::digits10) + << m_value.number_float; + o << ss.str(); + } } - - o << buf; return; } diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index f4ddacf71..70079144f 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -6114,24 +6114,26 @@ class basic_json case value_t::number_float: { - // buffer size: precision (2^8-1 = 255) + other ('-.e-xxx' = 7) + null (1) - char buf[263]; - int len; - // check if number was parsed from a string if (m_type.bits.parsed) { // check if parsed number had an exponent given if (m_type.bits.has_exp) { + // buffer size: precision (2^8-1 = 255) + other ('-.e-xxx' = 7) + null (1) + char buf[263]; + int len; + // handle capitalization of the exponent if (m_type.bits.exp_cap) { - len = snprintf(buf, sizeof(buf), "%.*E", m_type.bits.precision, m_value.number_float) + 1; + len = snprintf(buf, sizeof(buf), "%.*E", + m_type.bits.precision, m_value.number_float) + 1; } else { - len = snprintf(buf, sizeof(buf), "%.*e", m_type.bits.precision, m_value.number_float) + 1; + len = snprintf(buf, sizeof(buf), "%.*e", + m_type.bits.precision, m_value.number_float) + 1; } // remove '+' sign from the exponent if necessary @@ -6152,40 +6154,40 @@ class basic_json } } } + + o << buf; } else { // no exponent - output as a decimal - snprintf(buf, sizeof(buf), "%.*f", - m_type.bits.precision, m_value.number_float); + std::stringstream ss; + ss.imbue(std::locale("C")); // fix locale problems + ss << std::setprecision(m_type.bits.precision) + << std::fixed << m_value.number_float; + o << ss.str(); } } - else if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - if (std::signbit(m_value.number_float)) - { - o << "-0.0"; - } - else - { - o << "0.0"; - } - return; - } else { - // Otherwise 6, 15 or 16 digits of precision allows - // round-trip IEEE 754 string->float->string, - // string->double->string or string->long double->string; - // to be safe, we read this value from - // std::numeric_limits::digits10 - snprintf(buf, sizeof(buf), "%.*g", - std::numeric_limits::digits10, - m_value.number_float); + if (m_value.number_float == 0) + { + // special case for zero to get "0.0"/"-0.0" + o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); + } + else + { + // Otherwise 6, 15 or 16 digits of precision allows + // round-trip IEEE 754 string->float->string, + // string->double->string or string->long double->string; + // to be safe, we read this value from + // std::numeric_limits::digits10 + std::stringstream ss; + ss.imbue(std::locale("C")); // fix locale problems + ss << std::setprecision(std::numeric_limits::digits10) + << m_value.number_float; + o << ss.str(); + } } - - o << buf; return; } diff --git a/test/unit.cpp b/test/unit.cpp index b440a28bc..85f8b2294 100644 --- a/test/unit.cpp +++ b/test/unit.cpp @@ -12369,5 +12369,32 @@ TEST_CASE("regression tests") j_long_double = 1.23e45L; CHECK(j_long_double.get() == 1.23e45L); } + + SECTION("issue #228 - double values are serialized with commas as decimal points") + { + json j1a = 23.42; + json j1b = json::parse("23.42"); + + json j2a = 2342e-2; + //issue #230 + //json j2b = json::parse("2342e-2"); + + json j3a = 10E3; + json j3b = json::parse("10E3"); + json j3c = json::parse("10e3"); + + std::locale::global(std::locale("de_DE")); + + CHECK(j1a.dump() == "23.42"); + CHECK(j1b.dump() == "23.42"); + + CHECK(j2a.dump() == "23.42"); + //issue #230 + //CHECK(j2b.dump() == "23.42"); + + CHECK(j3a.dump() == "10000"); + CHECK(j3b.dump() == "1E04"); + CHECK(j3c.dump() == "1e04"); + } } From 0b60d970e916540ccc9abc4706c2621e174a0481 Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 5 Apr 2016 22:12:12 +0200 Subject: [PATCH 4/5] make code independent of concrete locale --- src/json.hpp | 17 +++++++++++++++-- src/json.hpp.re2c | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 13a6c49a9..8f671fb01 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -88,6 +88,19 @@ struct has_mapped_type static constexpr bool value = sizeof(test(0)) == 1; }; +/*! +@brief helper class to create locales with decimal point +@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 +*/ +class DecimalSeparator : public std::numpunct +{ + protected: + char do_decimal_point() const + { + return '.'; + } +}; + } /*! @@ -6161,7 +6174,7 @@ class basic_json { // no exponent - output as a decimal std::stringstream ss; - ss.imbue(std::locale("C")); + ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems ss << std::setprecision(m_type.bits.precision) << std::fixed << m_value.number_float; o << ss.str(); @@ -6182,7 +6195,7 @@ class basic_json // to be safe, we read this value from // std::numeric_limits::digits10 std::stringstream ss; - ss.imbue(std::locale("C")); + ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems ss << std::setprecision(std::numeric_limits::digits10) << m_value.number_float; o << ss.str(); diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 70079144f..ebf83d831 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -88,6 +88,19 @@ struct has_mapped_type static constexpr bool value = sizeof(test(0)) == 1; }; +/*! +@brief helper class to create locales with decimal point +@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 +*/ +class DecimalSeparator : public std::numpunct +{ + protected: + char do_decimal_point() const + { + return '.'; + } +}; + } /*! @@ -6161,7 +6174,7 @@ class basic_json { // no exponent - output as a decimal std::stringstream ss; - ss.imbue(std::locale("C")); // fix locale problems + ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems ss << std::setprecision(m_type.bits.precision) << std::fixed << m_value.number_float; o << ss.str(); @@ -6182,7 +6195,7 @@ class basic_json // to be safe, we read this value from // std::numeric_limits::digits10 std::stringstream ss; - ss.imbue(std::locale("C")); // fix locale problems + ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems ss << std::setprecision(std::numeric_limits::digits10) << m_value.number_float; o << ss.str(); From a744c62696d1631a1d5ecc7763536289d8dec3e1 Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 5 Apr 2016 22:28:27 +0200 Subject: [PATCH 5/5] made tests independent of "C" locale --- test/unit.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/unit.cpp b/test/unit.cpp index 85f8b2294..ab96364c8 100644 --- a/test/unit.cpp +++ b/test/unit.cpp @@ -12383,7 +12383,18 @@ TEST_CASE("regression tests") json j3b = json::parse("10E3"); json j3c = json::parse("10e3"); - std::locale::global(std::locale("de_DE")); + // class to create a locale that would use a comma for decimals + class CommaDecimalSeparator : public std::numpunct + { + protected: + char do_decimal_point() const + { + return ','; + } + }; + + // change locale to mess with decimal points + std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator)); CHECK(j1a.dump() == "23.42"); CHECK(j1b.dump() == "23.42");