From 6b78b5c2be7e24cd4a1c59c7fa13a62c2ae33508 Mon Sep 17 00:00:00 2001 From: Alex Astashyn Date: Sat, 3 Dec 2016 19:05:09 -0500 Subject: [PATCH] Added strtonum for locale-independent number parsing --- src/json.hpp | 374 ++++++++++++++++++++++------------- src/json.hpp.re2c | 374 ++++++++++++++++++++++------------- test/src/unit-regression.cpp | 4 +- 3 files changed, 483 insertions(+), 269 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 6fed0a122..e28333bd4 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -9060,65 +9060,225 @@ basic_json_parser_66: return result; } + + + + + + + + + + + + + + + + + /*! - @brief parse floating point number + @brief parse string into a built-in arithmetic type as if + the current locale is POSIX. - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + e.g. const auto x = static_cast("123.4"); - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number + throw if can't parse the entire string as a number, + or if the destination type is integral and the value + is outside of the type's range. */ - long double str_to_float_t(long double* /* type */, char** endptr) const + struct strtonum { - return std::strtold(reinterpret_cast(m_start), endptr); - } + public: + strtonum(const char* start, const char* end) + : m_start(start), m_end(end) + {} - /*! - @brief parse floating point number + template::value>::type > + operator T() const + { + T val{0}; - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + if(parse(val, std::is_integral())) { + return val; + } - @param[in] type the @ref number_float_t in use + throw std::invalid_argument( + std::string() + + "Can't parse '" + + std::string(m_start, m_end) + + "' as type " + + typeid(T).name()); + } - @param[in,out] endptr recieves a pointer to the first character after - the number + /// return true iff token matches ^[+-]\d+$ + bool is_integral() const + { + const char* p = m_start; - @return the floating point number - */ - double str_to_float_t(double* /* type */, char** endptr) const - { - return std::strtod(reinterpret_cast(m_start), endptr); - } + if(!p) { + return false; + } - /*! - @brief parse floating point number + if(*p == '-' or *p == '+') { + ++p; + } - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + if(p == m_end) { + return false; + } - @param[in] type the @ref number_float_t in use + while(p < m_end and *p >= '0' and *p <= '9') { + ++p; + } - @param[in,out] endptr recieves a pointer to the first character after - the number + return p == m_end; + } - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } + private: + const char* const m_start = nullptr; + const char* const m_end = nullptr; + + static void strtof(float& f, + const char* str, + char** endptr) + { + f = std::strtof(str, endptr); + } + + static void strtof(double& f, + const char* str, + char** endptr) + { + f = std::strtod(str, endptr); + } + + static void strtof(long double& f, + const char* str, + char** endptr) + { + f = std::strtold(str, endptr); + } + + template + bool parse(T& value, /*is_integral=*/std::false_type) const + { + const char* data = m_start; + const size_t len = static_cast(m_end - m_start); + + const char decimal_point_char = + std::use_facet< std::numpunct >( + std::locale()).decimal_point(); + + // replace decimal separator with locale-specific + // version, if necessary; data will be repointed + // to either buf or tempstr containing the fixed + // string. + std::string tempstr; + std::array buf; + do { + if(decimal_point_char == '.') { + break; // don't need to convert + } + + const size_t ds_pos = static_cast( + std::find(m_start, m_end, '.') - m_start ); + + if(ds_pos == len) { + break; // no decimal separator + } + + // copy the data into the local buffer or + // tempstr, if buffer is too small; + // replace decimal separator, and update + // data to point to the modified bytes + if(len + 1 < buf.size()) { + std::copy(m_start, m_end, buf.data()); + buf[len] = 0; + buf[ds_pos] = decimal_point_char; + data = buf.data(); + } else { + tempstr.assign(m_start, m_end); + tempstr[ds_pos] = decimal_point_char; + data = tempstr.c_str(); + } + } while(0); + + char* endptr = nullptr; + value = 0; + strtof(value, data, &endptr); + + + + // note that reading past the end is OK, the data may be, + // for example, "123.", where the parsed token only contains "123", + // but strtod will read the dot as well. + const bool ok = endptr >= data + len + and len > 0; + + if(ok and value == 0.0 and *data == '-') { + // some implementations forget to negate the zero + value = -0.0; + } + + if(!ok) { + std::cerr << "data:" << data + << " token:" << std::string(m_start, len) + << " value:" << value + << " len:" << len + << " parsed_len:" << (endptr - data) << std::endl; + } + + return ok; + } + + template + bool parse(T& value, /*is_integral=*/std::true_type) const + { + const char* beg = m_start; + const char* const end = m_end; + + if(beg == end) { + return false; + } + + const bool is_negative = (*beg == '-'); + + // json numbers can't start with '+' but + // this code is not json-specific + if(is_negative or *beg == '+') { + ++beg; // skip the leading sign + } + + bool valid = beg < end // must have some digits; + and ( T(-1) < 0 // type must be signed + or !is_negative); // if value is negative + value = 0; + + while(beg < end and valid) { + const uint8_t c = static_cast(*beg - '0'); + const T upd_value = value * 10 + c; + valid &= value <= std::numeric_limits::max() / 10 + and value <= upd_value // did not overflow + and c < 10; // char was digit + value = upd_value; + ++beg; + } + + if(is_negative) { + value = 0 - value; + } + + if(!valid) { + std::cerr << " token:" << std::string(m_start, m_end) + << std::endl; + } + + return valid; + } + }; /*! @brief return number value for number tokens @@ -9126,111 +9286,57 @@ basic_json_parser_66: This function translates the last token into the most appropriate number type (either integer, unsigned integer or floating point), which is passed back to the caller via the result parameter. + + integral numbers that don't fit into the the range of the respective + type are parsed as number_float_t - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. + floating-point values do not satisfy std::isfinite predicate + are converted to value_t::null + + throws if the entire string [m_start .. m_cursor) cannot be + interpreted as a number - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). - - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. + @param[out] result @ref basic_json object to receive the number. */ void get_number(basic_json& result) const { assert(m_start != nullptr); + assert(m_start < m_cursor); + + strtonum num(reinterpret_cast(m_start), + reinterpret_cast(m_cursor)); - const lexer::lexer_char_t* curptr = m_start; + const bool is_negative = *m_start == '-'; - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; - - // maximum absolute value of the relevant integer type - number_unsigned_t max; - - // temporarily store the type to avoid unecessary bitfield access - value_t type; - - // look for sign - if (*curptr == '-') - { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; - } - else - { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); - } - - // count the significant figures - for (; curptr < m_cursor; curptr++) - { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') + try { + if(not num.is_integral()) { + ; + } + else if(is_negative) { - if (*curptr == '.') - { - // don't count '.' but change to float - type = value_t::number_float; - continue; - } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; - break; - } - - // skip if definitely not an integer - if (type != value_t::number_float) + result.m_type = value_t::number_integer; + result.m_value = static_cast(num); + return; + } + else { - // multiply last value by ten and add the new digit - auto temp = value * 10 + *curptr - '0'; - - // test for overflow - if (temp < value || temp > max) - { - // overflow - type = value_t::number_float; - } - else - { - // no overflow - save it - value = temp; - } + result.m_type = value_t::number_unsigned; + result.m_value = static_cast(num); + return; } + } catch (std::invalid_argument&) { + ; // overflow - will parse as double } - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - result.m_value.number_integer = -static_cast(value); - } - else - { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); + result.m_type = value_t::number_float; + result.m_value = static_cast(num); - // replace infinity and NAN by null - if (not std::isfinite(result.m_value.number_float)) - { - type = value_t::null; - result.m_value = basic_json::json_value(); - } + // replace infinity and NAN by null + if (not std::isfinite(result.m_value.number_float)) + { + result.m_type = value_t::null; + result.m_value = basic_json::json_value(); } - - // save the type - result.m_type = type; } private: diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index c6a99b89f..72c413ba7 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -8209,65 +8209,225 @@ class basic_json return result; } + + + + + + + + + + + + + + + + + /*! - @brief parse floating point number + @brief parse string into a built-in arithmetic type as if + the current locale is POSIX. - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + e.g. const auto x = static_cast("123.4"); - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number + throw if can't parse the entire string as a number, + or if the destination type is integral and the value + is outside of the type's range. */ - long double str_to_float_t(long double* /* type */, char** endptr) const + struct strtonum { - return std::strtold(reinterpret_cast(m_start), endptr); - } + public: + strtonum(const char* start, const char* end) + : m_start(start), m_end(end) + {} - /*! - @brief parse floating point number + template::value>::type > + operator T() const + { + T val{0}; - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + if(parse(val, std::is_integral())) { + return val; + } - @param[in] type the @ref number_float_t in use + throw std::invalid_argument( + std::string() + + "Can't parse '" + + std::string(m_start, m_end) + + "' as type " + + typeid(T).name()); + } - @param[in,out] endptr recieves a pointer to the first character after - the number + /// return true iff token matches ^[+-]\d+$ + bool is_integral() const + { + const char* p = m_start; - @return the floating point number - */ - double str_to_float_t(double* /* type */, char** endptr) const - { - return std::strtod(reinterpret_cast(m_start), endptr); - } + if(!p) { + return false; + } - /*! - @brief parse floating point number + if(*p == '-' or *p == '+') { + ++p; + } - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + if(p == m_end) { + return false; + } - @param[in] type the @ref number_float_t in use + while(p < m_end and *p >= '0' and *p <= '9') { + ++p; + } - @param[in,out] endptr recieves a pointer to the first character after - the number + return p == m_end; + } - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } + private: + const char* const m_start = nullptr; + const char* const m_end = nullptr; + + static void strtof(float& f, + const char* str, + char** endptr) + { + f = std::strtof(str, endptr); + } + + static void strtof(double& f, + const char* str, + char** endptr) + { + f = std::strtod(str, endptr); + } + + static void strtof(long double& f, + const char* str, + char** endptr) + { + f = std::strtold(str, endptr); + } + + template + bool parse(T& value, /*is_integral=*/std::false_type) const + { + const char* data = m_start; + const size_t len = static_cast(m_end - m_start); + + const char decimal_point_char = + std::use_facet< std::numpunct >( + std::locale()).decimal_point(); + + // replace decimal separator with locale-specific + // version, if necessary; data will be repointed + // to either buf or tempstr containing the fixed + // string. + std::string tempstr; + std::array buf; + do { + if(decimal_point_char == '.') { + break; // don't need to convert + } + + const size_t ds_pos = static_cast( + std::find(m_start, m_end, '.') - m_start ); + + if(ds_pos == len) { + break; // no decimal separator + } + + // copy the data into the local buffer or + // tempstr, if buffer is too small; + // replace decimal separator, and update + // data to point to the modified bytes + if(len + 1 < buf.size()) { + std::copy(m_start, m_end, buf.data()); + buf[len] = 0; + buf[ds_pos] = decimal_point_char; + data = buf.data(); + } else { + tempstr.assign(m_start, m_end); + tempstr[ds_pos] = decimal_point_char; + data = tempstr.c_str(); + } + } while(0); + + char* endptr = nullptr; + value = 0; + strtof(value, data, &endptr); + + + + // note that reading past the end is OK, the data may be, + // for example, "123.", where the parsed token only contains "123", + // but strtod will read the dot as well. + const bool ok = endptr >= data + len + and len > 0; + + if(ok and value == 0.0 and *data == '-') { + // some implementations forget to negate the zero + value = -0.0; + } + + if(!ok) { + std::cerr << "data:" << data + << " token:" << std::string(m_start, len) + << " value:" << value + << " len:" << len + << " parsed_len:" << (endptr - data) << std::endl; + } + + return ok; + } + + template + bool parse(T& value, /*is_integral=*/std::true_type) const + { + const char* beg = m_start; + const char* const end = m_end; + + if(beg == end) { + return false; + } + + const bool is_negative = (*beg == '-'); + + // json numbers can't start with '+' but + // this code is not json-specific + if(is_negative or *beg == '+') { + ++beg; // skip the leading sign + } + + bool valid = beg < end // must have some digits; + and ( T(-1) < 0 // type must be signed + or !is_negative); // if value is negative + value = 0; + + while(beg < end and valid) { + const uint8_t c = static_cast(*beg - '0'); + const T upd_value = value * 10 + c; + valid &= value <= std::numeric_limits::max() / 10 + and value <= upd_value // did not overflow + and c < 10; // char was digit + value = upd_value; + ++beg; + } + + if(is_negative) { + value = 0 - value; + } + + if(!valid) { + std::cerr << " token:" << std::string(m_start, m_end) + << std::endl; + } + + return valid; + } + }; /*! @brief return number value for number tokens @@ -8275,111 +8435,57 @@ class basic_json This function translates the last token into the most appropriate number type (either integer, unsigned integer or floating point), which is passed back to the caller via the result parameter. + + integral numbers that don't fit into the the range of the respective + type are parsed as number_float_t - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. + floating-point values do not satisfy std::isfinite predicate + are converted to value_t::null + + throws if the entire string [m_start .. m_cursor) cannot be + interpreted as a number - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). - - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. + @param[out] result @ref basic_json object to receive the number. */ void get_number(basic_json& result) const { assert(m_start != nullptr); + assert(m_start < m_cursor); + + strtonum num(reinterpret_cast(m_start), + reinterpret_cast(m_cursor)); - const lexer::lexer_char_t* curptr = m_start; + const bool is_negative = *m_start == '-'; - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; - - // maximum absolute value of the relevant integer type - number_unsigned_t max; - - // temporarily store the type to avoid unecessary bitfield access - value_t type; - - // look for sign - if (*curptr == '-') - { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; - } - else - { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); - } - - // count the significant figures - for (; curptr < m_cursor; curptr++) - { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') + try { + if(not num.is_integral()) { + ; + } + else if(is_negative) { - if (*curptr == '.') - { - // don't count '.' but change to float - type = value_t::number_float; - continue; - } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; - break; - } - - // skip if definitely not an integer - if (type != value_t::number_float) + result.m_type = value_t::number_integer; + result.m_value = static_cast(num); + return; + } + else { - // multiply last value by ten and add the new digit - auto temp = value * 10 + *curptr - '0'; - - // test for overflow - if (temp < value || temp > max) - { - // overflow - type = value_t::number_float; - } - else - { - // no overflow - save it - value = temp; - } + result.m_type = value_t::number_unsigned; + result.m_value = static_cast(num); + return; } + } catch (std::invalid_argument&) { + ; // overflow - will parse as double } - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - result.m_value.number_integer = -static_cast(value); - } - else - { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); + result.m_type = value_t::number_float; + result.m_value = static_cast(num); - // replace infinity and NAN by null - if (not std::isfinite(result.m_value.number_float)) - { - type = value_t::null; - result.m_value = basic_json::json_value(); - } + // replace infinity and NAN by null + if (not std::isfinite(result.m_value.number_float)) + { + result.m_type = value_t::null; + result.m_value = basic_json::json_value(); } - - // save the type - result.m_type = type; } private: diff --git a/test/src/unit-regression.cpp b/test/src/unit-regression.cpp index e04513caf..679b7f3d0 100644 --- a/test/src/unit-regression.cpp +++ b/test/src/unit-regression.cpp @@ -375,7 +375,7 @@ TEST_CASE("regression tests") }; // change locale to mess with decimal points - std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator)); + auto orig_locale = std::locale::global(std::locale(std::locale(), new CommaDecimalSeparator)); CHECK(j1a.dump() == "23.42"); CHECK(j1b.dump() == "23.42"); @@ -399,6 +399,8 @@ TEST_CASE("regression tests") CHECK(j3c.dump() == "10000"); //CHECK(j3b.dump() == "1E04"); // roundtrip error //CHECK(j3c.dump() == "1e04"); // roundtrip error + + std::locale::global(orig_locale); } SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator")