json_test.cpp

/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <cmath>
#include <limits>
#include <iostream>
#include <gtest/gtest.h>
#include <boost/next_prior.hpp>
#include "json.h"

//
// MSVC 2013 does not support unicode/raw string literals
//
#if __GNUC_PREREQ(4, 7)
TEST(Json, Unicode)
{
    auto val = parseJson("\"I \u2665 UTF-8\"");
    EXPECT_EQ("I \u2665 UTF-8", val.asString());
    val = parseJson("\"I \\u2665 UTF-8\"");
    EXPECT_EQ("I \u2665 UTF-8", val.asString());
    val = parseJson("\"I \U0001D11E playing in G-clef\"");
    EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());

    val = parseJson("\"I \\uD834\\uDD1E playing in G-clef\"");
    EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
}
#endif

TEST(Json, Parse)
{
    auto num = parseJson("12");
    EXPECT_TRUE(num.isInt());
    EXPECT_EQ(num, 12);
    num = parseJson("12e5");
    EXPECT_TRUE(num.isDouble());
    EXPECT_EQ(num, 12e5);
    auto numAs1 = num.asDouble();
    EXPECT_EQ(numAs1, 12e5);
    EXPECT_EQ(num, 12e5);
    EXPECT_EQ(num, 1200000);

    auto largeNumber = parseJson("4611686018427387904");
    EXPECT_TRUE(largeNumber.isInt());
    EXPECT_EQ(largeNumber, 4611686018427387904L);

    auto negative = parseJson("-123");
    EXPECT_EQ(negative, -123);

    auto bfalse = parseJson("false");
    auto btrue = parseJson("true");
    EXPECT_EQ(bfalse, false);
    EXPECT_EQ(btrue, true);

    auto null = parseJson("null");
    EXPECT_TRUE(null == nullptr);

    auto doub1 = parseJson("12.0");
    auto doub2 = parseJson("12e2");
    EXPECT_EQ(doub1, 12.0);
    EXPECT_EQ(doub2, 12e2);
    EXPECT_EQ(std::numeric_limits<double>::infinity(),
        parseJson("Infinity").asDouble());
    EXPECT_EQ(-std::numeric_limits<double>::infinity(),
        parseJson("-Infinity").asDouble());
    EXPECT_TRUE(std::isnan(parseJson("NaN").asDouble()));

    // case matters
    EXPECT_THROW(parseJson("infinity"), std::runtime_error);
    EXPECT_THROW(parseJson("inf"), std::runtime_error);
    EXPECT_THROW(parseJson("nan"), std::runtime_error);

    auto array = parseJson(
        "[12,false, false  , null , [12e4,32, [], 12]]");
    EXPECT_EQ(array.size(), 5);
    if (array.size() == 5)
    {
        EXPECT_EQ(boost::prior(array.end())->size(), 4);
    }

    EXPECT_THROW(parseJson("\n[12,\n\nnotvalidjson"),
        std::runtime_error);

    EXPECT_THROW(parseJson("12e2e2"),
        std::runtime_error);

    EXPECT_THROW(parseJson("{\"foo\":12,\"bar\":42} \"something\""),
        std::runtime_error);

    dynamic value = dynamic::object
        ("foo", "bar")
        ("junk", 12)
        ("another", 32.2)
        ("a",
        {
        dynamic::object("a", "b")
        ("c", "d"),
        12.5,
        "Yo Dawg",
        { "heh" },
        nullptr
    }
    )
        ;

    // Print then parse and get the same thing, hopefully.
    EXPECT_EQ(value, parseJson(toJson(value)));


    // Test an object with non-string values.
    dynamic something = parseJson(
        "{\"old_value\":40,\"changed\":true,\"opened\":false}");
    dynamic expected = dynamic::object
        ("old_value", 40)
        ("changed", true)
        ("opened", false);
    EXPECT_EQ(something, expected);
}

TEST(Json, ParseTrailingComma)
{
    json::serialization_opts on, off;
    on.allow_trailing_comma = true;
    off.allow_trailing_comma = false;

    dynamic arr{ 1, 2 };
    EXPECT_EQ(arr, parseJson("[1, 2]", on));
    EXPECT_EQ(arr, parseJson("[1, 2,]", on));
    EXPECT_EQ(arr, parseJson("[1, 2, ]", on));
    EXPECT_EQ(arr, parseJson("[1, 2 , ]", on));
    EXPECT_EQ(arr, parseJson("[1, 2 ,]", on));
    EXPECT_THROW(parseJson("[1, 2,]", off), std::runtime_error);

    dynamic obj = dynamic::object("a", 1);
    EXPECT_EQ(obj, parseJson("{\"a\": 1}", on));
    EXPECT_EQ(obj, parseJson("{\"a\": 1,}", on));
    EXPECT_EQ(obj, parseJson("{\"a\": 1, }", on));
    EXPECT_EQ(obj, parseJson("{\"a\": 1 , }", on));
    EXPECT_EQ(obj, parseJson("{\"a\": 1 ,}", on));
    EXPECT_THROW(parseJson("{\"a\":1,}", off), std::runtime_error);
}

TEST(Json, JavascriptSafe)
{
    auto badDouble = (1ll << 63ll) + 1;
    dynamic badDyn = badDouble;
    EXPECT_EQ(toJson(badDouble), to<std::string>(badDouble));
    json::serialization_opts opts;
    opts.javascript_safe = true;
    EXPECT_ANY_THROW(json::serialize(badDouble, opts));

    auto okDouble = 1ll << 63ll;
    dynamic okDyn = okDouble;
    EXPECT_EQ(toJson(okDouble), to<std::string>(okDouble));
}

#if __GNUC_PREREQ(4, 7)
TEST(Json, Produce) 
{
    auto value = parseJson(R"( "f\"oo" )");
    EXPECT_EQ(toJson(value), R"("f\"oo")");
    value = parseJson("\"Control code: \001 \002 \x1f\"");
    EXPECT_EQ(toJson(value), R"("Control code: \u0001 \u0002 \u001f")");

    // We're not allowed to have non-string keys in json.
    EXPECT_THROW(toJson(dynamic::object("abc", "xyz")(42.33, "asd")),
        std::runtime_error);
}

TEST(Json, JsonEscape)
{
    json::serialization_opts opts;
    EXPECT_EQ(
        json::serialize("\b\f\n\r\x01\t\\\"/\v\a", opts),
        R"("\b\f\n\r\u0001\t\\\"/\u000b\u0007")");
}
#endif

TEST(Json, JsonNonAsciiEncoding)
{
    json::serialization_opts opts;
    opts.encode_non_ascii = true;

    // simple tests
    EXPECT_EQ(json::serialize("\x1f", opts), R"("\u001f")");
    EXPECT_EQ(json::serialize("\xc2\xa2", opts), R"("\u00a2")");
    EXPECT_EQ(json::serialize("\xe2\x82\xac", opts), R"("\u20ac")");

#if __GNUC_PREREQ(4, 7)
    // multiple unicode encodings
    EXPECT_EQ(
        json::serialize("\x1f\xe2\x82\xac", opts),
        R"("\u001f\u20ac")");
    EXPECT_EQ(
        json::serialize("\x1f\xc2\xa2\xe2\x82\xac", opts),
        R"("\u001f\u00a2\u20ac")");
    EXPECT_EQ(
        json::serialize("\xc2\x80\xef\xbf\xbf", opts),
        R"("\u0080\uffff")");
    EXPECT_EQ(
        json::serialize("\xe0\xa0\x80\xdf\xbf", opts),
        R"("\u0800\u07ff")");

    // first possible sequence of a certain length
    EXPECT_EQ(json::serialize("\xc2\x80", opts), R"("\u0080")");
    EXPECT_EQ(json::serialize("\xe0\xa0\x80", opts), R"("\u0800")");

    // last possible sequence of a certain length
    EXPECT_EQ(json::serialize("\xdf\xbf", opts), R"("\u07ff")");
    EXPECT_EQ(json::serialize("\xef\xbf\xbf", opts), R"("\uffff")");

    // other boundary conditions
    EXPECT_EQ(json::serialize("\xed\x9f\xbf", opts), R"("\ud7ff")");
    EXPECT_EQ(json::serialize("\xee\x80\x80", opts), R"("\ue000")");
    EXPECT_EQ(json::serialize("\xef\xbf\xbd", opts), R"("\ufffd")");
#endif

    // incomplete sequences
    EXPECT_ANY_THROW(json::serialize("a\xed\x9f", opts));
    EXPECT_ANY_THROW(json::serialize("b\xee\x80", opts));
    EXPECT_ANY_THROW(json::serialize("c\xef\xbf", opts));

    // impossible bytes
    EXPECT_ANY_THROW(json::serialize("\xfe", opts));
    EXPECT_ANY_THROW(json::serialize("\xff", opts));

    // Sample overlong sequences
    EXPECT_ANY_THROW(json::serialize("\xc0\xaf", opts));
    EXPECT_ANY_THROW(json::serialize("\xe0\x80\xaf", opts));

    // Maximum overlong sequences
    EXPECT_ANY_THROW(json::serialize("\xc1\xbf", opts));
    EXPECT_ANY_THROW(json::serialize("\x30\x9f\xbf", opts));

    // illegal code positions
    EXPECT_ANY_THROW(json::serialize("\xed\xa0\x80", opts));
    EXPECT_ANY_THROW(json::serialize("\xed\xbf\xbf", opts));

    // Overlong representation of NUL character
    EXPECT_ANY_THROW(json::serialize("\xc0\x80", opts));
    EXPECT_ANY_THROW(json::serialize("\xe0\x80\x80", opts));

    // Longer than 3 byte encodings
    EXPECT_ANY_THROW(json::serialize("\xf4\x8f\xbf\xbf", opts));
    EXPECT_ANY_THROW(json::serialize("\xed\xaf\xbf\xed\xbf\xbf", opts));
}

TEST(Json, UTF8Retention)
{
    // test retention with valid utf8 strings
    std::string input = "\u2665";
    std::string jsonInput = toJson(input);
    std::string output = parseJson(jsonInput).asString();
    std::string jsonOutput = toJson(output);

    EXPECT_EQ(input, output);
    EXPECT_EQ(jsonInput, jsonOutput);

    // test retention with invalid utf8 - note that non-ascii chars are retained
    // as is, and no unicode encoding is attempted so no exception is thrown.
    EXPECT_EQ(
        toJson("a\xe0\xa0\x80z\xc0\x80"),
        "\"a\xe0\xa0\x80z\xc0\x80\""
        );
}

TEST(Json, UTF8EncodeNonAsciiRetention)
{

    json::serialization_opts opts;
    opts.encode_non_ascii = true;

    // test encode_non_ascii valid utf8 strings
    std::string input = "\u2665";
    std::string jsonInput = json::serialize(input, opts);
    std::string output = parseJson(jsonInput).asString();
    std::string jsonOutput = json::serialize(output, opts);

    EXPECT_EQ(input, output);
    EXPECT_EQ(jsonInput, jsonOutput);

    // test encode_non_ascii with invalid utf8 - note that an attempt to encode
    // non-ascii to unicode will result is a utf8 validation and throw exceptions.
    EXPECT_ANY_THROW(json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
    EXPECT_ANY_THROW(json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
}

#if __GNUC_PREREQ(4, 7)
TEST(Json, UTF8Validation)
{
    json::serialization_opts opts;
    opts.validate_utf8 = true;

    // test validate_utf8 valid utf8 strings - note that we only validate the
    // for utf8 but don't encode non-ascii to unicode so they are retained as is.
    EXPECT_EQ(json::serialize("a\xc2\x80z", opts), "\"a\xc2\x80z\"");
    EXPECT_EQ(
        json::serialize("a\xe0\xa0\x80z", opts),
        "\"a\xe0\xa0\x80z\"");
    EXPECT_EQ(
        json::serialize("a\xe0\xa0\x80m\xc2\x80z", opts),
        "\"a\xe0\xa0\x80m\xc2\x80z\"");

    // test validate_utf8 with invalid utf8
    EXPECT_ANY_THROW(json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
    EXPECT_ANY_THROW(json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));

    opts.skip_invalid_utf8 = true;
    EXPECT_EQ(json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
        "\"a\xe0\xa0\x80z\ufffd\ufffd\"");
    EXPECT_EQ(json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
        "\"a\xe0\xa0\x80z\ufffd\ufffd\ufffd\"");
    EXPECT_EQ(json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
        "\"z\ufffd\ufffdz\xe0\xa0\x80\"");

    opts.encode_non_ascii = true;
    EXPECT_EQ(json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
        "\"a\\u0800z\\ufffd\\ufffd\"");
    EXPECT_EQ(json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
        "\"a\\u0800z\\ufffd\\ufffd\\ufffd\"");
    EXPECT_EQ(json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
        "\"z\\ufffd\\ufffdz\\u0800\"");
}
#endif


TEST(Json, ParseNonStringKeys)
{
    // test string keys
    EXPECT_EQ("a", parseJson("{\"a\":[]}").items().begin()->first.asString());

    // check that we don't allow non-string keys as this violates the
    // strict JSON spec (though it is emitted by the output of
    // dynamic with operator <<).
    EXPECT_THROW(parseJson("{1:[]}"), std::runtime_error);

    // check that we can parse colloquial JSON if the option is set
    json::serialization_opts opts;
    opts.allow_non_string_keys = true;

    auto val = parseJson("{1:[]}", opts);
    EXPECT_EQ(1, val.items().begin()->first.asInt());


    // test we can still read in strings
    auto sval = parseJson("{\"a\":[]}", opts);
    EXPECT_EQ("a", sval.items().begin()->first.asString());

    // test we can read in doubles
    auto dval = parseJson("{1.5:[]}", opts);
    EXPECT_EQ(1.5, dval.items().begin()->first.asDouble());
}

TEST(Json, SortKeys)
{
    json::serialization_opts opts_on, opts_off;
    opts_on.sort_keys = true;
    opts_off.sort_keys = false;

    dynamic value = dynamic::object
        ("foo", "bar")
        ("junk", 12)
        ("another", 32.2)
        ("a",
        {
        dynamic::object("a", "b")
        ("c", "d"),
        12.5,
        "Yo Dawg",
        { "heh" },
        nullptr
    }
    )
        ;

    std::string sorted_keys =
        R"({"a":[{"a":"b","c":"d"},12.5,"Yo Dawg",["heh"],null],)"
        R"("another":32.2,"foo":"bar","junk":12})";

    EXPECT_EQ(value, parseJson(json::serialize(value, opts_on)));
    EXPECT_EQ(value, parseJson(json::serialize(value, opts_off)));

    EXPECT_EQ(sorted_keys, json::serialize(value, opts_on));
}