From 5039f1d7bcb053a5a41770bba47236def34ff772 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Tue, 1 Sep 2026 17:49:07 +0400 Subject: [PATCH 01/12] Handle `null` values in JSON parsing for array columns and update test cases. --- .../kotlinx/dataframe/impl/io/readJson.kt | 18 +- .../kotlinx/dataframe/impl/io/writeJson.kt | 56 +++++-- .../jetbrains/kotlinx/dataframe/io/json.kt | 16 +- .../kotlinx/dataframe/io/JsonTests.kt | 156 +++++++++++++++--- .../io/convertDataRowsWithOpenApi.kt | 12 ++ .../resources/api/read/readJsonTypeClash.html | 2 +- 6 files changed, 213 insertions(+), 47 deletions(-) diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt index 74de9fa296..89d986cc76 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt @@ -308,10 +308,15 @@ internal fun fromJsonListAnyColumns( .asList() .splitByIndices(startIndices.asSequence()) .toList() + // records that hold no array at all (like `null`) are not empty lists + .mapIndexed { i, list -> if (records[i] is JsonArray) list else null } DataColumn.createValueColumn( name = ARRAY_COLUMN_NAME, values = columnValues, - type = List::class.createType(listOf(KTypeProjection.invariant(elementType))), + type = List::class.createType( + arguments = listOf(KTypeProjection.invariant(elementType)), + nullable = records.any { it !is JsonArray }, + ), ) } @@ -478,9 +483,9 @@ internal fun fromJsonListArrayAndValueColumns( // list element type can be JsonObject, JsonArray or primitive // So first, we gather all properties of objects to merge including "array" and "value" if needed // so the resulting type of a property with instances 123, ["abc"], and { "a": 1, "b": 2 } will be - // { array: List, value: Int?, a: Int?, b: Int? } + // { array: List?, value: Int?, a: Int?, b: Int? } // and instances will look like - // { "array": [], "value": 123, "a": null, "b": null } + // { "array": null, "value": 123, "a": null, "b": null } val nameGenerator = ColumnNameGenerator() records.forEach { record -> @@ -668,10 +673,15 @@ internal fun fromJsonListArrayAndValueColumns( .asList() .splitByIndices(startIndices.asSequence()) .toList() + // records that hold no array at all (like `null`) are not empty lists + .mapIndexed { i, list -> if (records[i] is JsonArray) list else null } DataColumn.createValueColumn( name = colName, values = columnValues, - type = List::class.createType(listOf(KTypeProjection.invariant(elementType))), + type = List::class.createType( + arguments = listOf(KTypeProjection.invariant(elementType)), + nullable = records.any { it !is JsonArray }, + ), ) } diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt index ee2024add4..c889e37b7b 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt @@ -20,9 +20,10 @@ import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.ColumnsContainer import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.api.FormattedFrame +import org.jetbrains.kotlinx.dataframe.api.dataFrameOf import org.jetbrains.kotlinx.dataframe.api.indices +import org.jetbrains.kotlinx.dataframe.api.isEmpty import org.jetbrains.kotlinx.dataframe.api.isList -import org.jetbrains.kotlinx.dataframe.api.rows import org.jetbrains.kotlinx.dataframe.api.schema import org.jetbrains.kotlinx.dataframe.api.take import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup @@ -39,6 +40,7 @@ import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NROW import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.TYPE import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.TYPES import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.VERSION +import org.jetbrains.kotlinx.dataframe.indices import org.jetbrains.kotlinx.dataframe.io.ARRAY_COLUMN_NAME import org.jetbrains.kotlinx.dataframe.io.Base64ImageEncodingOptions import org.jetbrains.kotlinx.dataframe.io.CustomEncoder @@ -291,11 +293,11 @@ internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { null } else { // check that value in this column is not null only when other values are null - val isValidValueColumn = rows().all { row -> - if (valueCol[row] != null) { + val isValidValueColumn = indices().all { row -> + if (!valueCol.isNullAt(row)) { allColumns.all { col -> if (col.name != valueCol.name) { - col[row] == null + col.isNullAt(row) } else { true } @@ -304,7 +306,7 @@ internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { true } } - if (isValidValueColumn) { + if (isValidValueColumn && valueCol.holdsAnyValue()) { valueCol } else { null @@ -319,6 +321,11 @@ internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { internal val AnyFrame.isPossibleToFindUnnamedColumns: Boolean get() = columns().size != 1 +// An unnamed `value`/`array` column always holds at least one value, as it's only created for +// records that actually hold a value or an array. +// A column of `null`s only was created by the user, like dataFrameOf("value" to listOf(null, null)). +private fun AnyCol.holdsAnyValue(): Boolean = indices.any { !isNullAt(it) } + internal fun AnyFrame.extractArrayColumn(): DataColumn<*>? { val allColumns = columns() @@ -330,11 +337,11 @@ internal fun AnyFrame.extractArrayColumn(): DataColumn<*>? { null } else { // check that value in this column is not null only when other values are null - val isValidArrayColumn = rows().all { row -> - if (arrayCol[row] != null) { + val isValidArrayColumn = indices().all { row -> + if (!arrayCol.isNullAt(row)) { allColumns.all { col -> if (col.name != arrayCol.name) { - col[row] == null + col.isNullAt(row) } else { true } @@ -343,7 +350,7 @@ internal fun AnyFrame.extractArrayColumn(): DataColumn<*>? { true } } - if (isValidArrayColumn) { + if (isValidArrayColumn && arrayCol.holdsAnyValue()) { arrayCol } else { null @@ -358,25 +365,44 @@ internal fun encodeFrame(frame: AnyFrame): JsonArray { val arraysAreFrames = arrayColumn?.kind() == ColumnKind.Frame + // the unnamed `value`/`array` columns encode the record itself, all other columns encode it as an object + val objectColumns = frame.columns().filter { it.name != valueColumn?.name && it.name != arrayColumn?.name } + val hasUnnamedColumns = objectColumns.size != frame.columnsCount() + val objectFrame = dataFrameOf(objectColumns) + val data = frame.indices().map { rowIndex -> when { - valueColumn != null -> valueColumn[rowIndex] + valueColumn?.isNullAt(rowIndex) == false -> valueColumn[rowIndex] - arrayColumn != null -> arrayColumn[rowIndex]?.let { + arrayColumn?.isNullAt(rowIndex) == false -> if (arraysAreFrames) { - encodeFrame(it as AnyFrame) + encodeFrame(arrayColumn[rowIndex] as AnyFrame) } else { - null + JsonArray((arrayColumn[rowIndex] as List<*>).map { convert(it) }) } - } - else -> encodeRow(frame, rowIndex) + // a record that's neither a single value nor an array is encoded as an object, + // unless it holds no values at all; then the record was `null` itself + hasUnnamedColumns && objectColumns.all { it.isNullAt(rowIndex) } -> null + + else -> encodeRow(objectFrame, rowIndex) } } return buildJsonArray { addAll(data.map { convert(it) }) } } +/** + * Whether the value of this column at [rowIndex] carries no data; + * empty [FrameColumn] values and [ColumnGroup] rows filled with `null`s count as no data too. + */ +private fun AnyCol.isNullAt(rowIndex: Int): Boolean = + when (this) { + is ColumnGroup<*> -> columns().all { it.isNullAt(rowIndex) } + is FrameColumn<*> -> this[rowIndex].isEmpty() + else -> this[rowIndex] == null + } + internal fun encodeDataFrameWithMetadata( frame: AnyFrame, rowLimit: Int, diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt index bfa885eb8b..baf6c9d158 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt @@ -103,15 +103,15 @@ public class JSON( * ] * ``` * - * [ARRAY_AND_VALUE_COLUMNS] (default) will create a [DataFrame] looking like (including `null` and `[]` values): + * [ARRAY_AND_VALUE_COLUMNS] (default) will create a [DataFrame] looking like (including `null` values): * ``` - * ⌌----------------------------------------------⌍ - * | | a:{b:Int?, value:String?, array:List}| - * |--|-------------------------------------------| - * | 0| { b:null, value:"text", array:[] }| - * | 1| { b:2, value:null, array:[] }| - * | 2| { b:null, value:null, array:[6, 7, 8] }| - * ⌎----------------------------------------------⌏ + * ⌌-----------------------------------------------⌍ + * | | a:{b:Int?, value:String?, array:List?}| + * |--|--------------------------------------------| + * | 0| { b:null, value:"text", array:null }| + * | 1| { b:2, value:null, array:null }| + * | 2| { b:null, value:null, array:[6, 7, 8] }| + * ⌎-----------------------------------------------⌏ * ``` * So, for the type clashing argument it will create a [ColumnGroup] with the properties `value`, `array`, * and the unwrapped properties of the objects the property can be. diff --git a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt index 073f77f529..e4daa98ee7 100644 --- a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt +++ b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt @@ -32,6 +32,7 @@ import org.jetbrains.kotlinx.dataframe.api.convert import org.jetbrains.kotlinx.dataframe.api.dataFrameOf import org.jetbrains.kotlinx.dataframe.api.forEach import org.jetbrains.kotlinx.dataframe.api.format +import org.jetbrains.kotlinx.dataframe.api.getColumn import org.jetbrains.kotlinx.dataframe.api.getColumnGroup import org.jetbrains.kotlinx.dataframe.api.getColumns import org.jetbrains.kotlinx.dataframe.api.getFrameColumn @@ -169,7 +170,8 @@ class JsonTests { group.columnsCount() shouldBe 3 group["b"].type() shouldBe typeOf() group["value"].type() shouldBe typeOf() - group["array"].type() shouldBe typeOf>() + group["array"].type() shouldBe typeOf?>() + group["array"].values().toList() shouldBe listOf(null, null, listOf(6, 7, 8)) } @Test @@ -207,8 +209,8 @@ class JsonTests { val df = DataFrame.readJsonStr(json).alsoDebug() df.columnsCount() shouldBe 1 df.rowsCount() shouldBe 3 - df["a"].type() shouldBe typeOf>() - df[1]["a"] shouldBe emptyList() + df["a"].type() shouldBe typeOf?>() + df[1]["a"] shouldBe null } @Test @@ -225,8 +227,8 @@ class JsonTests { val df = DataFrame.readJsonStr(json, typeClashTactic = ANY_COLUMNS).alsoDebug() df.columnsCount() shouldBe 1 df.rowsCount() shouldBe 3 - df["a"].type() shouldBe typeOf>() - df[1]["a"] shouldBe emptyList() + df["a"].type() shouldBe typeOf?>() + df[1]["a"] shouldBe null } @Test @@ -489,7 +491,8 @@ class JsonTests { val df = dataFrameOf("a")(listOf(1, 2, 3), null) val text = df.toJson() val df1 = DataFrame.readJsonStr(text) - df1["a"][1] shouldBe emptyList() + df1["a"][1] shouldBe null + df1 shouldBe df } @Test @@ -497,7 +500,8 @@ class JsonTests { val df = dataFrameOf("a")(listOf(1, 2, 3), null) val text = df.toJson() val df1 = DataFrame.readJsonStr(text, typeClashTactic = ANY_COLUMNS) - df1["a"][1] shouldBe emptyList() + df1["a"][1] shouldBe null + df1 shouldBe df } @Test @@ -621,7 +625,7 @@ class JsonTests { schema shouldNotContain "Void?" group["array1"].type() shouldBe typeOf() - group["array2"].type() shouldBe typeOf>() + group["array2"].type() shouldBe typeOf?>() } @Test @@ -703,11 +707,13 @@ class JsonTests { df.columnsCount() shouldBe 1 df.rowsCount() shouldBe 6 val a = df["a"] as ValueColumn<*> - a.type() shouldBe typeOf>() + a.type() shouldBe typeOf?>() a[0] shouldBe listOf(1, 2, 3) a[1] shouldBe listOf(null) - a[2..5].forEach { - it shouldBe emptyList() + a[2] shouldBe emptyList() + // json `null` and missing values are read as `null`, not as an empty list + a[3..5].forEach { + it shouldBe null } } @@ -811,12 +817,12 @@ class JsonTests { // ⌌-------------------------------------------------------⌍ // | | a:{b:{value:Int?, array:List}, c:Int?, d:Any?}| // |--|----------------------------------------------------| -// | 0| { b:{ value:1, array:[] }, c:null, d:null }| +// | 0| { b:{ value:1, array:null }, c:null, d:null }| // | 1| { b:{ value:null, array:[1, 2, 3] }, c:2, d:null }| -// | 2| { b:{ value:null, array:[] }, c:null, d:null }| -// | 3| { b:{ value:null, array:[] }, c:null, d:null }| -// | 4| { b:{ value:null, array:[] }, c:null, d:null }| -// | 5| { b:{ value:null, array:[] }, c:null, d:null }| +// | 2| { b:{ value:null, array:null }, c:null, d:null }| +// | 3| { b:{ value:null, array:null }, c:null, d:null }| +// | 4| { b:{ value:null, array:null }, c:null, d:null }| +// | 5| { b:{ value:null, array:null }, c:null, d:null }| // ⌎-------------------------------------------------------⌏ noKeyValue.columnsCount() shouldBe 1 noKeyValue.rowsCount() shouldBe 6 @@ -826,12 +832,12 @@ class JsonTests { it["b"].type() shouldBe typeOf>() it["b"]["value"].type() shouldBe typeOf() - it["b"]["array"].type() shouldBe typeOf>() + it["b"]["array"].type() shouldBe typeOf?>() it["c"].type() shouldBe typeOf() it["d"].type() shouldBe nothingType(nullable = true) it[0].let { - (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to 1, "array" to emptyList()) + (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to 1, "array" to null) it["c"] shouldBe null it["d"] shouldBe null } @@ -842,7 +848,8 @@ class JsonTests { } (it as ColumnGroup<*>)[2..5].forEach { it.let { - (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to null, "array" to emptyList()) + // json `null` and missing values are read as `null`, not as an empty list + (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to null, "array" to null) it["c"] shouldBe null it["d"] shouldBe null } @@ -1199,6 +1206,117 @@ class JsonTests { DataFrame.readJsonStr(json) shouldBe df } + @Test + fun `null is not read as empty list in array column`() { + // https://github.com/Kotlin/dataframe/issues/2048 + @Language("json") + val json = """[null, [123], []]""" + + for (typeClashTactic in JSON.TypeClashTactic.entries) { + DataFrame.readJsonStr(json, typeClashTactic = typeClashTactic).alsoDebug().let { + it.columnsCount() shouldBe 1 + it.rowsCount() shouldBe 3 + it.getColumn(0).let { + it.type() shouldBe typeOf?>() + it[0] shouldBe null + it[1] shouldBe listOf(123) + it[2] shouldBe emptyList() + } + } + } + } + + @Test + fun `null in nested array column`() { + // https://github.com/Kotlin/dataframe/issues/2048 + @Language("json") + val json = + """ + [ + {"a":[1,2,3]}, + {"a":[]}, + {"a":null}, + {} + ] + """.trimIndent() + + for (typeClashTactic in JSON.TypeClashTactic.entries) { + DataFrame.readJsonStr(json, typeClashTactic = typeClashTactic).alsoDebug().let { + it.columnsCount() shouldBe 1 + it.rowsCount() shouldBe 4 + it["a"].let { + it.type() shouldBe typeOf?>() + it[0] shouldBe listOf(1, 2, 3) + it[1] shouldBe emptyList() + it[2] shouldBe null + it[3] shouldBe null + } + } + } + } + + @Test + fun `value column round trip keeps the other columns`() { + // https://github.com/Kotlin/dataframe/issues/2045 + @Language("json") + val json = """[1,{"label":"record"},null]""" + + val df = DataFrame.readJsonStr(json).alsoDebug() + df["value"].type() shouldBe typeOf() + df["label"].type() shouldBe typeOf() + + df.toJson() shouldBe json + } + + @Test + fun `array column round trip keeps the other columns`() { + // https://github.com/Kotlin/dataframe/issues/2046 + @Language("json") + val json = """[{"label":"record"},[123]]""" + + val df = DataFrame.readJsonStr(json).alsoDebug() + df["array"].type() shouldBe typeOf?>() + df["array"][0] shouldBe null + df["label"].type() shouldBe typeOf() + + df.toJson() shouldBe json + } + + @Test + fun `array of objects column round trip keeps the other columns`() { + // https://github.com/Kotlin/dataframe/issues/2046 + @Language("json") + val json = """[{"label":"record"},[{"a":123}]]""" + + val df = DataFrame.readJsonStr(json).alsoDebug() + df["array"] shouldBe instanceOf>() + + df.toJson() shouldBe json + } + + @Test + fun `value and array column round trip`() { + // https://github.com/Kotlin/dataframe/issues/2045, https://github.com/Kotlin/dataframe/issues/2046 + @Language("json") + val json = """[1,{"label":"record"},[123],null,[]]""" + + DataFrame.readJsonStr(json).alsoDebug().toJson() shouldBe json + } + + @Test + fun `user-made value and array columns of nulls only are not unnamed columns`() { + // https://github.com/Kotlin/dataframe/issues/2045 + dataFrameOf( + "value" to columnOf(null, null), + "name" to columnOf("Alice", "Bob"), + ).toJson() shouldBe """[{"value":null,"name":"Alice"},{"value":null,"name":"Bob"}]""" + + dataFrameOf( + "array" to columnOf?>(null, null), + "name" to columnOf("Alice", "Bob"), + ).toJson() shouldBe """[{"array":null,"name":"Alice"},{"array":null,"name":"Bob"}]""" + } + @Test fun `parse invalid literal`() { // https://github.com/Kotlin/kotlinx.serialization/issues/2511 diff --git a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt index d4bac2d410..bed48f4122 100644 --- a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt +++ b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt @@ -7,6 +7,7 @@ import org.jetbrains.kotlinx.dataframe.api.convert import org.jetbrains.kotlinx.dataframe.api.convertTo import org.jetbrains.kotlinx.dataframe.api.with import kotlin.reflect.KType +import kotlin.reflect.full.withNullability import kotlin.reflect.jvm.jvmErasure import kotlin.reflect.typeOf @@ -33,6 +34,17 @@ public fun ConvertSchemaDsl<*>.convertDataRowsWithOpenApi() { it } } + + // A missing or `null` json array is read as `null`, so read it as an empty list + // when the schema requires the list to be non-nullable. + convertIf({ fromType, toSchema -> + fromType.isMarkedNullable && + !toSchema.type.isMarkedNullable && + fromType.jvmErasure == List::class && + fromType.withNullability(false) == toSchema.type + }) { + it ?: emptyList() + } } /** diff --git a/docs/StardustDocs/resources/api/read/readJsonTypeClash.html b/docs/StardustDocs/resources/api/read/readJsonTypeClash.html index 5e8693e372..6f78b6ef3e 100644 --- a/docs/StardustDocs/resources/api/read/readJsonTypeClash.html +++ b/docs/StardustDocs/resources/api/read/readJsonTypeClash.html @@ -459,7 +459,7 @@ /**/ From 09d258c124c31c81eea7b1d41b969d3a11243d4f Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Tue, 1 Sep 2026 18:04:49 +0400 Subject: [PATCH 02/12] document json IO null support --- .../jetbrains/kotlinx/dataframe/io/json.kt | 32 +++++++++++++++++++ .../kotlinx/dataframe/io/JsonTests.kt | 24 ++++++++++++++ docs/StardustDocs/topics/dataSources/JSON.md | 10 ++++++ docs/StardustDocs/topics/read.md | 13 ++++++-- docs/StardustDocs/topics/write.md | 19 +++++++++++ 5 files changed, 96 insertions(+), 2 deletions(-) diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt index baf6c9d158..f51acc10e8 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt @@ -116,6 +116,19 @@ public class JSON( * So, for the type clashing argument it will create a [ColumnGroup] with the properties `value`, `array`, * and the unwrapped properties of the objects the property can be. * + * Each of these columns only holds a value for the records that actually have that shape; + * for all other records it holds `null`, which makes all of them nullable: + * + * - `value` holds the JSON primitive, or `null` if the record is not a primitive. + * - `array` holds the JSON array as a [List], or `null` if the record is not an array + * (note that this is different from an empty JSON array `[]`, which is read as an empty list). + * When the arrays contain objects, `array` becomes a [FrameColumn] instead, in which a record + * without an array is an empty [DataFrame], as frame columns cannot hold `null`. + * - the unwrapped object properties hold `null` if the record is not an object, + * or if the object does not have that property. + * + * A JSON `null` record, as well as a missing property, is `null` in all these columns. + * * [ANY_COLUMNS] will create a [DataFrame] looking like: * ``` * ⌌-------------⌍ @@ -473,6 +486,25 @@ public fun DataRow.Companion.readJsonStr( unifyNumbers = unifyNumbers, ).single() +/** + * Converts this [DataFrame] to a JSON array; each row becomes an element of that array. + * + * By default, a row is written as a JSON object: a [ColumnGroup] becomes a nested object, + * a [FrameColumn] becomes a nested array of objects, and a [List] value becomes a JSON array. + * + * There's one exception, so that a [DataFrame] read from JSON with a type clash can be written back + * to its original form (see [TypeClashTactic]): when this frame has an unnamed `value` or `array` column, + * a row is instead written as the value it holds. Per row, in order: + * the `value` primitive, else the `array` array, else an object of the remaining columns, + * else — when the row holds no values at all — `null`. + * + * Note that a row of `null`s only is indistinguishable from a JSON `null` record, so + * `[1,{"label":"record"},{"label":null}]` is written back as `[1,{"label":"record"},null]`. + * Both forms are read into the exact same [DataFrame]. + * + * @param prettyPrint Whether to format the output with indentation and line breaks. `false` by default. + * @return This [DataFrame] as a JSON string. + */ public fun AnyFrame.toJson(prettyPrint: Boolean = false): String { val json = Json { this.prettyPrint = prettyPrint diff --git a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt index e4daa98ee7..1fbf2f457e 100644 --- a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt +++ b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt @@ -1301,6 +1301,30 @@ class JsonTests { val json = """[1,{"label":"record"},[123],null,[]]""" DataFrame.readJsonStr(json).alsoDebug().toJson() shouldBe json + + // the example documented in docs/StardustDocs/topics/write.md + @Language("json") + val documented = """[1,{"label":"record"},[123],null]""" + + DataFrame.readJsonStr(documented).toJson() shouldBe documented + } + + @Test + fun `a null record and an object of nulls only are indistinguishable`() { + // https://github.com/Kotlin/dataframe/issues/2045 + // both jsons are read into the exact same DataFrame, so writing it back can only produce one of them + val nullRecord = DataFrame.readJsonStr("""[1,{"label":"record"},null]""").alsoDebug() + val objectOfNulls = DataFrame.readJsonStr("""[1,{"label":"record"},{"label":null}]""").alsoDebug() + + objectOfNulls shouldBe nullRecord + + // a row without any values is written back as a `null` record + nullRecord.toJson() shouldBe """[1,{"label":"record"},null]""" + objectOfNulls.toJson() shouldBe """[1,{"label":"record"},null]""" + + // without a type clash there's no unnamed `value`/`array` column, so rows stay objects + DataFrame.readJsonStr("""[{"label":"record"},{"label":null}]""") + .toJson() shouldBe """[{"label":"record"},{"label":null}]""" } @Test diff --git a/docs/StardustDocs/topics/dataSources/JSON.md b/docs/StardustDocs/topics/dataSources/JSON.md index 0eac509407..27fce7ba02 100644 --- a/docs/StardustDocs/topics/dataSources/JSON.md +++ b/docs/StardustDocs/topics/dataSources/JSON.md @@ -48,6 +48,13 @@ val df = DataFrame.readJson("https://kotlin.github.io/dataframe/resources/exampl +A JSON `null`, as well as a missing property, is read as `null`, so the corresponding column becomes nullable. +This includes array columns: `[null, [123], []]` is read as a `List?` column holding `null`, `[123]`, and +an empty list. + +When the same property holds values of different shapes across records, extra "value" and "array" columns are +created; see [managing type clashes](read.md#manage-type-clashes) for the exact rules. + ## Write You can write a [`DataFrame`](DataFrame.md) to a JSON file using the [`writeJson()`](write.md#writing-to-json) method: @@ -59,3 +66,6 @@ df.writeJson("example.json") ``` + +Frames read from JSON with a type clash are written back to their original form; see +[writing to JSON](write.md#writing-to-json) for the details. diff --git a/docs/StardustDocs/topics/read.md b/docs/StardustDocs/topics/read.md index addc83bb1b..a155f8502a 100644 --- a/docs/StardustDocs/topics/read.md +++ b/docs/StardustDocs/topics/read.md @@ -447,14 +447,23 @@ By default, if a type clash occurs when reading JSON, a new [`column group`](Dat any number of object properties: * "value" will be set to the value of the JSON element if it's a primitive, else it will be `null`. -* "array" will be set to the array of values if the JSON element is an array, else it will be `[]`. +* "array" will be set to the array of values if the JSON element is an array, else it will be `null`. If the JSON element is an object, then each property will spread out to its own column in the group, else these columns will be `null`. +Since every one of these columns only holds a value for the elements that actually have that shape, all of them +are nullable. A JSON `null` element, as well as a missing property, is `null` in all of them. + +> An empty JSON array `[]` is read as an empty list, which is different from `null`, meaning "there is no array here". +> +> When the arrays contain objects, "array" becomes a [`FrameColumn`](DataColumn.md#framecolumn) instead. There, an +> element without an array is an empty [`DataFrame`](DataFrame.md), because frame columns cannot hold `null`. +> {style="note"} + In this case `typeClashTactic = JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS`. -For example, this is how the following JSON will be read (including `null` and `[]` values): +For example, this is how the following JSON will be read (including `null` values): diff --git a/docs/StardustDocs/topics/write.md b/docs/StardustDocs/topics/write.md index eaaede94dd..f89f259f1b 100644 --- a/docs/StardustDocs/topics/write.md +++ b/docs/StardustDocs/topics/write.md @@ -49,6 +49,25 @@ val jsonStr = df.toJson(prettyPrint = true) +Each row is written as an element of a JSON array: a [`ColumnGroup`](DataColumn.md#columngroup) becomes a nested +object, a [`FrameColumn`](DataColumn.md#framecolumn) becomes a nested array of objects, and a `List` value becomes +a JSON array. + +There's one exception, so that a [`DataFrame`](DataFrame.md) read from JSON with a +[type clash](read.md#manage-type-clashes) can be written back to its original form: when the frame has an unnamed +"value" or "array" column, a row is instead written as the value it holds — the "value" primitive, else the "array" +array, else an object of the remaining columns, else — when the row holds no values at all — `null`. For example: + +```kotlin +DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""").toJson() +// [1,{"label":"record"},[123],null] +``` + +> A row of `null`s only is indistinguishable from a JSON `null` element, so +> `[1,{"label":"record"},{"label":null}]` is written back as `[1,{"label":"record"},null]`. +> Both forms are read into the exact same [`DataFrame`](DataFrame.md). +> {style="note"} + ### Write to Excel spreadsheet Add dependency: From 92e94a88fad3ae8fcef6ad37db7a2d9956ab4e0c Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Wed, 2 Sep 2026 15:18:47 +0400 Subject: [PATCH 03/12] Add JSON read/write sample --- docs/StardustDocs/topics/write.md | 9 ++++++++- .../kotlinx/dataframe/samples/api/Write.kt | 13 ++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/StardustDocs/topics/write.md b/docs/StardustDocs/topics/write.md index f89f259f1b..4d81623b95 100644 --- a/docs/StardustDocs/topics/write.md +++ b/docs/StardustDocs/topics/write.md @@ -58,11 +58,18 @@ There's one exception, so that a [`DataFrame`](DataFrame.md) read from JSON with "value" or "array" column, a row is instead written as the value it holds — the "value" primitive, else the "array" array, else an object of the remaining columns, else — when the row holds no values at all — `null`. For example: + + ```kotlin DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""").toJson() -// [1,{"label":"record"},[123],null] ``` +Output: + +[1,{"label":"record"},[123],null] + + + > A row of `null`s only is indistinguishable from a JSON `null` element, so > `[1,{"label":"record"},{"label":null}]` is written back as `[1,{"label":"record"},null]`. > Both forms are read into the exact same [`DataFrame`](DataFrame.md). diff --git a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt index 6cc3311aa3..8db46529bc 100644 --- a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt +++ b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt @@ -6,10 +6,12 @@ import io.kotest.matchers.string.shouldStartWith import org.apache.arrow.vector.types.pojo.Schema import org.apache.poi.ss.usermodel.Sheet import org.apache.poi.ss.usermodel.WorkbookFactory +import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.remove import org.jetbrains.kotlinx.dataframe.io.ArrowWriter import org.jetbrains.kotlinx.dataframe.io.arrowWriter +import org.jetbrains.kotlinx.dataframe.io.readJsonStr import org.jetbrains.kotlinx.dataframe.io.saveArrowFeatherToByteArray import org.jetbrains.kotlinx.dataframe.io.saveArrowIPCToByteArray import org.jetbrains.kotlinx.dataframe.io.toCsvStr @@ -20,11 +22,12 @@ import org.jetbrains.kotlinx.dataframe.io.writeCsv import org.jetbrains.kotlinx.dataframe.io.writeExcel import org.jetbrains.kotlinx.dataframe.io.writeJson import org.jetbrains.kotlinx.dataframe.io.writeMismatchMessage +import org.jetbrains.kotlinx.dataframe.samples.DataFrameSampleHelper import org.junit.Test import java.io.File import kotlin.io.path.deleteExisting -class Write : TestBase { +class Write : TestBase, DataFrameSampleHelper("write", "io") { val df = peopleDf @@ -76,6 +79,14 @@ class Write : TestBase { """.trimIndent() } + @Test + fun readAndWriteJson() { + // SampleStart + DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""").toJson() + // SampleEnd + .saveSample() + } + @Test fun writeXls() { useTempFile { file -> From cc1eb047c689f6f4638564a581a8ecf8faade7a7 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Fri, 18 Sep 2026 15:44:29 +0400 Subject: [PATCH 04/12] update documentation regarding "array" / "value" columns in JSON IO --- .../jetbrains/kotlinx/dataframe/io/json.kt | 25 +- .../kotlinx/dataframe/io/JsonTests.kt | 45 ++ .../api/read/readJsonArrayColumn.html | 510 +++++++++++++++++ .../read/readJsonArrayOfObjectsColumn.html | 520 ++++++++++++++++++ .../api/read/readJsonTopLevelTypeClash.html | 511 +++++++++++++++++ .../resources/api/read/readJsonTypeClash.html | 2 +- .../api/read/readJsonValueColumn.html | 510 +++++++++++++++++ docs/StardustDocs/topics/_shadow_resources.md | 4 + docs/StardustDocs/topics/dataSources/JSON.md | 12 +- docs/StardustDocs/topics/io/read.md | 70 ++- docs/StardustDocs/topics/write.md | 22 +- .../kotlinx/dataframe/samples/api/Read.kt | 32 ++ 12 files changed, 2238 insertions(+), 25 deletions(-) create mode 100644 docs/StardustDocs/resources/api/read/readJsonArrayColumn.html create mode 100644 docs/StardustDocs/resources/api/read/readJsonArrayOfObjectsColumn.html create mode 100644 docs/StardustDocs/resources/api/read/readJsonTopLevelTypeClash.html create mode 100644 docs/StardustDocs/resources/api/read/readJsonValueColumn.html diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt index a090768cd3..d66c601ad0 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt @@ -129,6 +129,10 @@ public class JSON( * * A JSON `null` record, as well as a missing property, is `null` in all these columns. * + * Note that a type clash is not the only source of `value`/`array` columns; at the top level they're created + * for any JSON element that isn't an object. + * For more information: [See the "value" and "array" columns on the documentation website.](https://kotlin.github.io/dataframe/read.html#value-and-array-columns) + * * [ANY_COLUMNS] will create a [DataFrame] looking like: * ``` * ⌌-------------⌍ @@ -493,16 +497,23 @@ public fun DataRow.Companion.readJsonStr( * a [FrameColumn] becomes a nested array of objects, and a [List] value becomes a JSON array. * * There's one exception, so that a [DataFrame] read from JSON with a top-level type clash can be written back - * to its original form (see [TypeClashTactic]). When one of the top-level columns is detected as a `value` or - * `array` column — it's named `value` or `array` and only holds a value in rows where every other column holds - * none — a row is written as the record it was read from: - * - the value of the `value` column, if the row has one; - * - otherwise the array of the `array` column, if the row has one; + * to its original form (see [TypeClashTactic]). Such a clash puts the elements that aren't objects into columns + * named "value" and "array" + * ([see them on the documentation website](https://kotlin.github.io/dataframe/read.html#value-and-array-columns)), + * beside the objects' own properties. When one of the top-level columns is detected + * as such a "value" or "array" column — it's named "value" or "array" and only holds a value in rows where every + * other column holds none — a row is written as the record it was read from: + * - the value of the "value" column, if the row has one; + * - otherwise the array of the "array" column, if the row has one; * - otherwise an object of the remaining columns, if any of them holds a value; * - otherwise `null`. * - * Note that a type clash inside a [ColumnGroup] is not restored like this; the group is written as an object, - * so its `value` and `array` columns show up as ordinary properties. + * Only a *top-level* clash is restored like this: + * - A clash inside a [ColumnGroup] isn't: the group is written as an object, so its "value" and "array" columns + * show up as ordinary properties. + * - A single-column [DataFrame] isn't either, since a lone "value"/"array" column is indistinguishable from one + * the user named that way. So a JSON array of non-objects, which is read into just a "value" or "array" column, + * doesn't round-trip: `[1,2,3]` is written back as `[{"value":1},{"value":2},{"value":3}]`. * * Note too that some JSON records are read into the exact same [DataFrame], so writing it back can only produce * one of them: a row without any values is written as `null`, so `[1,{"label":"record"},{"label":null}]` becomes diff --git a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt index 04ea016c2f..a4f2b6a1a0 100644 --- a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt +++ b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt @@ -1368,6 +1368,51 @@ class JsonTests { DataFrame.readJsonStr(json) shouldBe df } + @Test + fun `value and array columns are created for nameless json elements`() { + for (typeClashTactic in JSON.TypeClashTactic.entries) { + fun read( + @Language("json") json: String, + ) = DataFrame.readJsonStr(json, typeClashTactic = typeClashTactic) + + read("""[1, 2, 3]""").let { + it.columnNames() shouldBe listOf("value") + it["value"].type() shouldBe typeOf() + } + read("""[[1], [2]]""").let { + it.columnNames() shouldBe listOf("array") + it["array"].type() shouldBe typeOf>() + } + read("""[[{ "a": 1 }], [{ "a": 2 }]]""").let { + it.columnNames() shouldBe listOf("array") + it["array"] shouldBe instanceOf>() + } + + // a property keeps its own name when there's no clash + read("""[{ "a": [1, 2] }, { "a": [3] }]""")["a"].type() shouldBe typeOf>() + read("""[{ "a": { "b": 1 } }, { "a": { "b": 2 } }]""")["a"].let { + it shouldBe instanceOf>() + (it as ColumnGroup<*>).columnNames() shouldBe listOf("b") + } + } + + // a top-level clash has no property to group under, so the columns end up side by side + DataFrame.readJsonStr("""[1, { "label": "record" }]""").let { + it["value"].type() shouldBe typeOf() + it["label"].type() shouldBe typeOf() + } + } + + @Test + fun `a lone value or array column is not an unnamed column`() { + DataFrame.readJsonStr("""[1,2,3]""").toJson() shouldBe """[{"value":1},{"value":2},{"value":3}]""" + DataFrame.readJsonStr("""[[1],[2]]""").toJson() shouldBe """[{"array":[1]},{"array":[2]}]""" + + // as soon as a clash gives the frame a second column, the round trip holds again + DataFrame.readJsonStr("""[1,{"label":"record"}]""").toJson() shouldBe """[1,{"label":"record"}]""" + DataFrame.readJsonStr("""[{"label":"record"},[1]]""").toJson() shouldBe """[{"label":"record"},[1]]""" + } + @Test fun `null is not read as empty list in array column`() { // https://github.com/Kotlin/dataframe/issues/2048 diff --git a/docs/StardustDocs/resources/api/read/readJsonArrayColumn.html b/docs/StardustDocs/resources/api/read/readJsonArrayColumn.html new file mode 100644 index 0000000000..6b7302f05d --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonArrayColumn.html @@ -0,0 +1,510 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/resources/api/read/readJsonArrayOfObjectsColumn.html b/docs/StardustDocs/resources/api/read/readJsonArrayOfObjectsColumn.html new file mode 100644 index 0000000000..9ce22aa863 --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonArrayOfObjectsColumn.html @@ -0,0 +1,520 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/resources/api/read/readJsonTopLevelTypeClash.html b/docs/StardustDocs/resources/api/read/readJsonTopLevelTypeClash.html new file mode 100644 index 0000000000..a2ecfaaa6e --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonTopLevelTypeClash.html @@ -0,0 +1,511 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/resources/api/read/readJsonTypeClash.html b/docs/StardustDocs/resources/api/read/readJsonTypeClash.html index 5e8693e372..6f78b6ef3e 100644 --- a/docs/StardustDocs/resources/api/read/readJsonTypeClash.html +++ b/docs/StardustDocs/resources/api/read/readJsonTypeClash.html @@ -459,7 +459,7 @@ /**/ diff --git a/docs/StardustDocs/resources/api/read/readJsonValueColumn.html b/docs/StardustDocs/resources/api/read/readJsonValueColumn.html new file mode 100644 index 0000000000..451a9f6c44 --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonValueColumn.html @@ -0,0 +1,510 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/topics/_shadow_resources.md b/docs/StardustDocs/topics/_shadow_resources.md index 568f46883f..2429c8ddd0 100644 --- a/docs/StardustDocs/topics/_shadow_resources.md +++ b/docs/StardustDocs/topics/_shadow_resources.md @@ -253,11 +253,15 @@ + + + + diff --git a/docs/StardustDocs/topics/dataSources/JSON.md b/docs/StardustDocs/topics/dataSources/JSON.md index 4269a2aed7..95a7976fe1 100644 --- a/docs/StardustDocs/topics/dataSources/JSON.md +++ b/docs/StardustDocs/topics/dataSources/JSON.md @@ -55,8 +55,12 @@ a value in every column. This also holds for array columns: `[null, [123], []]` is read as a `List?` column holding `null`, `[123]`, and an empty list. -When the same property holds values of different shapes across records, extra "value" and "array" columns are -created; see [managing type clashes](read.md#manage-type-clashes) for the exact rules. +JSON elements that aren't objects have no property name to use as a column name, so they're read into the +special [`value` and `array` columns](read.md#value-and-array-columns). +The same special columns are also used +to resolve [type clashes](read.md#manage-type-clashes) +when the same property contains values of different shapes +across records. ## Write @@ -70,5 +74,5 @@ df.writeJson("example.json") -A [`DataFrame`](DataFrame.md) read from JSON with a type clash is written back to its original form; see -[writing to JSON](write.md#writing-to-json) for the details. +A [`DataFrame`](DataFrame.md) read from JSON with a top-level type clash is written back to its original form; +see [writing to JSON](write.md#writing-to-json) for the details and for the cases that don't round-trip. diff --git a/docs/StardustDocs/topics/io/read.md b/docs/StardustDocs/topics/io/read.md index 53db8d8679..9a7d14fc05 100644 --- a/docs/StardustDocs/topics/io/read.md +++ b/docs/StardustDocs/topics/io/read.md @@ -441,17 +441,77 @@ has the `Double` type because it's the smallest unified number type for `Int` an ### JSON parsing options -#### Manage type clashes +#### The "value" and "array" columns {id="value-and-array-columns"} + +A JSON element that isn't an object has no property name to be used as a column name, so the reader falls back to +two fixed names: + +* "value" holds the JSON element if it's a primitive, else it's `null`. +* "array" holds the JSON element as a list if it's an array, else it's `null`. + +They're created in two situations. The first is **at the top level**, where the elements of the JSON array are the +rows themselves and so have no name at all. This happens regardless of the `typeClashTactic` and even when there's +no clash to begin with. + +An array of primitives is read into a single "value" column: + + + +```kotlin +val df = DataFrame.readJsonStr("""[1, 2, 3]""") +``` + + + + +An array of arrays, into a single "array" column of lists: + + + +```kotlin +val df = DataFrame.readJsonStr("""[[1], [2]]""") +``` + + + -By default, if a type clash occurs when reading JSON, a new [`column group`](DataColumn.md#columngroup) is created consisting of: "value", "array", and -any number of object properties: +And when those arrays hold objects, "array" becomes a [`FrameColumn`](DataColumn.md#framecolumn): -* "value" will be set to the value of the JSON element if it's a primitive, else it will be `null`. -* "array" will be set to the array of values if the JSON element is an array, else it will be `null`. + + +```kotlin +val df = DataFrame.readJsonStr("""[[{ "a": 1 }], [{ "a": 2 }]]""") +``` + + + + +The second situation is a **type clash**, described below. + +#### Manage type clashes + +By default, if a type clash occurs when reading JSON — the same property holds elements of different shapes across +records — a new [`column group`](DataColumn.md#columngroup) is created for that property, consisting of: "value", +"array", and any number of object properties. If the JSON element is an object, then each property will spread out to its own column in the group, else these columns will be `null`. +A clash at the top level has no property to group under, so there "value", "array", and the objects' properties +become sibling columns of the [`DataFrame`](DataFrame.md) itself: + + + +```kotlin +val df = DataFrame.readJsonStr("""[1, { "label": "record" }]""") +``` + + + + +Without a clash there's no such group: a property whose elements are all arrays simply becomes a `List` column under +its own name, and one whose elements are all objects becomes a group of just those objects' properties. + Since every one of these columns only holds a value for the elements that actually have that shape, all of them are nullable. A JSON `null` element, as well as a missing property, is `null` in all of them. diff --git a/docs/StardustDocs/topics/write.md b/docs/StardustDocs/topics/write.md index 1d356f1e9f..0329bd0ea7 100644 --- a/docs/StardustDocs/topics/write.md +++ b/docs/StardustDocs/topics/write.md @@ -53,10 +53,12 @@ Each row is written as an element of a JSON array: a [`ColumnGroup`](DataColumn. object, a [`FrameColumn`](DataColumn.md#framecolumn) becomes a nested array of objects, and a `List` value becomes a JSON array. -There's one exception, so that a [`DataFrame`](DataFrame.md) read from JSON with a -[type clash](read.md#manage-type-clashes) can be written back to its original form. -When one of its top-level columns is detected as a "value" or "array" column — it's named `value` or `array` and -only holds a value in rows where every other column holds none — a row is written as the record it was read from: +There's one exception, so that a [`DataFrame`](DataFrame.md) read from JSON with a top-level +[type clash](read.md#manage-type-clashes) can be written back to its original form. Such a clash puts the +elements that aren't objects into ["value" and "array" columns](read.md#value-and-array-columns), beside the +objects' own properties. When one of the top-level columns is detected as such a "value" or "array" column — +it's named `value` or `array` and only holds a value in rows where every other column holds none — a row is +written as the record it was read from: * the value of the "value" column, if the row has one; * otherwise the array of the "array" column, if the row has one; @@ -77,10 +79,14 @@ Output: -> Only a top-level type clash is restored like this. A clash inside a -> [`ColumnGroup`](DataColumn.md#columngroup) is written as the group's object, so its "value" and "array" -> columns show up as ordinary properties: `[{"a":"text"},{"a":[6,7,8]}]` is written back as -> `[{"a":{"value":"text","array":null}},{"a":{"value":null,"array":[6,7,8]}}]`. +> Only a *top-level* clash is restored like this: +> * a clash inside a [`ColumnGroup`](DataColumn.md#columngroup) isn't — the group is written as its object, so +> its "value" and "array" columns show up as ordinary properties: `[{"a":"text"},{"a":[6,7,8]}]` is written +> back as `[{"a":{"value":"text","array":null}},{"a":{"value":null,"array":[6,7,8]}}]`; +> * a single-column [`DataFrame`](DataFrame.md) isn't either, since a lone "value"/"array" column can't be told +> apart from one the user named that way. So a JSON array of non-objects, which is read into just a "value" or +> "array" column, doesn't round-trip: `[1,2,3]` is written back as +> `[{"value":1},{"value":2},{"value":3}]`. > {style="note"} > Some JSON records are read into the exact same [`DataFrame`](DataFrame.md), so writing it back can only diff --git a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt index c0f2aa8441..39e57119af 100644 --- a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt +++ b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt @@ -401,6 +401,38 @@ class Read : DataFrameSampleHelper("read", "api") { .saveDfHtmlSample() } + @Test + fun readJsonValueColumn() { + // SampleStart + val df = DataFrame.readJsonStr("""[1, 2, 3]""") + // SampleEnd + .saveDfHtmlSample() + } + + @Test + fun readJsonArrayColumn() { + // SampleStart + val df = DataFrame.readJsonStr("""[[1], [2]]""") + // SampleEnd + .saveDfHtmlSample() + } + + @Test + fun readJsonArrayOfObjectsColumn() { + // SampleStart + val df = DataFrame.readJsonStr("""[[{ "a": 1 }], [{ "a": 2 }]]""") + // SampleEnd + .saveDfHtmlSample() + } + + @Test + fun readJsonTopLevelTypeClash() { + // SampleStart + val df = DataFrame.readJsonStr("""[1, { "label": "record" }]""") + // SampleEnd + .saveDfHtmlSample() + } + @Test fun readJsonTypeClash() { // SampleStart From 1c1a4fa7622e00344dfd1cefc72a26c977d2dca6 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Fri, 18 Sep 2026 16:20:49 +0400 Subject: [PATCH 05/12] json io null fixes --- .../kotlinx/dataframe/impl/io/readJson.kt | 96 +++++++++---------- .../kotlinx/dataframe/impl/io/writeJson.kt | 25 ++--- docs/StardustDocs/topics/{ => io}/write.md | 4 +- 3 files changed, 61 insertions(+), 64 deletions(-) rename docs/StardustDocs/topics/{ => io}/write.md (99%) diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt index 89d986cc76..3b6715adc1 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt @@ -300,33 +300,12 @@ internal fun fromJsonListAnyColumns( isLenient = isLenient, ) - val res = when { - parsed.isSingleUnnamedColumn() -> { - val col = (parsed.getColumn(0) as UnnamedColumn).col - val elementType = col.type - val columnValues = col.values - .asList() - .splitByIndices(startIndices.asSequence()) - .toList() - // records that hold no array at all (like `null`) are not empty lists - .mapIndexed { i, list -> if (records[i] is JsonArray) list else null } - DataColumn.createValueColumn( - name = ARRAY_COLUMN_NAME, - values = columnValues, - type = List::class.createType( - arguments = listOf(KTypeProjection.invariant(elementType)), - nullable = records.any { it !is JsonArray }, - ), - ) - } - - else -> - parsed.unwrapUnnamedColumns() - .chunked( - startIndices = startIndices, - name = ARRAY_COLUMN_NAME, // will be erased - ) - } + val res = createArrayColumn( + parsed = parsed, + name = ARRAY_COLUMN_NAME, // will be erased + records = records, + startIndices = startIndices, + ) listOf(UnnamedColumn(res)) } @@ -455,6 +434,40 @@ internal fun fromJsonListAnyColumns( private fun AnyFrame.isSingleUnnamedColumn() = columnsCount() == 1 && getColumn(0) is UnnamedColumn +/** + * Collects the arrays of [records] into a single column named [name]: a `List` value column when the arrays + * hold single values, else a [FrameColumn] holding the objects of each array. + * + * @param parsed The contents of all arrays of [records], parsed and concatenated into one [DataFrame]. + * @param startIndices For each record, the index in [parsed] where its array starts, so that [parsed] can be + * split back into one entry per record. + */ +private fun createArrayColumn( + parsed: AnyFrame, + name: String, + records: List<*>, + startIndices: Iterable, +): DataColumn = + if (parsed.isSingleUnnamedColumn()) { + val col = (parsed.getColumn(0) as UnnamedColumn).col + val columnValues = col.values + .asList() + .splitByIndices(startIndices.asSequence()) + .toList() + // records that hold no array at all (like `null`) are not empty lists + .mapIndexed { i, list -> list.takeIf { records[i] is JsonArray } } + DataColumn.createValueColumn( + name = name, + values = columnValues, + type = List::class.createType( + arguments = listOf(KTypeProjection.invariant(col.type)), + nullable = records.any { it !is JsonArray }, + ), + ) + } else { + parsed.unwrapUnnamedColumns().chunked(startIndices, name) + } + /** * Json to DataFrame converter that creates allows creates `value` and `array` accessors * instead of [Any] columns. @@ -664,29 +677,12 @@ internal fun fromJsonListArrayAndValueColumns( jsonPath = jsonPath.appendArrayWithWildcard(), ) - val res = when { - parsed.isSingleUnnamedColumn() -> { - val col = (parsed.getColumn(0) as UnnamedColumn).col - val elementType = col.type - val columnValues = - col.values - .asList() - .splitByIndices(startIndices.asSequence()) - .toList() - // records that hold no array at all (like `null`) are not empty lists - .mapIndexed { i, list -> if (records[i] is JsonArray) list else null } - DataColumn.createValueColumn( - name = colName, - values = columnValues, - type = List::class.createType( - arguments = listOf(KTypeProjection.invariant(elementType)), - nullable = records.any { it !is JsonArray }, - ), - ) - } - - else -> parsed.unwrapUnnamedColumns().chunked(startIndices, colName) - } + val res = createArrayColumn( + parsed = parsed, + name = colName, + records = records, + startIndices = startIndices, + ) UnnamedColumn(res) } diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt index cf3f471201..cdb222967a 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt @@ -275,15 +275,15 @@ internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { ?.takeIf { it.kind() == ColumnKind.Value } // 'readJson' cannot have created an all-null 'value' column, consider it a regular one ?.takeIf { it.holdsAnyValue() } - ?.takeIf { valueCol -> + ?.takeUnless { valueCol -> val otherCols = allColumns - valueCol - // It's a valid 'value' column only if for each value it holds, - // the corresponding cells in the other columns hold no value. - val isValidValueColumn = indices().all { row -> - !valueCol.holdsValueAt(row) || otherCols.all { !it.holdsValueAt(row) } + // It's not a 'value' column if any of the values it holds sits next to + // a value in one of the other columns. + val isInvalidValueColumn = indices().any { row -> + valueCol.holdsValueAt(row) && otherCols.any { it.holdsValueAt(row) } } - isValidValueColumn + isInvalidValueColumn } } @@ -309,14 +309,15 @@ internal fun AnyFrame.extractArrayColumn(): DataColumn<*>? { ?.takeIf { it.kind() == ColumnKind.Frame || it.isList() } // 'readJson' cannot have created an all-null 'array' column, consider it a regular one ?.takeIf { it.holdsAnyValue() } - ?.takeIf { arrayCol -> + ?.takeUnless { arrayCol -> val otherCols = allColumns - arrayCol - // It's a valid 'array' column only if for each value it holds, - // the corresponding cells in the other columns hold no value. - val isValidArrayColumn = indices().all { row -> - !arrayCol.holdsValueAt(row) || otherCols.all { !it.holdsValueAt(row) } + // It's not an 'array' column if any of the arrays it holds sits next to + // a value in one of the other columns. + val isInvalidArrayColumn = indices().any { row -> + arrayCol.holdsValueAt(row) && otherCols.any { it.holdsValueAt(row) } } - isValidArrayColumn + + isInvalidArrayColumn } } diff --git a/docs/StardustDocs/topics/write.md b/docs/StardustDocs/topics/io/write.md similarity index 99% rename from docs/StardustDocs/topics/write.md rename to docs/StardustDocs/topics/io/write.md index 0329bd0ea7..70484e7620 100644 --- a/docs/StardustDocs/topics/write.md +++ b/docs/StardustDocs/topics/io/write.md @@ -89,8 +89,8 @@ Output: > `[{"value":1},{"value":2},{"value":3}]`. > {style="note"} -> Some JSON records are read into the exact same [`DataFrame`](DataFrame.md), so writing it back can only -> produce one of them: +> Some JSON records are read into the exact same [`DataFrame`](DataFrame.md), +> so writing it back can only produce: > * a row without any values is written as `null`, so `[1,{"label":"record"},{"label":null}]` becomes > `[1,{"label":"record"},null]`; > * an empty array of objects is read as an empty nested [`DataFrame`](DataFrame.md), just like no array at all, From 93268d3e23b129f20c28ff1d4d18317f2ad926a0 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Fri, 18 Sep 2026 17:25:44 +0400 Subject: [PATCH 06/12] fix JSON IO null with metadata --- .../kotlinx/dataframe/impl/io/writeJson.kt | 3 ++- .../kotlinx/dataframe/io/JsonTests.kt | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt index cdb222967a..a27bc8e412 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt @@ -177,10 +177,11 @@ internal fun encodeValue(col: AnyCol, index: Int, customEncoders: List matchingEncoder.encode(col[index]) + // a `null` list is not an empty one, just like in `encodeRow` col.isList() -> col[index]?.let { list -> val values = (list as List<*>).map { convert(it) } JsonArray(values) - } ?: JsonArray(emptyList()) + } ?: JsonPrimitive(null) col.typeClass in valueTypes -> convert(col[index]) diff --git a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt index a4f2b6a1a0..78638ab728 100644 --- a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt +++ b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt @@ -1548,6 +1548,24 @@ class JsonTests { .dataWithMetadata() shouldBe """[1,{"label":"record"},[123],null]""" } + @Test + fun `a null list is written as null with metadata too`() { + fun AnyFrame.dataWithMetadata(): String = + parseJsonStr(toJsonWithMetadata(rowsCount()))[KOTLIN_DATAFRAME]!!.jsonArray.toString() + + val df = DataFrame.readJsonStr("""[{"a":[1]},{"a":[]},{"a":null}]""").alsoDebug() + df["a"].type() shouldBe typeOf?>() + + val expected = """[{"a":[1]},{"a":[]},{"a":null}]""" + df.toJson() shouldBe expected + df.dataWithMetadata() shouldBe expected + + // a user-made nullable `List` column behaves the same + dataFrameOf( + "a" to columnOf?>(listOf(1), emptyList(), null), + ).dataWithMetadata() shouldBe expected + } + @Test fun `a column named array that holds no arrays is not an unnamed column`() { // a user-made `array` column of single values must not be encoded as if it held JSON arrays From 6ff442cf8ed827a91b57eedc1e1308cdcf1bbd68 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Tue, 22 Sep 2026 18:16:40 +0400 Subject: [PATCH 07/12] fix toJsonWithMetadata --- .../kotlinx/dataframe/impl/io/writeJson.kt | 45 +++++------ .../kotlinx/dataframe/io/JsonTests.kt | 78 +++++++++++++++++-- 2 files changed, 88 insertions(+), 35 deletions(-) diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt index a27bc8e412..81f8361115 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt @@ -253,18 +253,23 @@ private fun createJsonTypeDescriptor(columnSchema: ColumnSchema): JsonObject = }, ) +/** + * Unlike [encodeFrame], this is the wire format of the Kotlin Notebook plugin, described in + * `docs/serialization_format.md`: every row has to be an object with one entry per column of [frame]. + * So the unnamed `value`/`array` columns are encoded as regular columns here, instead of as the JSON record + * they were read from. + */ internal fun encodeFrameWithMetadata( frame: AnyFrame, rowLimit: Int? = null, customEncoders: List = emptyList(), -): JsonArray = - encodeFrameRows( - frame = frame, - encodeArrayOfObjects = { encodeFrameWithMetadata(it, rowLimit, customEncoders) }, - encodeObject = { objectFrame, rowIndex -> - encodeRowWithMetadata(objectFrame, rowIndex, rowLimit, customEncoders) - }, - ) +): JsonArray { + val data = frame.indices().map { rowIndex -> + encodeRowWithMetadata(frame, rowIndex, rowLimit, customEncoders) + } + + return buildJsonArray { addAll(data.map { convert(it) }) } +} internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { val allColumns = columns() @@ -322,26 +327,12 @@ internal fun AnyFrame.extractArrayColumn(): DataColumn<*>? { } } -internal fun encodeFrame(frame: AnyFrame): JsonArray = - encodeFrameRows( - frame = frame, - encodeArrayOfObjects = { encodeFrame(it) }, - encodeObject = { objectFrame, rowIndex -> encodeRow(objectFrame, rowIndex) }, - ) - /** * Encodes each row of [frame] as the JSON record it was read from: the value of the unnamed "value" column, - * else the array of the unnamed "array" column, else an object built from the remaining columns with - * [encodeObject], else — when the row holds no values at all — `null`. - * - * Arrays of objects are encoded with [encodeArrayOfObjects], so that the caller decides whether the nested - * frame is written with or without metadata. + * else the array of the unnamed "array" column, else an object of the remaining columns, + * else — when the row holds no values at all — `null`. */ -private inline fun encodeFrameRows( - frame: AnyFrame, - encodeArrayOfObjects: (AnyFrame) -> JsonElement, - encodeObject: (objectFrame: AnyFrame, rowIndex: Int) -> JsonElement?, -): JsonArray { +internal fun encodeFrame(frame: AnyFrame): JsonArray { val valueColumn = frame.extractValueColumn() val arrayColumn = frame.extractArrayColumn() @@ -358,7 +349,7 @@ private inline fun encodeFrameRows( arrayColumn?.holdsValueAt(rowIndex) == true -> if (arraysAreFrames) { - encodeArrayOfObjects(arrayColumn[rowIndex] as AnyFrame) + encodeFrame(arrayColumn[rowIndex] as AnyFrame) } else { JsonArray((arrayColumn[rowIndex] as List<*>).map { convert(it) }) } @@ -367,7 +358,7 @@ private inline fun encodeFrameRows( // unless it holds no values at all; then the record was `null` itself hasUnnamedColumns && objectColumns.none { it.holdsValueAt(rowIndex) } -> null - else -> encodeObject(objectFrame, rowIndex) + else -> encodeRow(objectFrame, rowIndex) } } diff --git a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt index 78638ab728..839e6be1c5 100644 --- a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt +++ b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt @@ -1534,18 +1534,80 @@ class JsonTests { .toJson() shouldBe """[{"label":"record"},{"label":null}]""" } + /** + * Asserts that [this] is serialized according to `docs/serialization_format.md`: every entry of + * `kotlin_dataframe` — and of every nested group and frame — is an object holding exactly the columns + * that the accompanying metadata declares. + */ + private fun AnyFrame.shouldMatchSerializationFormat(): List { + fun checkRows(rows: List, declaredColumns: List) { + rows.forEach { row -> + // fails right here if a row is a bare value, a bare array or `null` instead of an object + row.jsonObject.keys shouldBe declaredColumns.toSet() + + row.jsonObject.values.forEach { cell -> + val metadata = (cell as? JsonObject)?.get(METADATA)?.jsonObject ?: return@forEach + val nestedColumns = metadata[COLUMNS]!!.jsonArray.map { it.jsonPrimitive.content } + when (metadata[KIND]!!.jsonPrimitive.content) { + ColumnKind.Group.toString() -> checkRows(listOf(cell.jsonObject[DATA]!!), nestedColumns) + ColumnKind.Frame.toString() -> checkRows(cell.jsonObject[DATA]!!.jsonArray, nestedColumns) + } + } + } + } + + val json = parseJsonStr(toJsonWithMetadata(rowsCount())) + val data = json[KOTLIN_DATAFRAME]!!.jsonArray + val columns = json[METADATA]!!.jsonObject[COLUMNS]!!.jsonArray.map { it.jsonPrimitive.content } + + columns shouldBe columnNames() + checkRows(data, columns) + return data + } + @Test - fun `type clash round trip with metadata`() { + fun `a type clash keeps every declared column with metadata`() { // https://github.com/Kotlin/dataframe/issues/2046 - // the notebook rendering path must extract the unnamed columns just like `toJson` does - fun AnyFrame.dataWithMetadata(): String = - parseJsonStr(toJsonWithMetadata(rowsCount()))[KOTLIN_DATAFRAME]!!.jsonArray.toString() + // `toJsonWithMetadata` is the Kotlin Notebook plugin's wire format (docs/serialization_format.md): + // unlike `toJson`, it must not collapse a row to the JSON record it was read from — the plugin renders + // cells by looking up `metadata.columns`, so the JSON has to mirror the dataframe structure exactly + DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""").let { df -> + df.shouldMatchSerializationFormat().toString() shouldBe + """[{"label":null,"value":1,"array":null},{"label":"record","value":null,"array":null},""" + + """{"label":null,"value":null,"array":[123]},{"label":null,"value":null,"array":null}]""" + + // and `toJson` does collapse them, that's the point of the two being different + df.toJson() shouldBe """[1,{"label":"record"},[123],null]""" + } + + // an `array` frame column becomes a NestedFrame, and the sibling `label` column is kept + DataFrame.readJsonStr("""[{"label":"record"},[{"a":123}]]""").let { df -> + val data = df.shouldMatchSerializationFormat() + + data[0].jsonObject["label"]!!.jsonPrimitive.content shouldBe "record" + data[1].jsonObject["array"]!! + .jsonObject[DATA]!! + .jsonArray + .toString() shouldBe """[{"a":123}]""" + + df.toJson() shouldBe """[{"label":"record"},[{"a":123}]]""" + } + } + + @Test + fun `json with metadata mirrors the dataframe structure`() { + // a nested type clash, so the `value`/`array` columns sit inside a column group + DataFrame.readJsonStr("""[{"a":"text"},{"a":{"b":2}},{"a":[6,7,8]}]""").shouldMatchSerializationFormat() + + // arrays of objects, so `array` is a frame column nested in a group + DataFrame.readJsonStr("""[{"a":"text"},{"a":[{"b":2}]}]""").shouldMatchSerializationFormat() - DataFrame.readJsonStr("""[{"label":"record"},[{"a":123}]]""") - .dataWithMetadata() shouldBe """[{"label":"record"},[{"a":123}]]""" + // a single unnamed column, and one that holds no values at all + DataFrame.readJsonStr("""[1,2,3]""").shouldMatchSerializationFormat() + DataFrame.readJsonStr("""[null,null]""").shouldMatchSerializationFormat() - DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""") - .dataWithMetadata() shouldBe """[1,{"label":"record"},[123],null]""" + // a real file with groups and frame columns several levels deep + DataFrame.readJson(testJson("repositories")).shouldMatchSerializationFormat() } @Test From 21a5d68dd5b781917f96b7de2f00b2eef5536790 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Tue, 22 Sep 2026 18:55:14 +0400 Subject: [PATCH 08/12] Update JSON IO docs: clarify "value" and "array" column behavior, type clash tactics, and examples --- docs/StardustDocs/topics/io/read.md | 61 +++++++++++++++-------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/docs/StardustDocs/topics/io/read.md b/docs/StardustDocs/topics/io/read.md index 13db26ad4d..ec25d001b9 100644 --- a/docs/StardustDocs/topics/io/read.md +++ b/docs/StardustDocs/topics/io/read.md @@ -441,7 +441,7 @@ has the `Double` type because it's the smallest unified number type for `Int` an ### JSON parsing options -#### The "value" and "array" columns {id="value-and-array-columns"} +#### The "value" and "array" columns for standalone values {id="value-and-array-columns"} A JSON element that isn't an object has no property name to be used as a column name, so the reader falls back to two fixed names: @@ -453,7 +453,7 @@ They're created in two situations. The first is **at the top level**, where the rows themselves and so have no name at all. This happens regardless of the `typeClashTactic` and even when there's no clash to begin with. -An array of primitives is read into a single "value" column: +* An array of primitives is read into a single "value" column: @@ -464,7 +464,7 @@ val df = DataFrame.readJsonStr("""[1, 2, 3]""") -An array of arrays, into a single "array" column of lists: +* An array of arrays, into a single "array" column of lists: @@ -475,7 +475,7 @@ val df = DataFrame.readJsonStr("""[[1], [2]]""") -And when those arrays hold objects, "array" becomes a [`FrameColumn`](DataColumn.md#framecolumn): +* When those arrays hold objects, "array" becomes a [`FrameColumn`](DataColumn.md#framecolumn): @@ -491,11 +491,32 @@ The second situation is a **type clash**, described below. #### Manage type clashes By default, if a type clash occurs when reading JSON — the same property holds elements of different shapes across -records — a new [`column group`](DataColumn.md#columngroup) is created for that property, consisting of: "value", -"array", and any number of object properties. +records — a new [`column group`](DataColumn.md#columngroup) is created for that property. +This column group contains a column for each children property (across all records), and optionally +"value" and "array" columns following the logic of reading row values and arrays as +[described above](#value-and-array-columns). +Each value will spread out to its own column in the group, other columns will contain `null`. -If the JSON element is an object, then each property will spread out to its own column in the group, else these columns -will be `null`. +In this case `typeClashTactic = JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS`. + +For example, this is how the following JSON will be read (including `null` values): + + + +```kotlin +val text = """ + [ + { "a": "text" }, + { "a": { "b": 2 } }, + { "a": [ 6, 7, 8 ] } + ] +""".trimIndent() + +val df = DataFrame.readJsonStr(text) +``` + + + A clash at the top level has no property to group under, so there "value", "array", and the objects' properties become sibling columns of the [`DataFrame`](DataFrame.md) itself: @@ -521,28 +542,8 @@ are nullable. A JSON `null` element, as well as a missing property, is `null` in > element without an array is an empty [`DataFrame`](DataFrame.md), because frame columns cannot hold `null`. > {style="note"} -In this case `typeClashTactic = JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS`. - -For example, this is how the following JSON will be read (including `null` values): - - - -```kotlin -val text = """ - [ - { "a": "text" }, - { "a": { "b": 2 } }, - { "a": [ 6, 7, 8 ] } - ] -""".trimIndent() - -val df = DataFrame.readJsonStr(text) -``` - - - - -This makes it more convenient to work with the data, but it can be confusing if you're not expecting it or if you +Default type clash tactic makes it more convenient to work with the data, +but it can be confusing if you're not expecting it or if you just need the type to be an `Any`. For this case, you can set `typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS` to get the following: From 39d4e150b495990903ac66d3ad77834729cc2eee Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Tue, 22 Sep 2026 19:20:02 +0400 Subject: [PATCH 09/12] Refine JSON IO docs: clarify type clash handling and update examples --- docs/StardustDocs/topics/io/read.md | 15 +++++++++------ .../kotlinx/dataframe/samples/api/Read.kt | 8 ++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/StardustDocs/topics/io/read.md b/docs/StardustDocs/topics/io/read.md index ec25d001b9..b54d974315 100644 --- a/docs/StardustDocs/topics/io/read.md +++ b/docs/StardustDocs/topics/io/read.md @@ -491,8 +491,10 @@ The second situation is a **type clash**, described below. #### Manage type clashes By default, if a type clash occurs when reading JSON — the same property holds elements of different shapes across -records — a new [`column group`](DataColumn.md#columngroup) is created for that property. -This column group contains a column for each children property (across all records), and optionally +records (objects, values or arrays) +— a new [`column group`](DataColumn.md#columngroup) is created for that property. +This column group contains a column for each children object property +(across all records), and "value" and "array" columns following the logic of reading row values and arrays as [described above](#value-and-array-columns). Each value will spread out to its own column in the group, other columns will contain `null`. @@ -504,7 +506,7 @@ For example, this is how the following JSON will be read (including `null` value ```kotlin -val text = """ +val complexClashedJson = """ [ { "a": "text" }, { "a": { "b": 2 } }, @@ -512,14 +514,15 @@ val text = """ ] """.trimIndent() -val df = DataFrame.readJsonStr(text) +val df = DataFrame.readJsonStr(complexClashedJson) ``` A clash at the top level has no property to group under, so there "value", "array", and the objects' properties -become sibling columns of the [`DataFrame`](DataFrame.md) itself: +become top-level columns of the [`DataFrame`](DataFrame.md) itself, +as in [usual standalone values reading](#value-and-array-columns): @@ -551,7 +554,7 @@ For this case, you can set `typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS` ```kotlin -val df = DataFrame.readJsonStr(text, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) +val df = DataFrame.readJsonStr(complexClashedJson, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) ``` diff --git a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt index 39e57119af..304c227645 100644 --- a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt +++ b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt @@ -436,7 +436,7 @@ class Read : DataFrameSampleHelper("read", "api") { @Test fun readJsonTypeClash() { // SampleStart - val text = """ + val complexClashedJson = """ [ { "a": "text" }, { "a": { "b": 2 } }, @@ -444,14 +444,14 @@ class Read : DataFrameSampleHelper("read", "api") { ] """.trimIndent() - val df = DataFrame.readJsonStr(text) + val df = DataFrame.readJsonStr(complexClashedJson) // SampleEnd .saveDfHtmlSample() } @Test fun readJsonTypeClashTactic() { - val text = """ + val complexClashedJson = """ [ { "a": "text" }, { "a": { "b": 2 } }, @@ -459,7 +459,7 @@ class Read : DataFrameSampleHelper("read", "api") { ] """.trimIndent() // SampleStart - val df = DataFrame.readJsonStr(text, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) + val df = DataFrame.readJsonStr(complexClashedJson, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) // SampleEnd .saveDfHtmlSample() } From 36b173865b166482551eaaa7407a5b26629fc630 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Tue, 22 Sep 2026 19:32:16 +0400 Subject: [PATCH 10/12] Update JSON IO docs --- .../dataframe/documentation/DocumentationUrls.kt | 3 +++ dataframe-json/build.gradle.kts | 1 + .../kotlinx/dataframe/impl/io/writeJson.kt | 4 ++-- .../org/jetbrains/kotlinx/dataframe/io/json.kt | 16 +++++++++------- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt index 78d0d0d781..dbae32a286 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt @@ -549,6 +549,9 @@ public interface DocumentationUrls { /** [See `JsonPath` on the documentation website.]({@include [Url]}/read.html#specify-key-value-paths) */ public typealias JsonPath = Nothing + /** [See the "value" and "array" columns on the documentation website.]({@include [Url]}/read.html#value-and-array-columns) */ + public typealias JsonValueAndArrayColumns = Nothing + /** [Deephaven CSV](https://github.com/deephaven/deephaven-csv) */ public typealias Deephaven = Nothing diff --git a/dataframe-json/build.gradle.kts b/dataframe-json/build.gradle.kts index d9b9916dcb..06640ae127 100644 --- a/dataframe-json/build.gradle.kts +++ b/dataframe-json/build.gradle.kts @@ -3,6 +3,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { with(conventions.plugins.dfbuild) { alias(kotlinJvm8) + alias(kodex) } with(libs.plugins) { alias(publisher) diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt index 81f8361115..e5f65bfe9b 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt @@ -283,8 +283,8 @@ internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { ?.takeIf { it.holdsAnyValue() } ?.takeUnless { valueCol -> val otherCols = allColumns - valueCol - // It's not a 'value' column if any of the values it holds sits next to - // a value in one of the other columns. + // A column is not considered a valid value column if any non-null value is joined + // with another value of another column in the same row. val isInvalidValueColumn = indices().any { row -> valueCol.holdsValueAt(row) && otherCols.any { it.holdsValueAt(row) } } diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt index 9e1953a84e..e3bcbd7bc8 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt @@ -18,6 +18,7 @@ import org.jetbrains.kotlinx.dataframe.codeGen.AbstractDefaultReadMethod import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup import org.jetbrains.kotlinx.dataframe.columns.FrameColumn +import org.jetbrains.kotlinx.dataframe.documentation.DocumentationUrls import org.jetbrains.kotlinx.dataframe.documentation.UnifyingNumbers import org.jetbrains.kotlinx.dataframe.impl.io.encodeDataFrameWithMetadata import org.jetbrains.kotlinx.dataframe.impl.io.encodeFrame @@ -117,7 +118,7 @@ public class JSON( * and the unwrapped properties of the objects the property can be. * * Each of these columns only holds a value for the records that actually have that shape; - * for all other records it holds `null`, which makes all of them nullable: + * for all other records it holds `null`, which makes all columns nullable: * * - `value` holds the JSON primitive, or `null` if the record is not a primitive. * - `array` holds the JSON array as a [List], or `null` if the record is not an array @@ -131,7 +132,7 @@ public class JSON( * * Note that a type clash is not the only source of `value`/`array` columns; at the top level they're created * for any JSON element that isn't an object. - * For more information: [See the "value" and "array" columns on the documentation website.](https://kotlin.github.io/dataframe/read.html#value-and-array-columns) + * For more information: {@include [DocumentationUrls.JsonValueAndArrayColumns]} * * [ANY_COLUMNS] will create a [DataFrame] looking like: * ``` @@ -498,11 +499,10 @@ public fun DataRow.Companion.readJsonStr( * * There's one exception, so that a [DataFrame] read from JSON with a top-level type clash can be written back * to its original form (see [TypeClashTactic]). Such a clash puts the elements that aren't objects into columns - * named "value" and "array" - * ([see them on the documentation website](https://kotlin.github.io/dataframe/read.html#value-and-array-columns)), - * beside the objects' own properties. When one of the top-level columns is detected - * as such a "value" or "array" column — it's named "value" or "array" and only holds a value in rows where every - * other column holds none — a row is written as the record it was read from: + * named "value" and "array", beside the objects' own properties. + * When one of the top-level columns is detected as such a "value" or "array" column — it's named "value" or + * "array" and only holds a value in rows where every other column holds none — a row is written as the record + * it was read from: * - the value of the "value" column, if the row has one; * - otherwise the array of the "array" column, if the row has one; * - otherwise an object of the remaining columns, if any of them holds a value; @@ -515,6 +515,8 @@ public fun DataRow.Companion.readJsonStr( * the user named that way. So a JSON array of non-objects, which is read into just a "value" or "array" column, * doesn't round-trip: `[1,2,3]` is written back as `[{"value":1},{"value":2},{"value":3}]`. * + * For more information: {@include [DocumentationUrls.JsonValueAndArrayColumns]} + * * Note too that some JSON records are read into the exact same [DataFrame], so writing it back can only produce * one of them: a row without any values is written as `null`, so `[1,{"label":"record"},{"label":null}]` becomes * `[1,{"label":"record"},null]`; and an empty array of objects is read as an empty nested [DataFrame], just like From 87f8e53e30de2dde37425bafdecef38f44c672b3 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Tue, 22 Sep 2026 19:47:41 +0400 Subject: [PATCH 11/12] Add `dataframe-json` to tracked `generated-sources` in workflows and documentation --- .github/workflows/generated-sources-master.yml | 2 +- .github/workflows/generated-sources.yml | 2 +- AGENTS.md | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generated-sources-master.yml b/.github/workflows/generated-sources-master.yml index 4f75fc3145..f1bceb72b3 100644 --- a/.github/workflows/generated-sources-master.yml +++ b/.github/workflows/generated-sources-master.yml @@ -27,7 +27,7 @@ jobs: run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' + git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-json/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' git diff --staged --quiet || git commit -m "Automated commit of generated code" git push env: diff --git a/.github/workflows/generated-sources.yml b/.github/workflows/generated-sources.yml index 35cd3a8e31..7b4d3d51bf 100644 --- a/.github/workflows/generated-sources.yml +++ b/.github/workflows/generated-sources.yml @@ -44,7 +44,7 @@ jobs: - name: Check for changes in generated sources id: git-diff run: | - git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' + git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-json/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' echo "changed=$(if git diff --staged --quiet; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT - name: Commit and push if changes diff --git a/AGENTS.md b/AGENTS.md index 23d7bbb31b..790ecf2aee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,7 +76,8 @@ Public KDocs use KoDEx notations (`{@include [X]}`, `@set`/`@get`/`$`, `@sample` KDocs. `core:processKDocsMain` processes KDocs into `generated-sources`; `changeJarTask` makes `sources.jar` use those. - **Don't hand-edit generated sources, and don't grep/crawl the tree.** Every module built with KoDEx (`core`, - `dataframe-csv`, `dataframe-excel`, `dataframe-geo`, `dataframe-jdbc`) has a tracked `generated-sources/` copy, + `dataframe-csv`, `dataframe-json`, `dataframe-excel`, `dataframe-geo`, `dataframe-jdbc`) has a tracked + `generated-sources/` copy, plus checked-in accessors under `**/src/generated-dataschema-accessors/` and generated **HTML** under `docs/StardustDocs/resources/`. In `generated-sources/` the **implementation and public API are the same** as the hand-written `src/main/kotlin/…` — KoDEx only expands the KDoc and strips the doc-only fragments (declarations From 09d85a0f226de836b132ad4c77afcd882c589468 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Mon, 28 Sep 2026 15:49:30 +0400 Subject: [PATCH 12/12] Refactor: Extract `debugChecksEnabled` for better readability and reuse --- .../org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt index ca5e9580a4..a58ecae28b 100644 --- a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt +++ b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt @@ -24,6 +24,9 @@ import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.typeOf class DataColumns : BaseTest() { + + private val debugChecksEnabled: Boolean = BuildConfig::class.java.getField("DEBUG").getBoolean(null) + @Test fun `create column with platform type from Api`() { val df1 = listOf(1, 2, 3).toDataFrame { @@ -48,7 +51,7 @@ class DataColumns : BaseTest() { @Test fun `allow no nulls in frame columns`() { - if (BuildConfig.DEBUG) { + if (debugChecksEnabled) { shouldThrow { DataColumn.createFrameColumn( name = "",