diff --git a/.github/workflows/generated-sources-master.yml b/.github/workflows/generated-sources-master.yml index ec3f13618f..486a706eb7 100644 --- a/.github/workflows/generated-sources-master.yml +++ b/.github/workflows/generated-sources-master.yml @@ -31,7 +31,7 @@ jobs: run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' + git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-json/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' git diff --staged --quiet || git commit -m "Automated commit of generated code" git push env: diff --git a/.github/workflows/generated-sources.yml b/.github/workflows/generated-sources.yml index 35cd3a8e31..7b4d3d51bf 100644 --- a/.github/workflows/generated-sources.yml +++ b/.github/workflows/generated-sources.yml @@ -44,7 +44,7 @@ jobs: - name: Check for changes in generated sources id: git-diff run: | - git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' + git add './core/generated-sources' './dataframe-csv/generated-sources' './dataframe-json/generated-sources' './dataframe-geo/generated-sources' './dataframe-excel/generated-sources' './dataframe-jdbc/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects' echo "changed=$(if git diff --staged --quiet; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT - name: Commit and push if changes diff --git a/AGENTS.md b/AGENTS.md index 23d7bbb31b..790ecf2aee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,7 +76,8 @@ Public KDocs use KoDEx notations (`{@include [X]}`, `@set`/`@get`/`$`, `@sample` KDocs. `core:processKDocsMain` processes KDocs into `generated-sources`; `changeJarTask` makes `sources.jar` use those. - **Don't hand-edit generated sources, and don't grep/crawl the tree.** Every module built with KoDEx (`core`, - `dataframe-csv`, `dataframe-excel`, `dataframe-geo`, `dataframe-jdbc`) has a tracked `generated-sources/` copy, + `dataframe-csv`, `dataframe-json`, `dataframe-excel`, `dataframe-geo`, `dataframe-jdbc`) has a tracked + `generated-sources/` copy, plus checked-in accessors under `**/src/generated-dataschema-accessors/` and generated **HTML** under `docs/StardustDocs/resources/`. In `generated-sources/` the **implementation and public API are the same** as the hand-written `src/main/kotlin/…` — KoDEx only expands the KDoc and strips the doc-only fragments (declarations diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt index 78d0d0d781..dbae32a286 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/DocumentationUrls.kt @@ -549,6 +549,9 @@ public interface DocumentationUrls { /** [See `JsonPath` on the documentation website.]({@include [Url]}/read.html#specify-key-value-paths) */ public typealias JsonPath = Nothing + /** [See the "value" and "array" columns on the documentation website.]({@include [Url]}/read.html#value-and-array-columns) */ + public typealias JsonValueAndArrayColumns = Nothing + /** [Deephaven CSV](https://github.com/deephaven/deephaven-csv) */ public typealias Deephaven = Nothing diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt index ca5e9580a4..a58ecae28b 100644 --- a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt +++ b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt @@ -24,6 +24,9 @@ import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.typeOf class DataColumns : BaseTest() { + + private val debugChecksEnabled: Boolean = BuildConfig::class.java.getField("DEBUG").getBoolean(null) + @Test fun `create column with platform type from Api`() { val df1 = listOf(1, 2, 3).toDataFrame { @@ -48,7 +51,7 @@ class DataColumns : BaseTest() { @Test fun `allow no nulls in frame columns`() { - if (BuildConfig.DEBUG) { + if (debugChecksEnabled) { shouldThrow { DataColumn.createFrameColumn( name = "", diff --git a/dataframe-json/build.gradle.kts b/dataframe-json/build.gradle.kts index d9b9916dcb..06640ae127 100644 --- a/dataframe-json/build.gradle.kts +++ b/dataframe-json/build.gradle.kts @@ -3,6 +3,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { with(conventions.plugins.dfbuild) { alias(kotlinJvm8) + alias(kodex) } with(libs.plugins) { alias(publisher) diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt index 74de9fa296..3b6715adc1 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt @@ -300,28 +300,12 @@ internal fun fromJsonListAnyColumns( isLenient = isLenient, ) - val res = when { - parsed.isSingleUnnamedColumn() -> { - val col = (parsed.getColumn(0) as UnnamedColumn).col - val elementType = col.type - val columnValues = col.values - .asList() - .splitByIndices(startIndices.asSequence()) - .toList() - DataColumn.createValueColumn( - name = ARRAY_COLUMN_NAME, - values = columnValues, - type = List::class.createType(listOf(KTypeProjection.invariant(elementType))), - ) - } - - else -> - parsed.unwrapUnnamedColumns() - .chunked( - startIndices = startIndices, - name = ARRAY_COLUMN_NAME, // will be erased - ) - } + val res = createArrayColumn( + parsed = parsed, + name = ARRAY_COLUMN_NAME, // will be erased + records = records, + startIndices = startIndices, + ) listOf(UnnamedColumn(res)) } @@ -450,6 +434,40 @@ internal fun fromJsonListAnyColumns( private fun AnyFrame.isSingleUnnamedColumn() = columnsCount() == 1 && getColumn(0) is UnnamedColumn +/** + * Collects the arrays of [records] into a single column named [name]: a `List` value column when the arrays + * hold single values, else a [FrameColumn] holding the objects of each array. + * + * @param parsed The contents of all arrays of [records], parsed and concatenated into one [DataFrame]. + * @param startIndices For each record, the index in [parsed] where its array starts, so that [parsed] can be + * split back into one entry per record. + */ +private fun createArrayColumn( + parsed: AnyFrame, + name: String, + records: List<*>, + startIndices: Iterable, +): DataColumn = + if (parsed.isSingleUnnamedColumn()) { + val col = (parsed.getColumn(0) as UnnamedColumn).col + val columnValues = col.values + .asList() + .splitByIndices(startIndices.asSequence()) + .toList() + // records that hold no array at all (like `null`) are not empty lists + .mapIndexed { i, list -> list.takeIf { records[i] is JsonArray } } + DataColumn.createValueColumn( + name = name, + values = columnValues, + type = List::class.createType( + arguments = listOf(KTypeProjection.invariant(col.type)), + nullable = records.any { it !is JsonArray }, + ), + ) + } else { + parsed.unwrapUnnamedColumns().chunked(startIndices, name) + } + /** * Json to DataFrame converter that creates allows creates `value` and `array` accessors * instead of [Any] columns. @@ -478,9 +496,9 @@ internal fun fromJsonListArrayAndValueColumns( // list element type can be JsonObject, JsonArray or primitive // So first, we gather all properties of objects to merge including "array" and "value" if needed // so the resulting type of a property with instances 123, ["abc"], and { "a": 1, "b": 2 } will be - // { array: List, value: Int?, a: Int?, b: Int? } + // { array: List?, value: Int?, a: Int?, b: Int? } // and instances will look like - // { "array": [], "value": 123, "a": null, "b": null } + // { "array": null, "value": 123, "a": null, "b": null } val nameGenerator = ColumnNameGenerator() records.forEach { record -> @@ -659,24 +677,12 @@ internal fun fromJsonListArrayAndValueColumns( jsonPath = jsonPath.appendArrayWithWildcard(), ) - val res = when { - parsed.isSingleUnnamedColumn() -> { - val col = (parsed.getColumn(0) as UnnamedColumn).col - val elementType = col.type - val columnValues = - col.values - .asList() - .splitByIndices(startIndices.asSequence()) - .toList() - DataColumn.createValueColumn( - name = colName, - values = columnValues, - type = List::class.createType(listOf(KTypeProjection.invariant(elementType))), - ) - } - - else -> parsed.unwrapUnnamedColumns().chunked(startIndices, colName) - } + val res = createArrayColumn( + parsed = parsed, + name = colName, + records = records, + startIndices = startIndices, + ) UnnamedColumn(res) } diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt index 26a8d6a549..e5f65bfe9b 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt @@ -20,9 +20,9 @@ import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.ColumnsContainer import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.api.FormattedFrame -import org.jetbrains.kotlinx.dataframe.api.all -import org.jetbrains.kotlinx.dataframe.api.allNulls +import org.jetbrains.kotlinx.dataframe.api.dataFrameOf import org.jetbrains.kotlinx.dataframe.api.indices +import org.jetbrains.kotlinx.dataframe.api.isEmpty import org.jetbrains.kotlinx.dataframe.api.isList import org.jetbrains.kotlinx.dataframe.api.schema import org.jetbrains.kotlinx.dataframe.api.take @@ -40,6 +40,7 @@ import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NROW import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.TYPE import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.TYPES import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.VERSION +import org.jetbrains.kotlinx.dataframe.indices import org.jetbrains.kotlinx.dataframe.io.ARRAY_COLUMN_NAME import org.jetbrains.kotlinx.dataframe.io.Base64ImageEncodingOptions import org.jetbrains.kotlinx.dataframe.io.CustomEncoder @@ -176,10 +177,11 @@ internal fun encodeValue(col: AnyCol, index: Int, customEncoders: List matchingEncoder.encode(col[index]) + // a `null` list is not an empty one, just like in `encodeRow` col.isList() -> col[index]?.let { list -> val values = (list as List<*>).map { convert(it) } JsonArray(values) - } ?: JsonArray(emptyList()) + } ?: JsonPrimitive(null) col.typeClass in valueTypes -> convert(col[index]) @@ -251,30 +253,19 @@ private fun createJsonTypeDescriptor(columnSchema: ColumnSchema): JsonObject = }, ) +/** + * Unlike [encodeFrame], this is the wire format of the Kotlin Notebook plugin, described in + * `docs/serialization_format.md`: every row has to be an object with one entry per column of [frame]. + * So the unnamed `value`/`array` columns are encoded as regular columns here, instead of as the JSON record + * they were read from. + */ internal fun encodeFrameWithMetadata( frame: AnyFrame, rowLimit: Int? = null, customEncoders: List = emptyList(), ): JsonArray { - val valueColumn = frame.extractValueColumn() - val arrayColumn = frame.extractArrayColumn() - - val arraysAreFrames = arrayColumn?.kind() == ColumnKind.Frame - val data = frame.indices().map { rowIndex -> - valueColumn?.get(rowIndex) - ?: arrayColumn?.get(rowIndex)?.let { - if (arraysAreFrames) { - encodeFrameWithMetadata( - it as AnyFrame, - rowLimit, - customEncoders, - ) - } else { - null - } - } - ?: encodeRowWithMetadata(frame, rowIndex, rowLimit, customEncoders) + encodeRowWithMetadata(frame, rowIndex, rowLimit, customEncoders) } return buildJsonArray { addAll(data.map { convert(it) }) } @@ -289,16 +280,16 @@ internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { ?.maxByOrNull { it.name } ?.takeIf { it.kind() == ColumnKind.Value } // 'readJson' cannot have created an all-null 'value' column, consider it a regular one - ?.takeUnless { it.allNulls() } - ?.takeIf { valueCol -> + ?.takeIf { it.holdsAnyValue() } + ?.takeUnless { valueCol -> val otherCols = allColumns - valueCol - // It's a valid 'value' column only if for each non-null value, - // the corresponding cells in the other columns are all null. - val isValidValueColumn = this.all { row -> - valueCol[row] == null || otherCols.all { it[row] == null } + // A column is not considered a valid value column if any non-null value is joined + // with another value of another column in the same row. + val isInvalidValueColumn = indices().any { row -> + valueCol.holdsValueAt(row) && otherCols.any { it.holdsValueAt(row) } } - isValidValueColumn + isInvalidValueColumn } } @@ -308,6 +299,11 @@ internal fun AnyFrame.extractValueColumn(): DataColumn<*>? { internal val AnyFrame.isPossibleToFindUnnamedColumns: Boolean get() = columns().size != 1 +// An unnamed `value`/`array` column always holds at least one value, as it's only created for +// records that actually hold a value or an array. +// A column of `null`s only was created by the user, like dataFrameOf("value" to listOf(null, null)). +private fun AnyCol.holdsAnyValue(): Boolean = indices.any { holdsValueAt(it) } + internal fun AnyFrame.extractArrayColumn(): DataColumn<*>? { val allColumns = columns() @@ -315,46 +311,71 @@ internal fun AnyFrame.extractArrayColumn(): DataColumn<*>? { .filter { it.name.startsWith(ARRAY_COLUMN_NAME) } .takeIf { isPossibleToFindUnnamedColumns } ?.maxByOrNull { it.name } - ?.takeUnless { it.kind() == ColumnKind.Group } + // 'readJson' stores arrays either as a `List` value column or as a frame column + ?.takeIf { it.kind() == ColumnKind.Frame || it.isList() } // 'readJson' cannot have created an all-null 'array' column, consider it a regular one - ?.takeUnless { it.allNulls() } - ?.takeIf { arrayCol -> + ?.takeIf { it.holdsAnyValue() } + ?.takeUnless { arrayCol -> val otherCols = allColumns - arrayCol - // It's a valid 'array' column only if for each non-null value, - // the corresponding cells in the other columns are all null. - // TODO `readJson` turns arrays into empty lists instead of `null`, issue #2048 - val isValidArrayColumn = this.all { row -> - arrayCol[row] == null || otherCols.all { it[row] == null } + // It's not an 'array' column if any of the arrays it holds sits next to + // a value in one of the other columns. + val isInvalidArrayColumn = indices().any { row -> + arrayCol.holdsValueAt(row) && otherCols.any { it.holdsValueAt(row) } } - isValidArrayColumn + + isInvalidArrayColumn } } +/** + * Encodes each row of [frame] as the JSON record it was read from: the value of the unnamed "value" column, + * else the array of the unnamed "array" column, else an object of the remaining columns, + * else — when the row holds no values at all — `null`. + */ internal fun encodeFrame(frame: AnyFrame): JsonArray { val valueColumn = frame.extractValueColumn() val arrayColumn = frame.extractArrayColumn() val arraysAreFrames = arrayColumn?.kind() == ColumnKind.Frame + // the unnamed `value`/`array` columns encode the record itself, all other columns encode it as an object + val objectColumns = frame.columns().filter { it.name != valueColumn?.name && it.name != arrayColumn?.name } + val hasUnnamedColumns = objectColumns.size != frame.columnsCount() + val objectFrame = dataFrameOf(objectColumns) + val data = frame.indices().map { rowIndex -> when { - valueColumn != null -> valueColumn[rowIndex] + valueColumn?.holdsValueAt(rowIndex) == true -> valueColumn[rowIndex] - arrayColumn != null -> arrayColumn[rowIndex]?.let { + arrayColumn?.holdsValueAt(rowIndex) == true -> if (arraysAreFrames) { - encodeFrame(it as AnyFrame) + encodeFrame(arrayColumn[rowIndex] as AnyFrame) } else { - null + JsonArray((arrayColumn[rowIndex] as List<*>).map { convert(it) }) } - } - else -> encodeRow(frame, rowIndex) + // a record that's neither a single value nor an array is encoded as an object, + // unless it holds no values at all; then the record was `null` itself + hasUnnamedColumns && objectColumns.none { it.holdsValueAt(rowIndex) } -> null + + else -> encodeRow(objectFrame, rowIndex) } } return buildJsonArray { addAll(data.map { convert(it) }) } } +/** + * Whether the value of this column at [rowIndex] carries any data; + * empty [FrameColumn] values and [ColumnGroup] rows filled with `null`s carry none. + */ +private fun AnyCol.holdsValueAt(rowIndex: Int): Boolean = + when (this) { + is ColumnGroup<*> -> columns().any { it.holdsValueAt(rowIndex) } + is FrameColumn<*> -> !this[rowIndex].isEmpty() + else -> this[rowIndex] != null + } + internal fun encodeDataFrameWithMetadata( frame: AnyFrame, rowLimit: Int, diff --git a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt index 668e51676b..e3bcbd7bc8 100644 --- a/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt +++ b/dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt @@ -18,6 +18,7 @@ import org.jetbrains.kotlinx.dataframe.codeGen.AbstractDefaultReadMethod import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup import org.jetbrains.kotlinx.dataframe.columns.FrameColumn +import org.jetbrains.kotlinx.dataframe.documentation.DocumentationUrls import org.jetbrains.kotlinx.dataframe.documentation.UnifyingNumbers import org.jetbrains.kotlinx.dataframe.impl.io.encodeDataFrameWithMetadata import org.jetbrains.kotlinx.dataframe.impl.io.encodeFrame @@ -103,19 +104,36 @@ public class JSON( * ] * ``` * - * [ARRAY_AND_VALUE_COLUMNS] (default) will create a [DataFrame] looking like (including `null` and `[]` values): + * [ARRAY_AND_VALUE_COLUMNS] (default) will create a [DataFrame] looking like (including `null` values): * ``` - * ⌌----------------------------------------------⌍ - * | | a:{b:Int?, value:String?, array:List}| - * |--|-------------------------------------------| - * | 0| { b:null, value:"text", array:[] }| - * | 1| { b:2, value:null, array:[] }| - * | 2| { b:null, value:null, array:[6, 7, 8] }| - * ⌎----------------------------------------------⌏ + * ⌌-----------------------------------------------⌍ + * | | a:{b:Int?, value:String?, array:List?}| + * |--|--------------------------------------------| + * | 0| { b:null, value:"text", array:null }| + * | 1| { b:2, value:null, array:null }| + * | 2| { b:null, value:null, array:[6, 7, 8] }| + * ⌎-----------------------------------------------⌏ * ``` * So, for the type clashing argument it will create a [ColumnGroup] with the properties `value`, `array`, * and the unwrapped properties of the objects the property can be. * + * Each of these columns only holds a value for the records that actually have that shape; + * for all other records it holds `null`, which makes all columns nullable: + * + * - `value` holds the JSON primitive, or `null` if the record is not a primitive. + * - `array` holds the JSON array as a [List], or `null` if the record is not an array + * (note that this is different from an empty JSON array `[]`, which is read as an empty list). + * When the arrays contain objects, `array` becomes a [FrameColumn] instead, in which a record + * without an array is an empty [DataFrame], as frame columns cannot hold `null`. + * - the unwrapped object properties hold `null` if the record is not an object, + * or if the object does not have that property. + * + * A JSON `null` record, as well as a missing property, is `null` in all these columns. + * + * Note that a type clash is not the only source of `value`/`array` columns; at the top level they're created + * for any JSON element that isn't an object. + * For more information: {@include [DocumentationUrls.JsonValueAndArrayColumns]} + * * [ANY_COLUMNS] will create a [DataFrame] looking like: * ``` * ⌌-------------⌍ @@ -473,6 +491,40 @@ public fun DataRow.Companion.readJsonStr( unifyNumbers = unifyNumbers, ).single() +/** + * Converts this [DataFrame] to a JSON array; each row becomes an element of that array. + * + * By default, a row is written as a JSON object: a [ColumnGroup] becomes a nested object, + * a [FrameColumn] becomes a nested array of objects, and a [List] value becomes a JSON array. + * + * There's one exception, so that a [DataFrame] read from JSON with a top-level type clash can be written back + * to its original form (see [TypeClashTactic]). Such a clash puts the elements that aren't objects into columns + * named "value" and "array", beside the objects' own properties. + * When one of the top-level columns is detected as such a "value" or "array" column — it's named "value" or + * "array" and only holds a value in rows where every other column holds none — a row is written as the record + * it was read from: + * - the value of the "value" column, if the row has one; + * - otherwise the array of the "array" column, if the row has one; + * - otherwise an object of the remaining columns, if any of them holds a value; + * - otherwise `null`. + * + * Only a *top-level* clash is restored like this: + * - A clash inside a [ColumnGroup] isn't: the group is written as an object, so its "value" and "array" columns + * show up as ordinary properties. + * - A single-column [DataFrame] isn't either, since a lone "value"/"array" column is indistinguishable from one + * the user named that way. So a JSON array of non-objects, which is read into just a "value" or "array" column, + * doesn't round-trip: `[1,2,3]` is written back as `[{"value":1},{"value":2},{"value":3}]`. + * + * For more information: {@include [DocumentationUrls.JsonValueAndArrayColumns]} + * + * Note too that some JSON records are read into the exact same [DataFrame], so writing it back can only produce + * one of them: a row without any values is written as `null`, so `[1,{"label":"record"},{"label":null}]` becomes + * `[1,{"label":"record"},null]`; and an empty array of objects is read as an empty nested [DataFrame], just like + * no array at all, so `[{"label":"record"},[{"a":1}],[]]` becomes `[{"label":"record"},[{"a":1}],null]`. + * + * @param prettyPrint Whether to format the output with indentation and line breaks. `false` by default. + * @return This [DataFrame] as a JSON string. + */ public fun AnyFrame.toJson(prettyPrint: Boolean = false): String { val json = Json { this.prettyPrint = prettyPrint diff --git a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt index 252a4b39d0..839e6be1c5 100644 --- a/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt +++ b/dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/JsonTests.kt @@ -32,6 +32,7 @@ import org.jetbrains.kotlinx.dataframe.api.convert import org.jetbrains.kotlinx.dataframe.api.dataFrameOf import org.jetbrains.kotlinx.dataframe.api.forEach import org.jetbrains.kotlinx.dataframe.api.format +import org.jetbrains.kotlinx.dataframe.api.getColumn import org.jetbrains.kotlinx.dataframe.api.getColumnGroup import org.jetbrains.kotlinx.dataframe.api.getColumns import org.jetbrains.kotlinx.dataframe.api.getFrameColumn @@ -171,7 +172,8 @@ class JsonTests { group.columnsCount() shouldBe 3 group["b"].type() shouldBe typeOf() group["value"].type() shouldBe typeOf() - group["array"].type() shouldBe typeOf>() + group["array"].type() shouldBe typeOf?>() + group["array"].values().toList() shouldBe listOf(null, null, listOf(6, 7, 8)) } @Test @@ -209,8 +211,8 @@ class JsonTests { val df = DataFrame.readJsonStr(json).alsoDebug() df.columnsCount() shouldBe 1 df.rowsCount() shouldBe 3 - df["a"].type() shouldBe typeOf>() - df[1]["a"] shouldBe emptyList() + df["a"].type() shouldBe typeOf?>() + df[1]["a"] shouldBe null } @Test @@ -227,8 +229,8 @@ class JsonTests { val df = DataFrame.readJsonStr(json, typeClashTactic = ANY_COLUMNS).alsoDebug() df.columnsCount() shouldBe 1 df.rowsCount() shouldBe 3 - df["a"].type() shouldBe typeOf>() - df[1]["a"] shouldBe emptyList() + df["a"].type() shouldBe typeOf?>() + df[1]["a"] shouldBe null } @Test @@ -491,7 +493,8 @@ class JsonTests { val df = dataFrameOf("a")(listOf(1, 2, 3), null) val text = df.toJson() val df1 = DataFrame.readJsonStr(text) - df1["a"][1] shouldBe emptyList() + df1["a"][1] shouldBe null + df1 shouldBe df } @Test @@ -499,7 +502,8 @@ class JsonTests { val df = dataFrameOf("a")(listOf(1, 2, 3), null) val text = df.toJson() val df1 = DataFrame.readJsonStr(text, typeClashTactic = ANY_COLUMNS) - df1["a"][1] shouldBe emptyList() + df1["a"][1] shouldBe null + df1 shouldBe df } @Test @@ -574,8 +578,8 @@ class JsonTests { @Language("json") val mixedJson = """[{"label":"record"},0,null,null]""" - // TODO Issue #2045 - // DataFrame.readJsonStr(mixedJson).toJson() shouldBe """[{"label":"record"},0,null,null]""" + // Issue #2045 + DataFrame.readJsonStr(mixedJson).toJson() shouldBe """[{"label":"record"},0,null,null]""" // Even though `value` was created by `readJson`, taking a slice makes it all-null. // We can no longer tell the difference between a generated 'value' and a regular column that was named 'value' @@ -783,7 +787,7 @@ class JsonTests { schema shouldNotContain "Void?" group["array1"].type() shouldBe typeOf() - group["array2"].type() shouldBe typeOf>() + group["array2"].type() shouldBe typeOf?>() } @Test @@ -865,11 +869,13 @@ class JsonTests { df.columnsCount() shouldBe 1 df.rowsCount() shouldBe 6 val a = df["a"] as ValueColumn<*> - a.type() shouldBe typeOf>() + a.type() shouldBe typeOf?>() a[0] shouldBe listOf(1, 2, 3) a[1] shouldBe listOf(null) - a[2..5].forEach { - it shouldBe emptyList() + a[2] shouldBe emptyList() + // json `null` and missing values are read as `null`, not as an empty list + a[3..5].forEach { + it shouldBe null } } @@ -973,12 +979,12 @@ class JsonTests { // ⌌-------------------------------------------------------⌍ // | | a:{b:{value:Int?, array:List}, c:Int?, d:Any?}| // |--|----------------------------------------------------| -// | 0| { b:{ value:1, array:[] }, c:null, d:null }| +// | 0| { b:{ value:1, array:null }, c:null, d:null }| // | 1| { b:{ value:null, array:[1, 2, 3] }, c:2, d:null }| -// | 2| { b:{ value:null, array:[] }, c:null, d:null }| -// | 3| { b:{ value:null, array:[] }, c:null, d:null }| -// | 4| { b:{ value:null, array:[] }, c:null, d:null }| -// | 5| { b:{ value:null, array:[] }, c:null, d:null }| +// | 2| { b:{ value:null, array:null }, c:null, d:null }| +// | 3| { b:{ value:null, array:null }, c:null, d:null }| +// | 4| { b:{ value:null, array:null }, c:null, d:null }| +// | 5| { b:{ value:null, array:null }, c:null, d:null }| // ⌎-------------------------------------------------------⌏ noKeyValue.columnsCount() shouldBe 1 noKeyValue.rowsCount() shouldBe 6 @@ -988,12 +994,12 @@ class JsonTests { it["b"].type() shouldBe typeOf>() it["b"]["value"].type() shouldBe typeOf() - it["b"]["array"].type() shouldBe typeOf>() + it["b"]["array"].type() shouldBe typeOf?>() it["c"].type() shouldBe typeOf() it["d"].type() shouldBe nothingType(nullable = true) it[0].let { - (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to 1, "array" to emptyList()) + (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to 1, "array" to null) it["c"] shouldBe null it["d"] shouldBe null } @@ -1004,7 +1010,8 @@ class JsonTests { } (it as ColumnGroup<*>)[2..5].forEach { it.let { - (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to null, "array" to emptyList()) + // json `null` and missing values are read as `null`, not as an empty list + (it["b"] as DataRow<*>).toMap() shouldBe mapOf("value" to null, "array" to null) it["c"] shouldBe null it["d"] shouldBe null } @@ -1361,6 +1368,318 @@ class JsonTests { DataFrame.readJsonStr(json) shouldBe df } + @Test + fun `value and array columns are created for nameless json elements`() { + for (typeClashTactic in JSON.TypeClashTactic.entries) { + fun read( + @Language("json") json: String, + ) = DataFrame.readJsonStr(json, typeClashTactic = typeClashTactic) + + read("""[1, 2, 3]""").let { + it.columnNames() shouldBe listOf("value") + it["value"].type() shouldBe typeOf() + } + read("""[[1], [2]]""").let { + it.columnNames() shouldBe listOf("array") + it["array"].type() shouldBe typeOf>() + } + read("""[[{ "a": 1 }], [{ "a": 2 }]]""").let { + it.columnNames() shouldBe listOf("array") + it["array"] shouldBe instanceOf>() + } + + // a property keeps its own name when there's no clash + read("""[{ "a": [1, 2] }, { "a": [3] }]""")["a"].type() shouldBe typeOf>() + read("""[{ "a": { "b": 1 } }, { "a": { "b": 2 } }]""")["a"].let { + it shouldBe instanceOf>() + (it as ColumnGroup<*>).columnNames() shouldBe listOf("b") + } + } + + // a top-level clash has no property to group under, so the columns end up side by side + DataFrame.readJsonStr("""[1, { "label": "record" }]""").let { + it["value"].type() shouldBe typeOf() + it["label"].type() shouldBe typeOf() + } + } + + @Test + fun `a lone value or array column is not an unnamed column`() { + DataFrame.readJsonStr("""[1,2,3]""").toJson() shouldBe """[{"value":1},{"value":2},{"value":3}]""" + DataFrame.readJsonStr("""[[1],[2]]""").toJson() shouldBe """[{"array":[1]},{"array":[2]}]""" + + // as soon as a clash gives the frame a second column, the round trip holds again + DataFrame.readJsonStr("""[1,{"label":"record"}]""").toJson() shouldBe """[1,{"label":"record"}]""" + DataFrame.readJsonStr("""[{"label":"record"},[1]]""").toJson() shouldBe """[{"label":"record"},[1]]""" + } + + @Test + fun `null is not read as empty list in array column`() { + // https://github.com/Kotlin/dataframe/issues/2048 + @Language("json") + val json = """[null, [123], []]""" + + for (typeClashTactic in JSON.TypeClashTactic.entries) { + DataFrame.readJsonStr(json, typeClashTactic = typeClashTactic).alsoDebug().let { + it.columnsCount() shouldBe 1 + it.rowsCount() shouldBe 3 + it.getColumn(0).let { + it.type() shouldBe typeOf?>() + it[0] shouldBe null + it[1] shouldBe listOf(123) + it[2] shouldBe emptyList() + } + } + } + } + + @Test + fun `null in nested array column`() { + // https://github.com/Kotlin/dataframe/issues/2048 + @Language("json") + val json = + """ + [ + {"a":[1,2,3]}, + {"a":[]}, + {"a":null}, + {} + ] + """.trimIndent() + + for (typeClashTactic in JSON.TypeClashTactic.entries) { + DataFrame.readJsonStr(json, typeClashTactic = typeClashTactic).alsoDebug().let { + it.columnsCount() shouldBe 1 + it.rowsCount() shouldBe 4 + it["a"].let { + it.type() shouldBe typeOf?>() + it[0] shouldBe listOf(1, 2, 3) + it[1] shouldBe emptyList() + it[2] shouldBe null + it[3] shouldBe null + } + } + } + } + + @Test + fun `value column round trip keeps the other columns`() { + // https://github.com/Kotlin/dataframe/issues/2045 + @Language("json") + val json = """[1,{"label":"record"},null]""" + + val df = DataFrame.readJsonStr(json).alsoDebug() + df["value"].type() shouldBe typeOf() + df["label"].type() shouldBe typeOf() + + df.toJson() shouldBe json + } + + @Test + fun `array column round trip keeps the other columns`() { + // https://github.com/Kotlin/dataframe/issues/2046 + @Language("json") + val json = """[{"label":"record"},[123]]""" + + val df = DataFrame.readJsonStr(json).alsoDebug() + df["array"].type() shouldBe typeOf?>() + df["array"][0] shouldBe null + df["label"].type() shouldBe typeOf() + + df.toJson() shouldBe json + } + + @Test + fun `array of objects column round trip keeps the other columns`() { + // https://github.com/Kotlin/dataframe/issues/2046 + @Language("json") + val json = """[{"label":"record"},[{"a":123}]]""" + + val df = DataFrame.readJsonStr(json).alsoDebug() + df["array"] shouldBe instanceOf>() + + df.toJson() shouldBe json + } + + @Test + fun `value and array column round trip`() { + // https://github.com/Kotlin/dataframe/issues/2045, https://github.com/Kotlin/dataframe/issues/2046 + @Language("json") + val json = """[1,{"label":"record"},[123],null,[]]""" + + DataFrame.readJsonStr(json).alsoDebug().toJson() shouldBe json + + // the example documented in docs/StardustDocs/topics/write.md + @Language("json") + val documented = """[1,{"label":"record"},[123],null]""" + + DataFrame.readJsonStr(documented).toJson() shouldBe documented + } + + @Test + fun `a null record and an object of nulls only are indistinguishable`() { + // https://github.com/Kotlin/dataframe/issues/2045 + // both jsons are read into the exact same DataFrame, so writing it back can only produce one of them + val nullRecord = DataFrame.readJsonStr("""[1,{"label":"record"},null]""").alsoDebug() + val objectOfNulls = DataFrame.readJsonStr("""[1,{"label":"record"},{"label":null}]""").alsoDebug() + + objectOfNulls shouldBe nullRecord + + // a row without any values is written back as a `null` record + nullRecord.toJson() shouldBe """[1,{"label":"record"},null]""" + objectOfNulls.toJson() shouldBe """[1,{"label":"record"},null]""" + + // without a type clash there's no unnamed `value`/`array` column, so rows stay objects + DataFrame.readJsonStr("""[{"label":"record"},{"label":null}]""") + .toJson() shouldBe """[{"label":"record"},{"label":null}]""" + } + + /** + * Asserts that [this] is serialized according to `docs/serialization_format.md`: every entry of + * `kotlin_dataframe` — and of every nested group and frame — is an object holding exactly the columns + * that the accompanying metadata declares. + */ + private fun AnyFrame.shouldMatchSerializationFormat(): List { + fun checkRows(rows: List, declaredColumns: List) { + rows.forEach { row -> + // fails right here if a row is a bare value, a bare array or `null` instead of an object + row.jsonObject.keys shouldBe declaredColumns.toSet() + + row.jsonObject.values.forEach { cell -> + val metadata = (cell as? JsonObject)?.get(METADATA)?.jsonObject ?: return@forEach + val nestedColumns = metadata[COLUMNS]!!.jsonArray.map { it.jsonPrimitive.content } + when (metadata[KIND]!!.jsonPrimitive.content) { + ColumnKind.Group.toString() -> checkRows(listOf(cell.jsonObject[DATA]!!), nestedColumns) + ColumnKind.Frame.toString() -> checkRows(cell.jsonObject[DATA]!!.jsonArray, nestedColumns) + } + } + } + } + + val json = parseJsonStr(toJsonWithMetadata(rowsCount())) + val data = json[KOTLIN_DATAFRAME]!!.jsonArray + val columns = json[METADATA]!!.jsonObject[COLUMNS]!!.jsonArray.map { it.jsonPrimitive.content } + + columns shouldBe columnNames() + checkRows(data, columns) + return data + } + + @Test + fun `a type clash keeps every declared column with metadata`() { + // https://github.com/Kotlin/dataframe/issues/2046 + // `toJsonWithMetadata` is the Kotlin Notebook plugin's wire format (docs/serialization_format.md): + // unlike `toJson`, it must not collapse a row to the JSON record it was read from — the plugin renders + // cells by looking up `metadata.columns`, so the JSON has to mirror the dataframe structure exactly + DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""").let { df -> + df.shouldMatchSerializationFormat().toString() shouldBe + """[{"label":null,"value":1,"array":null},{"label":"record","value":null,"array":null},""" + + """{"label":null,"value":null,"array":[123]},{"label":null,"value":null,"array":null}]""" + + // and `toJson` does collapse them, that's the point of the two being different + df.toJson() shouldBe """[1,{"label":"record"},[123],null]""" + } + + // an `array` frame column becomes a NestedFrame, and the sibling `label` column is kept + DataFrame.readJsonStr("""[{"label":"record"},[{"a":123}]]""").let { df -> + val data = df.shouldMatchSerializationFormat() + + data[0].jsonObject["label"]!!.jsonPrimitive.content shouldBe "record" + data[1].jsonObject["array"]!! + .jsonObject[DATA]!! + .jsonArray + .toString() shouldBe """[{"a":123}]""" + + df.toJson() shouldBe """[{"label":"record"},[{"a":123}]]""" + } + } + + @Test + fun `json with metadata mirrors the dataframe structure`() { + // a nested type clash, so the `value`/`array` columns sit inside a column group + DataFrame.readJsonStr("""[{"a":"text"},{"a":{"b":2}},{"a":[6,7,8]}]""").shouldMatchSerializationFormat() + + // arrays of objects, so `array` is a frame column nested in a group + DataFrame.readJsonStr("""[{"a":"text"},{"a":[{"b":2}]}]""").shouldMatchSerializationFormat() + + // a single unnamed column, and one that holds no values at all + DataFrame.readJsonStr("""[1,2,3]""").shouldMatchSerializationFormat() + DataFrame.readJsonStr("""[null,null]""").shouldMatchSerializationFormat() + + // a real file with groups and frame columns several levels deep + DataFrame.readJson(testJson("repositories")).shouldMatchSerializationFormat() + } + + @Test + fun `a null list is written as null with metadata too`() { + fun AnyFrame.dataWithMetadata(): String = + parseJsonStr(toJsonWithMetadata(rowsCount()))[KOTLIN_DATAFRAME]!!.jsonArray.toString() + + val df = DataFrame.readJsonStr("""[{"a":[1]},{"a":[]},{"a":null}]""").alsoDebug() + df["a"].type() shouldBe typeOf?>() + + val expected = """[{"a":[1]},{"a":[]},{"a":null}]""" + df.toJson() shouldBe expected + df.dataWithMetadata() shouldBe expected + + // a user-made nullable `List` column behaves the same + dataFrameOf( + "a" to columnOf?>(listOf(1), emptyList(), null), + ).dataWithMetadata() shouldBe expected + } + + @Test + fun `a column named array that holds no arrays is not an unnamed column`() { + // a user-made `array` column of single values must not be encoded as if it held JSON arrays + dataFrameOf( + "array" to columnOf(1, null), + "name" to columnOf(null, "x"), + ).toJson() shouldBe """[{"array":1,"name":null},{"array":null,"name":"x"}]""" + } + + @Test + fun `an empty array of objects is written back as null`() { + // a frame column cannot hold `null`, so "no array here" and "an empty array here" are both + // read as an empty DataFrame and can only be written back as one of the two + val df = DataFrame.readJsonStr("""[{"label":"record"},[{"a":1}],[]]""").alsoDebug() + df["array"] shouldBe instanceOf>() + + df.toJson() shouldBe """[{"label":"record"},[{"a":1}],null]""" + + // arrays of primitives are read into a nullable `List` column, so there the two stay distinct + DataFrame.readJsonStr("""[{"a":[1]},{"a":[]},{"a":null}]""") + .toJson() shouldBe """[{"a":[1]},{"a":[]},{"a":null}]""" + } + + @Test + fun `a nested type clash keeps its value and array columns`() { + // https://github.com/Kotlin/dataframe/issues/2045 + // unlike a top-level clash, a clash inside a column group is written back as the group's object, + // so the unnamed `value`/`array` columns show up as regular properties + DataFrame.readJsonStr("""[{"a":"text"},{"a":{"b":2}},{"a":[6,7,8]}]""").alsoDebug().toJson() shouldBe + """ + [{"a":{"b":null,"value":"text","array":null}},{"a":{"b":2,"value":null,"array":null}},{"a":{"b":null,"value":null,"array":[6,7,8]}}] + """.trimIndent() + + // the example documented in docs/StardustDocs/topics/write.md + DataFrame.readJsonStr("""[{"a":"text"},{"a":[6,7,8]}]""").toJson() shouldBe + """[{"a":{"value":"text","array":null}},{"a":{"value":null,"array":[6,7,8]}}]""" + } + + @Test + fun `user-made value and array columns of nulls only are not unnamed columns`() { + // https://github.com/Kotlin/dataframe/issues/2045 + dataFrameOf( + "value" to columnOf(null, null), + "name" to columnOf("Alice", "Bob"), + ).toJson() shouldBe """[{"value":null,"name":"Alice"},{"value":null,"name":"Bob"}]""" + + dataFrameOf( + "array" to columnOf?>(null, null), + "name" to columnOf("Alice", "Bob"), + ).toJson() shouldBe """[{"array":null,"name":"Alice"},{"array":null,"name":"Bob"}]""" + } + @Test fun `parse invalid literal`() { // https://github.com/Kotlin/kotlinx.serialization/issues/2511 diff --git a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt index d4bac2d410..bed48f4122 100644 --- a/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt +++ b/dataframe-openapi/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/convertDataRowsWithOpenApi.kt @@ -7,6 +7,7 @@ import org.jetbrains.kotlinx.dataframe.api.convert import org.jetbrains.kotlinx.dataframe.api.convertTo import org.jetbrains.kotlinx.dataframe.api.with import kotlin.reflect.KType +import kotlin.reflect.full.withNullability import kotlin.reflect.jvm.jvmErasure import kotlin.reflect.typeOf @@ -33,6 +34,17 @@ public fun ConvertSchemaDsl<*>.convertDataRowsWithOpenApi() { it } } + + // A missing or `null` json array is read as `null`, so read it as an empty list + // when the schema requires the list to be non-nullable. + convertIf({ fromType, toSchema -> + fromType.isMarkedNullable && + !toSchema.type.isMarkedNullable && + fromType.jvmErasure == List::class && + fromType.withNullability(false) == toSchema.type + }) { + it ?: emptyList() + } } /** diff --git a/docs/StardustDocs/resources/api/read/readJsonArrayColumn.html b/docs/StardustDocs/resources/api/read/readJsonArrayColumn.html new file mode 100644 index 0000000000..6b7302f05d --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonArrayColumn.html @@ -0,0 +1,510 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/resources/api/read/readJsonArrayOfObjectsColumn.html b/docs/StardustDocs/resources/api/read/readJsonArrayOfObjectsColumn.html new file mode 100644 index 0000000000..9ce22aa863 --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonArrayOfObjectsColumn.html @@ -0,0 +1,520 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/resources/api/read/readJsonTopLevelTypeClash.html b/docs/StardustDocs/resources/api/read/readJsonTopLevelTypeClash.html new file mode 100644 index 0000000000..a2ecfaaa6e --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonTopLevelTypeClash.html @@ -0,0 +1,511 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/resources/api/read/readJsonTypeClash.html b/docs/StardustDocs/resources/api/read/readJsonTypeClash.html index 5e8693e372..6f78b6ef3e 100644 --- a/docs/StardustDocs/resources/api/read/readJsonTypeClash.html +++ b/docs/StardustDocs/resources/api/read/readJsonTypeClash.html @@ -459,7 +459,7 @@ /**/ diff --git a/docs/StardustDocs/resources/api/read/readJsonValueColumn.html b/docs/StardustDocs/resources/api/read/readJsonValueColumn.html new file mode 100644 index 0000000000..451a9f6c44 --- /dev/null +++ b/docs/StardustDocs/resources/api/read/readJsonValueColumn.html @@ -0,0 +1,510 @@ + + + + + +
+ +

+ + + diff --git a/docs/StardustDocs/topics/_shadow_resources.md b/docs/StardustDocs/topics/_shadow_resources.md index 2b19d28a10..7d244123b0 100644 --- a/docs/StardustDocs/topics/_shadow_resources.md +++ b/docs/StardustDocs/topics/_shadow_resources.md @@ -262,11 +262,15 @@ + + + + diff --git a/docs/StardustDocs/topics/dataSources/JSON.md b/docs/StardustDocs/topics/dataSources/JSON.md index 0eac509407..95a7976fe1 100644 --- a/docs/StardustDocs/topics/dataSources/JSON.md +++ b/docs/StardustDocs/topics/dataSources/JSON.md @@ -48,6 +48,20 @@ val df = DataFrame.readJson("https://kotlin.github.io/dataframe/resources/exampl +A JSON `null` is read as `null`, so the corresponding column becomes nullable. A property that some records +simply don't have is read as `null` as well — a [`DataFrame`](DataFrame.md) is rectangular, so every row needs +a value in every column. + +This also holds for array columns: `[null, [123], []]` is read as a `List?` column holding `null`, `[123]`, +and an empty list. + +JSON elements that aren't objects have no property name to use as a column name, so they're read into the +special [`value` and `array` columns](read.md#value-and-array-columns). +The same special columns are also used +to resolve [type clashes](read.md#manage-type-clashes) +when the same property contains values of different shapes +across records. + ## Write You can write a [`DataFrame`](DataFrame.md) to a JSON file using the [`writeJson()`](write.md#writing-to-json) method: @@ -59,3 +73,6 @@ df.writeJson("example.json") ``` + +A [`DataFrame`](DataFrame.md) read from JSON with a top-level type clash is written back to its original form; +see [writing to JSON](write.md#writing-to-json) for the details and for the cases that don't round-trip. diff --git a/docs/StardustDocs/topics/io/read.md b/docs/StardustDocs/topics/io/read.md index 4f00cca5c9..b54d974315 100644 --- a/docs/StardustDocs/topics/io/read.md +++ b/docs/StardustDocs/topics/io/read.md @@ -441,25 +441,72 @@ has the `Double` type because it's the smallest unified number type for `Int` an ### JSON parsing options -#### Manage type clashes +#### The "value" and "array" columns for standalone values {id="value-and-array-columns"} + +A JSON element that isn't an object has no property name to be used as a column name, so the reader falls back to +two fixed names: + +* "value" holds the JSON element if it's a primitive, else it's `null`. +* "array" holds the JSON element as a list if it's an array, else it's `null`. + +They're created in two situations. The first is **at the top level**, where the elements of the JSON array are the +rows themselves and so have no name at all. This happens regardless of the `typeClashTactic` and even when there's +no clash to begin with. + +* An array of primitives is read into a single "value" column: + + + +```kotlin +val df = DataFrame.readJsonStr("""[1, 2, 3]""") +``` + + + + +* An array of arrays, into a single "array" column of lists: + + + +```kotlin +val df = DataFrame.readJsonStr("""[[1], [2]]""") +``` + + + -By default, if a type clash occurs when reading JSON, a new [`column group`](DataColumn.md#columngroup) is created consisting of: "value", "array", and -any number of object properties: +* When those arrays hold objects, "array" becomes a [`FrameColumn`](DataColumn.md#framecolumn): -* "value" will be set to the value of the JSON element if it's a primitive, else it will be `null`. -* "array" will be set to the array of values if the JSON element is an array, else it will be `[]`. + -If the JSON element is an object, then each property will spread out to its own column in the group, else these columns -will be `null`. +```kotlin +val df = DataFrame.readJsonStr("""[[{ "a": 1 }], [{ "a": 2 }]]""") +``` + + + + +The second situation is a **type clash**, described below. + +#### Manage type clashes + +By default, if a type clash occurs when reading JSON — the same property holds elements of different shapes across +records (objects, values or arrays) +— a new [`column group`](DataColumn.md#columngroup) is created for that property. +This column group contains a column for each children object property +(across all records), and +"value" and "array" columns following the logic of reading row values and arrays as +[described above](#value-and-array-columns). +Each value will spread out to its own column in the group, other columns will contain `null`. In this case `typeClashTactic = JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS`. -For example, this is how the following JSON will be read (including `null` and `[]` values): +For example, this is how the following JSON will be read (including `null` values): ```kotlin -val text = """ +val complexClashedJson = """ [ { "a": "text" }, { "a": { "b": 2 } }, @@ -467,13 +514,39 @@ val text = """ ] """.trimIndent() -val df = DataFrame.readJsonStr(text) +val df = DataFrame.readJsonStr(complexClashedJson) ``` -This makes it more convenient to work with the data, but it can be confusing if you're not expecting it or if you +A clash at the top level has no property to group under, so there "value", "array", and the objects' properties +become top-level columns of the [`DataFrame`](DataFrame.md) itself, +as in [usual standalone values reading](#value-and-array-columns): + + + +```kotlin +val df = DataFrame.readJsonStr("""[1, { "label": "record" }]""") +``` + + + + +Without a clash there's no such group: a property whose elements are all arrays simply becomes a `List` column under +its own name, and one whose elements are all objects becomes a group of just those objects' properties. + +Since every one of these columns only holds a value for the elements that actually have that shape, all of them +are nullable. A JSON `null` element, as well as a missing property, is `null` in all of them. + +> An empty JSON array `[]` is read as an empty list, which is different from `null`, meaning "there is no array here". +> +> When the arrays contain objects, "array" becomes a [`FrameColumn`](DataColumn.md#framecolumn) instead. There, an +> element without an array is an empty [`DataFrame`](DataFrame.md), because frame columns cannot hold `null`. +> {style="note"} + +Default type clash tactic makes it more convenient to work with the data, +but it can be confusing if you're not expecting it or if you just need the type to be an `Any`. For this case, you can set `typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS` to get the following: @@ -481,7 +554,7 @@ For this case, you can set `typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS` ```kotlin -val df = DataFrame.readJsonStr(text, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) +val df = DataFrame.readJsonStr(complexClashedJson, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) ``` diff --git a/docs/StardustDocs/topics/write.md b/docs/StardustDocs/topics/io/write.md similarity index 73% rename from docs/StardustDocs/topics/write.md rename to docs/StardustDocs/topics/io/write.md index 4e73818479..cbb3bdeb42 100644 --- a/docs/StardustDocs/topics/write.md +++ b/docs/StardustDocs/topics/io/write.md @@ -49,6 +49,57 @@ val jsonStr = df.toJson(prettyPrint = true) +Each row is written as an element of a JSON array: a [`ColumnGroup`](DataColumn.md#columngroup) becomes a nested +object, a [`FrameColumn`](DataColumn.md#framecolumn) becomes a nested array of objects, and a `List` value becomes +a JSON array. + +There's one exception, so that a [`DataFrame`](DataFrame.md) read from JSON with a top-level +[type clash](read.md#manage-type-clashes) can be written back to its original form. Such a clash puts the +elements that aren't objects into ["value" and "array" columns](read.md#value-and-array-columns), beside the +objects' own properties. When one of the top-level columns is detected as such a "value" or "array" column — +it's named `value` or `array` and only holds a value in rows where every other column holds none — a row is +written as the record it was read from: + +* the value of the "value" column, if the row has one; +* otherwise the array of the "array" column, if the row has one; +* otherwise an object of the remaining columns, if any of them holds a value; +* otherwise `null`. + +For example: + + + +```kotlin +DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""").toJson() +``` + +Output: + +[1,{"label":"record"},[123],null] + + + +> Only a *top-level* clash is restored like this: +> * a clash inside a [`ColumnGroup`](DataColumn.md#columngroup) isn't — the group is written as its object, so +> its "value" and "array" columns show up as ordinary properties: `[{"a":"text"},{"a":[6,7,8]}]` is written +> back as `[{"a":{"value":"text","array":null}},{"a":{"value":null,"array":[6,7,8]}}]`; +> * a single-column [`DataFrame`](DataFrame.md) isn't either, since a lone "value"/"array" column can't be told +> apart from one the user named that way. So a JSON array of non-objects, which is read into just a "value" or +> "array" column, doesn't round-trip: `[1,2,3]` is written back as +> `[{"value":1},{"value":2},{"value":3}]`. +> {style="note"} + +> Some JSON records are read into the exact same [`DataFrame`](DataFrame.md), +> so writing it back can only produce: +> * a row without any values is written as `null`, so `[1,{"label":"record"},{"label":null}]` becomes +> `[1,{"label":"record"},null]`; +> * an empty array of objects is read as an empty nested [`DataFrame`](DataFrame.md), just like no array at all, +> so `[{"label":"record"},[{"a":1}],[]]` becomes `[{"label":"record"},[{"a":1}],null]`. +> +> Arrays of primitives don't have the latter problem: they're read into a nullable `List` column, where an empty +> list and `null` stay distinct. +> {style="note"} + ### Write to Excel spreadsheet Add dependency: diff --git a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt index c0f2aa8441..304c227645 100644 --- a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt +++ b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Read.kt @@ -401,10 +401,42 @@ class Read : DataFrameSampleHelper("read", "api") { .saveDfHtmlSample() } + @Test + fun readJsonValueColumn() { + // SampleStart + val df = DataFrame.readJsonStr("""[1, 2, 3]""") + // SampleEnd + .saveDfHtmlSample() + } + + @Test + fun readJsonArrayColumn() { + // SampleStart + val df = DataFrame.readJsonStr("""[[1], [2]]""") + // SampleEnd + .saveDfHtmlSample() + } + + @Test + fun readJsonArrayOfObjectsColumn() { + // SampleStart + val df = DataFrame.readJsonStr("""[[{ "a": 1 }], [{ "a": 2 }]]""") + // SampleEnd + .saveDfHtmlSample() + } + + @Test + fun readJsonTopLevelTypeClash() { + // SampleStart + val df = DataFrame.readJsonStr("""[1, { "label": "record" }]""") + // SampleEnd + .saveDfHtmlSample() + } + @Test fun readJsonTypeClash() { // SampleStart - val text = """ + val complexClashedJson = """ [ { "a": "text" }, { "a": { "b": 2 } }, @@ -412,14 +444,14 @@ class Read : DataFrameSampleHelper("read", "api") { ] """.trimIndent() - val df = DataFrame.readJsonStr(text) + val df = DataFrame.readJsonStr(complexClashedJson) // SampleEnd .saveDfHtmlSample() } @Test fun readJsonTypeClashTactic() { - val text = """ + val complexClashedJson = """ [ { "a": "text" }, { "a": { "b": 2 } }, @@ -427,7 +459,7 @@ class Read : DataFrameSampleHelper("read", "api") { ] """.trimIndent() // SampleStart - val df = DataFrame.readJsonStr(text, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) + val df = DataFrame.readJsonStr(complexClashedJson, typeClashTactic = JSON.TypeClashTactic.ANY_COLUMNS) // SampleEnd .saveDfHtmlSample() } diff --git a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt index 6cc3311aa3..8db46529bc 100644 --- a/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt +++ b/samples/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/Write.kt @@ -6,10 +6,12 @@ import io.kotest.matchers.string.shouldStartWith import org.apache.arrow.vector.types.pojo.Schema import org.apache.poi.ss.usermodel.Sheet import org.apache.poi.ss.usermodel.WorkbookFactory +import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.remove import org.jetbrains.kotlinx.dataframe.io.ArrowWriter import org.jetbrains.kotlinx.dataframe.io.arrowWriter +import org.jetbrains.kotlinx.dataframe.io.readJsonStr import org.jetbrains.kotlinx.dataframe.io.saveArrowFeatherToByteArray import org.jetbrains.kotlinx.dataframe.io.saveArrowIPCToByteArray import org.jetbrains.kotlinx.dataframe.io.toCsvStr @@ -20,11 +22,12 @@ import org.jetbrains.kotlinx.dataframe.io.writeCsv import org.jetbrains.kotlinx.dataframe.io.writeExcel import org.jetbrains.kotlinx.dataframe.io.writeJson import org.jetbrains.kotlinx.dataframe.io.writeMismatchMessage +import org.jetbrains.kotlinx.dataframe.samples.DataFrameSampleHelper import org.junit.Test import java.io.File import kotlin.io.path.deleteExisting -class Write : TestBase { +class Write : TestBase, DataFrameSampleHelper("write", "io") { val df = peopleDf @@ -76,6 +79,14 @@ class Write : TestBase { """.trimIndent() } + @Test + fun readAndWriteJson() { + // SampleStart + DataFrame.readJsonStr("""[1,{"label":"record"},[123],null]""").toJson() + // SampleEnd + .saveSample() + } + @Test fun writeXls() { useTempFile { file ->