fix: read Japanese, Chinese and Korean text in data files, imports and SQL files

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - SAP HANA database driver plugin. (#1966)
+- Shift JIS, EUC-JP, GB 18030, Big5, EUC-KR and UTF-16 options for CSV and SQL import.
 
 ### Fixed
 
@@ -20,6 +21,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Explain Analyze running write statements on Read-Only connections and skipping the Alert and Safe Mode confirmation.
 - Remote deletions of connections, groups, tags, SSH profiles and table favorites applied with their sync category off.
 - **Local only** connections taking edits and deletions made on another device.
+- Japanese, Chinese and Korean text in CSV, TSV and SQL files opening as garbled characters.
+- Every row of a UTF-16 data file rewritten on save.
+- Latin-1 and Windows-1252 SQL dumps importing double-encoded text through their own `SET NAMES`.
+- Data file window stuck on an error after reloading it with the wrong encoding.
+- Encoding chosen in File Properties lost when another app changes the file.
+- No columns found for a JSON Lines file over 256 KB with non-ASCII text.
+- VoiceOver reading the data file status bar's delimiter and encoding as only "File format".
+- CSV import writing bytes it could not read into the table as Latin-1 text.
+- Data file window picking up another app's in-place writes on HFS+ and exFAT volumes.
+- Wrong row number and blank-looking characters in the data file save error for text the encoding cannot store.
+- Yen sign failing to save in Shift JIS and EUC-JP data files and SQL files.
+- CSV import with single quotes merging rows at a double quote inside a field.
+- Binary values in Latin-1 and Windows-1252 SQL dumps imported as different bytes.
+- Shortcuts rejecting CSV and JSON files that are not UTF-8.
 - Redis Cluster through a tunnel failing to connect with advice to set Connection Mode to Cluster.
 - Redis `SCAN` typed in a query tab showing one page of keys with no next cursor to continue from.
 - etcd `lease revoke`, `auth disable` and user or role deletion skipping confirmation, and list commands gated as writes.

Packages/TableProCore/Sources/TableProTabularIO/DelimitedDialectDetector.swift

@@ -1,34 +1,12 @@
 import Foundation
 
 public enum DelimitedDialectDetector {
-    public struct EncodingSniff: Equatable, Sendable {
-        public let encoding: TabularTextEncoding
-        public let byteOrderMarkLength: Int
-
-        public var hasByteOrderMark: Bool { byteOrderMarkLength > 0 }
-    }
-
     public static let layoutScanLimit = 65_536
-    public static let encodingScanLimit = 262_144
 
     private static let candidateDelimiters: [UInt8] = [
         DelimitedDialect.comma, DelimitedDialect.tab, DelimitedDialect.semicolon, DelimitedDialect.pipe
     ]
 
-    public static func sniffEncoding(_ prefix: UnsafeBufferPointer<UInt8>) -> EncodingSniff {
-        if hasPrefix(prefix, [0xEF, 0xBB, 0xBF]) {
-            return EncodingSniff(encoding: .utf8, byteOrderMarkLength: 3)
-        }
-        if hasPrefix(prefix, [0xFF, 0xFE]) {
-            return EncodingSniff(encoding: .utf16LittleEndian, byteOrderMarkLength: 2)
-        }
-        if hasPrefix(prefix, [0xFE, 0xFF]) {
-            return EncodingSniff(encoding: .utf16BigEndian, byteOrderMarkLength: 2)
-        }
-        let probe = UnsafeBufferPointer(rebasing: prefix[0..<min(prefix.count, encodingScanLimit)])
-        return EncodingSniff(encoding: isValidUTF8Prefix(probe) ? .utf8 : .windows1252, byteOrderMarkLength: 0)
-    }
-
     public static func detect(
         _ bytes: UnsafeBufferPointer<UInt8>,
         contentStart: Int,
@@ -128,37 +106,4 @@ public enum DelimitedDialectDetector {
         }
         return .lf
     }
-
-    private static func hasPrefix(_ bytes: UnsafeBufferPointer<UInt8>, _ prefix: [UInt8]) -> Bool {
-        guard bytes.count >= prefix.count else { return false }
-        for (offset, byte) in prefix.enumerated() where bytes[offset] != byte {
-            return false
-        }
-        return true
-    }
-
-    private static func isValidUTF8Prefix(_ bytes: UnsafeBufferPointer<UInt8>) -> Bool {
-        var end = bytes.count
-        var continuation = 0
-        while end > 0, continuation < 3, (bytes[end - 1] & 0xC0) == 0x80 {
-            end -= 1
-            continuation += 1
-        }
-        if end > 0, bytes[end - 1] >= 0xC0 {
-            end -= 1
-        }
-        let trimmed = UnsafeBufferPointer(rebasing: bytes[0..<end])
-        var iterator = trimmed.makeIterator()
-        var decoder = UTF8()
-        while true {
-            switch decoder.decode(&iterator) {
-            case .scalarValue:
-                continue
-            case .emptyInput:
-                return true
-            case .error:
-                return false
-            }
-        }
-    }
 }

Packages/TableProCore/Sources/TableProTabularIO/DelimitedFileReader.swift

@@ -0,0 +1,96 @@
+import Foundation
+
+public struct DelimitedFileContents: Sendable {
+    public let source: DelimitedSource
+    public let undecodableLineCount: Int
+
+    public var dialect: DelimitedDialect { source.dialect }
+}
+
+public enum DelimitedFileReader {
+    private static let transcodingShare = 0.3
+
+    public static func read(
+        _ original: Data,
+        fileExtension: String,
+        dialectOverride: DelimitedDialect? = nil,
+        transcodedFileURL: URL,
+        progress: (@Sendable (Double) -> Void)? = nil,
+        isCancelled: @escaping @Sendable () -> Bool = { false }
+    ) async throws -> DelimitedFileContents {
+        let sniff = TabularEncodingDetector.sniff(original)
+        let encoding = dialectOverride?.encoding ?? sniff.encoding
+        let byteOrderMarkLength = encoding == sniff.encoding ? sniff.byteOrderMarkLength : 0
+        let text = try readText(
+            original,
+            encoding: encoding,
+            byteOrderMarkLength: byteOrderMarkLength,
+            transcodedFileURL: transcodedFileURL,
+            progress: { progress?($0 * transcodingShare) },
+            isCancelled: isCancelled
+        )
+        if isCancelled() { throw TabularCancellation() }
+        let detected = text.bytes.withUnsafeBytes { raw in
+            DelimitedDialectDetector.detect(
+                raw.bindMemory(to: UInt8.self),
+                contentStart: text.contentStart,
+                encoding: encoding,
+                hasByteOrderMark: byteOrderMarkLength > 0,
+                fileExtension: fileExtension
+            )
+        }
+        let buildStart = text.origin == nil ? 0 : transcodingShare
+        let source = try await DelimitedSourceBuilder.build(
+            bytes: text.bytes,
+            dialect: dialectOverride ?? detected,
+            byteEncoding: text.byteEncoding,
+            contentStart: text.contentStart,
+            origin: text.origin,
+            progress: { progress?(buildStart + $0 * (0.95 - buildStart)) },
+            isCancelled: isCancelled
+        )
+        return DelimitedFileContents(source: source, undecodableLineCount: text.undecodableLineCount)
+    }
+
+    private struct Text {
+        let bytes: Data
+        let byteEncoding: TabularTextEncoding
+        let contentStart: Int
+        let origin: TabularTranscodingMap?
+        let undecodableLineCount: Int
+    }
+
+    private static func readText(
+        _ original: Data,
+        encoding: TabularTextEncoding,
+        byteOrderMarkLength: Int,
+        transcodedFileURL: URL,
+        progress: (Double) -> Void,
+        isCancelled: () -> Bool
+    ) throws -> Text {
+        guard !encoding.readsInPlace else {
+            return Text(
+                bytes: original,
+                byteEncoding: encoding,
+                contentStart: byteOrderMarkLength,
+                origin: nil,
+                undecodableLineCount: 0
+            )
+        }
+        let transcoded = try TabularTextTranscoder.transcode(
+            original,
+            from: encoding,
+            skippingPrefix: byteOrderMarkLength,
+            to: transcodedFileURL,
+            progress: progress,
+            isCancelled: isCancelled
+        )
+        return Text(
+            bytes: try Data(contentsOf: transcodedFileURL, options: .alwaysMapped),
+            byteEncoding: .utf8,
+            contentStart: 0,
+            origin: transcoded.map,
+            undecodableLineCount: transcoded.undecodableLineCount
+        )
+    }
+}

Packages/TableProCore/Sources/TableProTabularIO/DelimitedSource.swift

@@ -8,6 +8,7 @@ public struct DelimitedSource: TabularSource {
     public let fieldCountOfFirstRow: Int
     public let maximumFieldCount: Int
     public let raggedRowCount: Int
+    public let origin: TabularTranscodingMap?
 
     public init(
         bytes: Data,
@@ -16,7 +17,8 @@ public struct DelimitedSource: TabularSource {
         byteEncoding: TabularTextEncoding,
         fieldCountOfFirstRow: Int,
         maximumFieldCount: Int,
-        raggedRowCount: Int
+        raggedRowCount: Int,
+        origin: TabularTranscodingMap? = nil
     ) {
         self.bytes = bytes
         self.index = index
@@ -25,6 +27,7 @@ public struct DelimitedSource: TabularSource {
         self.fieldCountOfFirstRow = fieldCountOfFirstRow
         self.maximumFieldCount = maximumFieldCount
         self.raggedRowCount = raggedRowCount
+        self.origin = origin
     }
 
     public var rowCount: Int { index.rowCount }
@@ -47,7 +50,8 @@ public struct DelimitedSource: TabularSource {
             byteEncoding: byteEncoding,
             fieldCountOfFirstRow: fieldCountOfFirstRow,
             maximumFieldCount: maximumFieldCount,
-            raggedRowCount: raggedRowCount
+            raggedRowCount: raggedRowCount,
+            origin: origin
         )
     }
 
@@ -141,6 +145,7 @@ public enum DelimitedSourceBuilder {
         dialect: DelimitedDialect,
         byteEncoding: TabularTextEncoding,
         contentStart: Int,
+        origin: TabularTranscodingMap? = nil,
         progress: (@Sendable (Double) -> Void)? = nil,
         isCancelled: @escaping @Sendable () -> Bool = { false }
     ) async throws -> DelimitedSource {
@@ -168,7 +173,8 @@ public enum DelimitedSourceBuilder {
             byteEncoding: byteEncoding,
             fieldCountOfFirstRow: counts.first,
             maximumFieldCount: counts.maximum,
-            raggedRowCount: counts.ragged
+            raggedRowCount: counts.ragged,
+            origin: origin
         )
     }
 

Packages/TableProCore/Sources/TableProTabularIO/DelimitedWriter.swift

@@ -22,13 +22,36 @@ public struct DelimitedWriter {
         self.source = source
     }
 
+    private enum SourceRowCopy {
+        case sourceBytes
+        case originalBytes(TabularTranscodingMap)
+        case fields
+    }
+
     public var canCopySourceBytes: Bool {
-        guard let source else { return false }
-        return source.byteEncoding == dialect.encoding
-            && source.dialect.delimiter == dialect.delimiter
-            && source.dialect.quote == dialect.quote
-            && source.dialect.escape == dialect.escape
-            && source.dialect.lineEnding == dialect.lineEnding
+        switch sourceRowCopy {
+        case .sourceBytes, .originalBytes:
+            return true
+        case .fields:
+            return false
+        }
+    }
+
+    private var sourceRowCopy: SourceRowCopy {
+        guard let source,
+              source.dialect.delimiter == dialect.delimiter,
+              source.dialect.quote == dialect.quote,
+              source.dialect.escape == dialect.escape,
+              source.dialect.lineEnding == dialect.lineEnding else {
+            return .fields
+        }
+        if source.byteEncoding == dialect.encoding {
+            return .sourceBytes
+        }
+        if let origin = source.origin, origin.encoding == dialect.encoding {
+            return .originalBytes(origin)
+        }
+        return .fields
     }
 
     public func write<Rows: Sequence>(
@@ -53,13 +76,21 @@ public struct DelimitedWriter {
             buffer.append(contentsOf: dialect.encoding.byteOrderMark)
         }
         let lineEnding = try encodedLineEnding()
+        let copy = sourceRowCopy
+        let originalRowStarts = originalRowBoundaries(for: copy)
         var pendingTerminator = false
         var outputRow = 0
         for row in rows {
             if pendingTerminator {
                 buffer.append(contentsOf: lineEnding)
             }
-            pendingTerminator = try append(row, outputRow: outputRow, lineEnding: lineEnding, into: &buffer)
+            pendingTerminator = try append(
+                row,
+                outputRow: outputRow,
+                copy: copy,
+                originalRowStarts: originalRowStarts,
+                into: &buffer
+            )
             outputRow += 1
             if buffer.count >= Self.flushThreshold {
                 if isCancelled() { throw TabularCancellation() }
@@ -104,7 +135,8 @@ public struct DelimitedWriter {
     private func append(
         _ row: DelimitedOutputRow,
         outputRow: Int,
-        lineEnding: [UInt8],
+        copy: SourceRowCopy,
+        originalRowStarts: [Int],
         into buffer: inout [UInt8]
     ) throws -> Bool {
         switch row {
@@ -113,28 +145,58 @@ public struct DelimitedWriter {
             return true
         case .source(let sourceRow):
             guard let source else { return false }
-            return try appendSourceRow(sourceRow, of: source, outputRow: outputRow, lineEnding: lineEnding, into: &buffer)
+            return try appendSourceRow(
+                sourceRow,
+                of: source,
+                outputRow: outputRow,
+                copy: copy,
+                originalRowStarts: originalRowStarts,
+                into: &buffer
+            )
         }
     }
 
     private func appendSourceRow(
         _ sourceRow: Int,
         of source: DelimitedSource,
         outputRow: Int,
-        lineEnding: [UInt8],
+        copy: SourceRowCopy,
+        originalRowStarts: [Int],
         into buffer: inout [UInt8]
     ) throws -> Bool {
-        guard canCopySourceBytes else {
+        let range = source.rawRange(ofRow: sourceRow)
+        switch copy {
+        case .fields:
             buffer.append(contentsOf: try encodeRow(source.decodedFields(row: sourceRow), outputRow: outputRow))
             return true
+        case .sourceBytes:
+            let terminatorLength = source.bytes.withUnsafeBytes { raw -> Int in
+                let bytes = raw.bindMemory(to: UInt8.self)
+                buffer.append(contentsOf: UnsafeBufferPointer(rebasing: bytes[range]))
+                return Self.terminatorLength(in: bytes, range: range)
+            }
+            return terminatorLength == 0
+        case .originalBytes(let origin):
+            let start = originalRowStarts[sourceRow]
+            let end = max(start, originalRowStarts[sourceRow + 1])
+            guard (end - start).isMultiple(of: origin.encoding.codeUnitLayout.width) else {
+                buffer.append(contentsOf: try encodeRow(source.decodedFields(row: sourceRow), outputRow: outputRow))
+                return true
+            }
+            origin.original.withUnsafeBytes { raw in
+                buffer.append(contentsOf: UnsafeBufferPointer(rebasing: raw.bindMemory(to: UInt8.self)[start..<end]))
+            }
+            let terminatorLength = source.bytes.withUnsafeBytes { raw in
+                Self.terminatorLength(in: raw.bindMemory(to: UInt8.self), range: range)
+            }
+            return terminatorLength == 0
         }
-        let range = source.rawRange(ofRow: sourceRow)
-        let terminatorLength = source.bytes.withUnsafeBytes { raw -> Int in
-            let bytes = raw.bindMemory(to: UInt8.self)
-            buffer.append(contentsOf: UnsafeBufferPointer(rebasing: bytes[range]))
-            return Self.terminatorLength(in: bytes, range: range)
-        }
-        return terminatorLength == 0
+    }
+
+    private func originalRowBoundaries(for copy: SourceRowCopy) -> [Int] {
+        guard case .originalBytes(let origin) = copy, let source else { return [] }
+        let index = source.index
+        return origin.originalOffsets(ofLineStarts: index.rowStarts + [index.endOffset], in: source.bytes)
     }
 
     private static func terminatorLength(in bytes: UnsafeBufferPointer<UInt8>, range: Range<Int>) -> Int {